From eae3635040cf1b601fb2aeef037786a16c1f81b9 Mon Sep 17 00:00:00 2001 From: notcpuid Date: Thu, 3 Jul 2025 13:50:18 +0300 Subject: [PATCH 1/4] refactor(asmjit): remove older asmjit, add new version --- pe-packer/asmjit/a64.h | 35 +- pe-packer/asmjit/arm.h | 39 +- pe-packer/asmjit/arm/a64archtraits_p.h | 56 +- pe-packer/asmjit/arm/a64assembler.cpp | 772 +- pe-packer/asmjit/arm/a64assembler.h | 21 +- pe-packer/asmjit/arm/a64builder.cpp | 18 +- pe-packer/asmjit/arm/a64builder.h | 8 +- pe-packer/asmjit/arm/a64compiler.cpp | 22 +- pe-packer/asmjit/arm/a64compiler.h | 41 +- pe-packer/asmjit/arm/a64emithelper.cpp | 77 +- pe-packer/asmjit/arm/a64emithelper_p.h | 15 +- pe-packer/asmjit/arm/a64emitter.h | 115 +- pe-packer/asmjit/arm/a64formatter.cpp | 23 +- pe-packer/asmjit/arm/a64formatter_p.h | 2 +- pe-packer/asmjit/arm/a64func.cpp | 89 +- pe-packer/asmjit/arm/a64func_p.h | 4 +- pe-packer/asmjit/arm/a64globals.h | 46 +- pe-packer/asmjit/arm/a64instapi.cpp | 117 +- pe-packer/asmjit/arm/a64instapi_p.h | 12 +- pe-packer/asmjit/arm/a64instdb.cpp | 1750 ++-- pe-packer/asmjit/arm/a64instdb.h | 9 +- pe-packer/asmjit/arm/a64instdb_p.h | 132 +- pe-packer/asmjit/arm/a64operand.cpp | 48 +- pe-packer/asmjit/arm/a64operand.h | 1331 ++- pe-packer/asmjit/arm/a64rapass.cpp | 212 +- pe-packer/asmjit/arm/a64rapass_p.h | 11 +- pe-packer/asmjit/arm/armformatter.cpp | 327 +- pe-packer/asmjit/arm/armformatter_p.h | 10 +- pe-packer/asmjit/arm/armglobals.h | 6 +- pe-packer/asmjit/arm/armoperand.h | 626 -- pe-packer/asmjit/arm/armutils.h | 48 +- pe-packer/asmjit/asmjit-scope-begin.h | 2 +- pe-packer/asmjit/asmjit-scope-end.h | 2 +- pe-packer/asmjit/asmjit.h | 4 +- pe-packer/asmjit/core.h | 518 +- pe-packer/asmjit/core/api-build_p.h | 21 +- pe-packer/asmjit/core/api-config.h | 543 +- pe-packer/asmjit/core/archcommons.h | 199 +- pe-packer/asmjit/core/archtraits.cpp | 67 +- pe-packer/asmjit/core/archtraits.h | 73 +- pe-packer/asmjit/core/assembler.cpp | 186 +- pe-packer/asmjit/core/assembler.h | 32 +- pe-packer/asmjit/core/builder.cpp | 435 +- pe-packer/asmjit/core/builder.h | 552 +- pe-packer/asmjit/core/builder_p.h | 9 +- pe-packer/asmjit/core/codebuffer.h | 24 +- pe-packer/asmjit/core/codeholder.cpp | 987 ++- pe-packer/asmjit/core/codeholder.h | 815 +- pe-packer/asmjit/core/codewriter.cpp | 167 +- pe-packer/asmjit/core/codewriter_p.h | 72 +- pe-packer/asmjit/core/compiler.cpp | 213 +- pe-packer/asmjit/core/compiler.h | 169 +- pe-packer/asmjit/core/compilerdefs.h | 56 +- pe-packer/asmjit/core/constpool.cpp | 59 +- pe-packer/asmjit/core/constpool.h | 47 +- pe-packer/asmjit/core/cpuinfo.cpp | 1254 +-- pe-packer/asmjit/core/cpuinfo.h | 878 +- pe-packer/asmjit/core/emithelper.cpp | 110 +- pe-packer/asmjit/core/emithelper_p.h | 16 +- pe-packer/asmjit/core/emitter.cpp | 82 +- pe-packer/asmjit/core/emitter.h | 296 +- pe-packer/asmjit/core/emitterutils.cpp | 8 +- pe-packer/asmjit/core/emitterutils_p.h | 7 +- pe-packer/asmjit/core/environment.cpp | 5 +- pe-packer/asmjit/core/environment.h | 216 +- pe-packer/asmjit/core/errorhandler.cpp | 2 +- pe-packer/asmjit/core/errorhandler.h | 4 +- pe-packer/asmjit/core/fixup.h | 282 + pe-packer/asmjit/core/formatter.cpp | 171 +- pe-packer/asmjit/core/formatter.h | 38 +- pe-packer/asmjit/core/formatter_p.h | 10 +- pe-packer/asmjit/core/func.cpp | 84 +- pe-packer/asmjit/core/func.h | 797 +- pe-packer/asmjit/core/funcargscontext.cpp | 153 +- pe-packer/asmjit/core/funcargscontext_p.h | 79 +- pe-packer/asmjit/core/globals.cpp | 5 +- pe-packer/asmjit/core/globals.h | 123 +- pe-packer/asmjit/core/inst.cpp | 58 +- pe-packer/asmjit/core/inst.h | 232 +- pe-packer/asmjit/core/instdb.cpp | 142 + pe-packer/asmjit/core/instdb_p.h | 41 + pe-packer/asmjit/core/jitallocator.cpp | 609 +- pe-packer/asmjit/core/jitallocator.h | 97 +- pe-packer/asmjit/core/jitruntime.cpp | 16 +- pe-packer/asmjit/core/jitruntime.h | 28 +- pe-packer/asmjit/core/logger.cpp | 8 +- pe-packer/asmjit/core/logger.h | 32 +- pe-packer/asmjit/core/misc_p.h | 2 +- pe-packer/asmjit/core/operand.cpp | 28 +- pe-packer/asmjit/core/operand.h | 2392 ++++-- pe-packer/asmjit/core/osutils.cpp | 79 +- pe-packer/asmjit/core/osutils.h | 12 +- pe-packer/asmjit/core/osutils_p.h | 2 +- pe-packer/asmjit/core/raassignment_p.h | 66 +- pe-packer/asmjit/core/rabuilders_p.h | 131 +- pe-packer/asmjit/core/radefs_p.h | 377 +- pe-packer/asmjit/core/ralocal.cpp | 212 +- pe-packer/asmjit/core/ralocal_p.h | 115 +- pe-packer/asmjit/core/rapass.cpp | 425 +- pe-packer/asmjit/core/rapass_p.h | 420 +- pe-packer/asmjit/core/rastack.cpp | 33 +- pe-packer/asmjit/core/rastack_p.h | 50 +- pe-packer/asmjit/core/string.cpp | 233 +- pe-packer/asmjit/core/string.h | 77 +- pe-packer/asmjit/core/support.cpp | 106 +- pe-packer/asmjit/core/support.h | 1279 +-- pe-packer/asmjit/core/support_p.h | 39 +- pe-packer/asmjit/core/target.cpp | 2 +- pe-packer/asmjit/core/target.h | 8 +- pe-packer/asmjit/core/type.cpp | 78 +- pe-packer/asmjit/core/type.h | 271 +- pe-packer/asmjit/core/virtmem.cpp | 555 +- pe-packer/asmjit/core/virtmem.h | 48 +- pe-packer/asmjit/core/zone.cpp | 354 +- pe-packer/asmjit/core/zone.h | 496 +- pe-packer/asmjit/core/zonehash.cpp | 14 +- pe-packer/asmjit/core/zonehash.h | 27 +- pe-packer/asmjit/core/zonelist.cpp | 2 +- pe-packer/asmjit/core/zonelist.h | 47 +- pe-packer/asmjit/core/zonestack.cpp | 23 +- pe-packer/asmjit/core/zonestack.h | 54 +- pe-packer/asmjit/core/zonestring.h | 13 +- pe-packer/asmjit/core/zonetree.cpp | 4 +- pe-packer/asmjit/core/zonetree.h | 51 +- pe-packer/asmjit/core/zonevector.cpp | 225 +- pe-packer/asmjit/core/zonevector.h | 356 +- pe-packer/asmjit/host.h | 33 + pe-packer/asmjit/x86.h | 35 +- pe-packer/asmjit/x86/x86archtraits_p.h | 125 +- pe-packer/asmjit/x86/x86assembler.cpp | 539 +- pe-packer/asmjit/x86/x86assembler.h | 53 +- pe-packer/asmjit/x86/x86builder.cpp | 18 +- pe-packer/asmjit/x86/x86builder.h | 25 +- pe-packer/asmjit/x86/x86compiler.cpp | 22 +- pe-packer/asmjit/x86/x86compiler.h | 296 +- pe-packer/asmjit/x86/x86emithelper.cpp | 279 +- pe-packer/asmjit/x86/x86emithelper_p.h | 26 +- pe-packer/asmjit/x86/x86emitter.h | 1943 +++-- pe-packer/asmjit/x86/x86formatter.cpp | 401 +- pe-packer/asmjit/x86/x86formatter_p.h | 2 +- pe-packer/asmjit/x86/x86func.cpp | 112 +- pe-packer/asmjit/x86/x86func_p.h | 2 +- pe-packer/asmjit/x86/x86globals.h | 500 +- pe-packer/asmjit/x86/x86instapi.cpp | 654 +- pe-packer/asmjit/x86/x86instapi_p.h | 9 +- pe-packer/asmjit/x86/x86instdb.cpp | 9018 ++++++++++----------- pe-packer/asmjit/x86/x86instdb.h | 178 +- pe-packer/asmjit/x86/x86instdb_p.h | 39 +- pe-packer/asmjit/x86/x86opcode_p.h | 4 +- pe-packer/asmjit/x86/x86operand.cpp | 98 +- pe-packer/asmjit/x86/x86operand.h | 1153 +-- pe-packer/asmjit/x86/x86rapass.cpp | 454 +- pe-packer/asmjit/x86/x86rapass_p.h | 15 +- pe-packer/pe-packer.vcxproj | 15 +- pe-packer/pe-packer.vcxproj.filters | 927 +-- 155 files changed, 25660 insertions(+), 17178 deletions(-) delete mode 100644 pe-packer/asmjit/arm/armoperand.h create mode 100644 pe-packer/asmjit/core/fixup.h create mode 100644 pe-packer/asmjit/core/instdb.cpp create mode 100644 pe-packer/asmjit/core/instdb_p.h create mode 100644 pe-packer/asmjit/host.h diff --git a/pe-packer/asmjit/a64.h b/pe-packer/asmjit/a64.h index 71eff85..a2fd47b 100644 --- a/pe-packer/asmjit/a64.h +++ b/pe-packer/asmjit/a64.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_A64_H_INCLUDED @@ -26,21 +26,13 @@ //! //! ### Register Operands //! -//! - \ref arm::Reg - Base class for any AArch32/AArch64 register. -//! - \ref arm::Gp - General purpose register: -//! - \ref arm::GpW - 32-bit register. -//! - \ref arm::GpX - 64-bit register. -//! - \ref arm::Vec - Vector (SIMD) register: -//! - \ref arm::VecB - 8-bit SIMD register. -//! - \ref arm::VecH - 16-bit SIMD register. -//! - \ref arm::VecS - 32-bit SIMD register. -//! - \ref arm::VecD - 64-bit SIMD register. -//! - \ref arm::VecV - 128-bit SIMD register. +//! - \ref a64::Gp - General purpose register (abstracts 32-bit and 64-bit general purpose registers). +//! - \ref a64::Vec - Vector register (abstracts B, H, S, D, and Q NEON register with possible element type and index). //! //! ### Memory Operands //! -//! - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features -//! including base, index, pre/post increment, and ARM-specific shift addressing and index extending. +//! - \ref a64::Mem - AArch64 memory operand that provides support for all ARM addressing features including base, +//! index, pre/post increment, and ARM-specific shift addressing + index extending. //! //! ### Other //! @@ -48,13 +40,16 @@ //! - \ref arm::Utils - Utilities that can help during code generation for AArch32 and AArch64. #include "./arm.h" -#include "./arm/a64assembler.h" -#include "./arm/a64builder.h" -#include "./arm/a64compiler.h" -#include "./arm/a64emitter.h" -#include "./arm/a64globals.h" -#include "./arm/a64instdb.h" -#include "./arm/a64operand.h" + +#include "asmjit-scope-begin.h" +#include "arm/a64assembler.h" +#include "arm/a64builder.h" +#include "arm/a64compiler.h" +#include "arm/a64emitter.h" +#include "arm/a64globals.h" +#include "arm/a64instdb.h" +#include "arm/a64operand.h" +#include "asmjit-scope-end.h" #endif // ASMJIT_A64_H_INCLUDED diff --git a/pe-packer/asmjit/arm.h b/pe-packer/asmjit/arm.h index 2f2c72b..ddc1e4f 100644 --- a/pe-packer/asmjit/arm.h +++ b/pe-packer/asmjit/arm.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_H_INCLUDED @@ -18,14 +18,16 @@ //! //! ### Emitters //! -//! - AArch64 +//! - AArch32 //! - \ref a32::Assembler - AArch32 assembler (must read, provides examples). -//! - \ref a64::Assembler - AArch64 assembler (must read, provides examples). //! - \ref a32::Builder - AArch32 builder. -//! - \ref a64::Builder - AArch64 builder. //! - \ref a32::Compiler - AArch32 compiler. -//! - \ref a64::Compiler - AArch64 compiler. //! - \ref a32::Emitter - AArch32 emitter (abstract). +//! +//! - AArch64 +//! - \ref a64::Assembler - AArch64 assembler (must read, provides examples). +//! - \ref a64::Builder - AArch64 builder. +//! - \ref a64::Compiler - AArch64 compiler. //! - \ref a64::Emitter - AArch64 emitter (abstract). //! //! ### Supported Instructions @@ -46,16 +48,13 @@ //! //! ### Register Operands //! -//! - \ref arm::Reg - Base class for any AArch32/AArch64 register. -//! - \ref arm::Gp - General purpose register: -//! - \ref arm::GpW - 32-bit register. -//! - \ref arm::GpX - 64-bit register (AArch64 only). -//! - \ref arm::Vec - Vector (SIMD) register: -//! - \ref arm::VecB - 8-bit SIMD register (AArch64 only). -//! - \ref arm::VecH - 16-bit SIMD register (AArch64 only). -//! - \ref arm::VecS - 32-bit SIMD register. -//! - \ref arm::VecD - 64-bit SIMD register. -//! - \ref arm::VecV - 128-bit SIMD register. +//! - AArch32: +//! - \ref a32::Gp - 32-bit general purpose register used by AArch32: +//! - \ref a32::Vec - Vector (SIMD) register. +//! +//! - AArch64: +//! - \ref a64::Gp - 32-bit or 64-bit general purpose register used by AArch64: +//! - \ref a64::Vec - Vector (SIMD) register. //! //! ### Memory Operands //! @@ -68,9 +67,11 @@ //! - \ref arm::DataType - Data type that is part of an instruction in AArch32 mode. //! - \ref arm::Utils - Utilities that can help during code generation for AArch32 and AArch64. -#include "./core.h" -#include "./arm/armglobals.h" -#include "./arm/armoperand.h" -#include "./arm/armutils.h" +#include "core.h" + +#include "asmjit-scope-begin.h" +#include "arm/armglobals.h" +#include "arm/armutils.h" +#include "asmjit-scope-end.h" #endif // ASMJIT_ARM_H_INCLUDED diff --git a/pe-packer/asmjit/arm/a64archtraits_p.h b/pe-packer/asmjit/arm/a64archtraits_p.h index 4b5bde6..f5dea59 100644 --- a/pe-packer/asmjit/arm/a64archtraits_p.h +++ b/pe-packer/asmjit/arm/a64archtraits_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64ARCHTRAITS_P_H_INCLUDED @@ -20,18 +20,28 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) static const constexpr ArchTraits a64ArchTraits = { // SP/FP/LR/PC. - Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFF, + Gp::kIdSp, Gp::kIdFp, Gp::kIdLr, 0xFFu, // Reserved. - { 0, 0, 0 }, + { 0u, 0u, 0u }, // HW stack alignment (AArch64 requires stack aligned to 16 bytes at HW level). - 16, + 16u, - // Min/max stack offset - byte addressing is the worst, VecQ addressing the best. + // Min/max stack offset - byte addressing is the worst, vec.q addressing the best. 4095, 65520, - // Instruction hints [Gp, Vec, ExtraVirt2, ExtraVirt3]. + // Supported register types. + 0u | (1u << uint32_t(RegType::kGp32 )) + | (1u << uint32_t(RegType::kGp64 )) + | (1u << uint32_t(RegType::kVec8 )) + | (1u << uint32_t(RegType::kVec16 )) + | (1u << uint32_t(RegType::kVec32 )) + | (1u << uint32_t(RegType::kVec64 )) + | (1u << uint32_t(RegType::kVec128)) + | (1u << uint32_t(RegType::kMask )), + + // Instruction hints [Gp, Vec, Mask, Extra]. {{ InstHints::kPushPop, InstHints::kPushPop, @@ -39,29 +49,19 @@ static const constexpr ArchTraits a64ArchTraits = { InstHints::kNoHints }}, - // RegInfo. - #define V(index) OperandSignature{RegTraits::kSignature} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId(RegTraits::kTypeId) - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. - #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kARM_GpW : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kARM_GpX : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kARM_VecS : \ - index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kARM_VecD : RegType::kNone) + #define V(index) (index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt8) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt8) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt16) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt16) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt32) ? RegType::kGp32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUInt64) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kUIntPtr) ? RegType::kGp64 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat32) ? RegType::kVec32 : \ + index + uint32_t(TypeId::_kBaseStart) == uint32_t(TypeId::kFloat64) ? RegType::kVec64 : RegType::kNone) {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, #undef V diff --git a/pe-packer/asmjit/arm/a64assembler.cpp b/pe-packer/asmjit/arm/a64assembler.cpp index 444bb1f..69a4422 100644 --- a/pe-packer/asmjit/arm/a64assembler.cpp +++ b/pe-packer/asmjit/arm/a64assembler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -21,12 +21,16 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) +// a64::Assembler - Utils +// ====================== + +static ASMJIT_INLINE_CONSTEXPR uint32_t diff(RegType a, RegType b) noexcept { return uint32_t(a) - uint32_t(b); } +static ASMJIT_INLINE_CONSTEXPR uint32_t diff(VecElementType elementType, VecElementType baseType) noexcept { return uint32_t(elementType) - uint32_t(baseType); } + // a64::Assembler - Cond // ===================== -static inline uint32_t condCodeToOpcodeCond(uint32_t cond) noexcept { - return (uint32_t(cond) - 2u) & 0xFu; -} +static inline uint32_t condCodeToOpcodeCond(uint32_t cond) noexcept { return (uint32_t(cond) - 2u) & 0xFu; } // a64::Assembler - Bits // ===================== @@ -49,8 +53,19 @@ static constexpr uint32_t kWX = InstDB::kWX; static const uint8_t armShiftOpToLdStOptMap[] = { ASMJIT_LOOKUP_TABLE_16(VALUE, 0) }; #undef VALUE -static inline constexpr uint32_t diff(RegType a, RegType b) noexcept { - return uint32_t(a) - uint32_t(b); +// a64::Assembler - ExtendOpToRegType +// ================================== + +static inline RegType extendOptionToRegType(uint32_t option) noexcept { + uint32_t pred = (uint32_t(RegType::kGp32) << (0x0 * 4)) | // 0b000 - UXTB. + (uint32_t(RegType::kGp32) << (0x1 * 4)) | // 0b001 - UXTH. + (uint32_t(RegType::kGp32) << (0x2 * 4)) | // 0b010 - UXTW. + (uint32_t(RegType::kGp64) << (0x3 * 4)) | // 0b011 - UXTX|LSL. + (uint32_t(RegType::kGp32) << (0x4 * 4)) | // 0b100 - SXTB. + (uint32_t(RegType::kGp32) << (0x5 * 4)) | // 0b101 - SXTH. + (uint32_t(RegType::kGp32) << (0x6 * 4)) | // 0b110 - SXTW. + (uint32_t(RegType::kGp64) << (0x7 * 4)) ; // 0b111 - SXTX. + return RegType((pred >> (option * 4u)) & 0xFu); } // asmjit::a64::Assembler - SizeOp @@ -59,39 +74,50 @@ static inline constexpr uint32_t diff(RegType a, RegType b) noexcept { //! Struct that contains Size (2 bits), Q flag, and S (scalar) flag. These values //! are used to encode Q, Size, and Scalar fields in an opcode. struct SizeOp { - enum : uint8_t { - k128BitShift = 0, - kScalarShift = 1, - kSizeShift = 2, + //! \name Constants + //! \{ - kQ = uint8_t(1u << k128BitShift), - kS = uint8_t(1u << kScalarShift), + static inline constexpr uint8_t k128BitShift = 0; + static inline constexpr uint8_t kScalarShift = 1; + static inline constexpr uint8_t kSizeShift = 2; - k00 = uint8_t(0 << kSizeShift), - k01 = uint8_t(1 << kSizeShift), - k10 = uint8_t(2 << kSizeShift), - k11 = uint8_t(3 << kSizeShift), + static inline constexpr uint8_t kQ = uint8_t(1u << k128BitShift); + static inline constexpr uint8_t kS = uint8_t(1u << kScalarShift); - k00Q = k00 | kQ, - k01Q = k01 | kQ, - k10Q = k10 | kQ, - k11Q = k11 | kQ, + static inline constexpr uint8_t k00 = uint8_t(0 << kSizeShift); + static inline constexpr uint8_t k01 = uint8_t(1 << kSizeShift); + static inline constexpr uint8_t k10 = uint8_t(2 << kSizeShift); + static inline constexpr uint8_t k11 = uint8_t(3 << kSizeShift); - k00S = k00 | kS, - k01S = k01 | kS, - k10S = k10 | kS, - k11S = k11 | kS, + static inline constexpr uint8_t k00Q = k00 | kQ; + static inline constexpr uint8_t k01Q = k01 | kQ; + static inline constexpr uint8_t k10Q = k10 | kQ; + static inline constexpr uint8_t k11Q = k11 | kQ; - kInvalid = 0xFFu, + static inline constexpr uint8_t k00S = k00 | kS; + static inline constexpr uint8_t k01S = k01 | kS; + static inline constexpr uint8_t k10S = k10 | kS; + static inline constexpr uint8_t k11S = k11 | kS; - // Masks used by SizeOpMap. - kSzQ = (0x3u << kSizeShift) | kQ, - kSzS = (0x3u << kSizeShift) | kS, - kSzQS = (0x3u << kSizeShift) | kQ | kS - }; + static inline constexpr uint8_t kInvalid = 0xFFu; + + // Masks used by SizeOpMap. + static inline constexpr uint8_t kSzQ = (0x3u << kSizeShift) | kQ; + static inline constexpr uint8_t kSzS = (0x3u << kSizeShift) | kS; + static inline constexpr uint8_t kSzQS = (0x3u << kSizeShift) | kQ | kS; + + //! \} + + //! \name Members + //! \{ uint8_t value; + //! \} + + //! \name Accessors + //! \{ + inline bool isValid() const noexcept { return value != kInvalid; } inline void makeInvalid() noexcept { value = kInvalid; } @@ -104,6 +130,8 @@ struct SizeOp { ASMJIT_ASSERT(size() > 0); value = uint8_t(value - (1u << kSizeShift)); } + + //! \} }; struct SizeOpTable { @@ -114,29 +142,29 @@ struct SizeOpTable { }; // 40 elements for each combination. - SizeOp array[(uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB) + 1) * 8]; + SizeOp array[(uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8) + 1) * 8]; }; #define VALUE_BIN(x) { \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB )) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB )) ? SizeOp::k00Q : SizeOp::kInvalid \ + x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00Q : \ + x == (((uint32_t(RegType::kVec64 ) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : SizeOp::kInvalid \ } #define VALUE_ANY(x) { \ - x == (((uint32_t(RegType::kARM_VecB) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k00S : \ - x == (((uint32_t(RegType::kARM_VecH) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k01S : \ - x == (((uint32_t(RegType::kARM_VecS) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k10S : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeNone)) ? SizeOp::k11S : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB )) ? SizeOp::k00 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeB )) ? SizeOp::k00Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeH )) ? SizeOp::k01 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeH )) ? SizeOp::k01Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeS )) ? SizeOp::k10 : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeS )) ? SizeOp::k10Q : \ - x == (((uint32_t(RegType::kARM_VecD) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeD )) ? SizeOp::k11S : \ - x == (((uint32_t(RegType::kARM_VecV) - uint32_t(RegType::kARM_VecB)) << 3) | (Vec::kElementTypeD )) ? SizeOp::k11Q : SizeOp::kInvalid \ + x == (((uint32_t(RegType::kVec8) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k00S : \ + x == (((uint32_t(RegType::kVec16) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k01S : \ + x == (((uint32_t(RegType::kVec32) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k10S : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kNone)) ? SizeOp::k11S : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kB )) ? SizeOp::k00Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kH )) ? SizeOp::k01Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10 : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kS )) ? SizeOp::k10Q : \ + x == (((uint32_t(RegType::kVec64) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11S : \ + x == (((uint32_t(RegType::kVec128) - uint32_t(RegType::kVec8)) << 3) | uint32_t(VecElementType::kD )) ? SizeOp::k11Q : SizeOp::kInvalid \ } static const SizeOpTable sizeOpTable[SizeOpTable::kCount] = { @@ -254,21 +282,22 @@ static const Operand_& significantSimdOp(const Operand_& o0, const Operand_& o1, return !(instFlags & InstDB::kInstFlagLong) ? o0 : o1; } -static inline SizeOp armElementTypeToSizeOp(uint32_t vecOpType, RegType regType, uint32_t elementType) noexcept { +static inline SizeOp armElementTypeToSizeOp(uint32_t vecOpType, RegType regType, VecElementType elementType) noexcept { // Instruction data or Assembler is wrong if this triggers an assertion failure. ASMJIT_ASSERT(vecOpType < InstDB::kVO_Count); // ElementType uses 3 bits in the operand signature, it should never overflow. - ASMJIT_ASSERT(elementType <= 0x7u); + ASMJIT_ASSERT(uint32_t(elementType) <= 0x7u); const SizeOpMap& map = sizeOpMap[vecOpType]; const SizeOpTable& table = sizeOpTable[map.tableId]; - size_t index = (Support::min(diff(regType, RegType::kARM_VecB), diff(RegType::kARM_VecV, RegType::kARM_VecB) + 1) << 3) | elementType; + size_t index = (Support::min(diff(regType, RegType::kVec8), diff(RegType::kVec128, RegType::kVec8) + 1) << 3) | uint32_t(elementType); SizeOp op = table.array[index]; SizeOp modifiedOp { uint8_t(op.value & map.sizeOpMask) }; - if (!Support::bitTest(map.acceptMask, op.value)) + if (!Support::bitTest(map.acceptMask, op.value)) { modifiedOp.makeInvalid(); + } return modifiedOp; } @@ -348,8 +377,9 @@ static uint32_t encodeMovSequence64(uint32_t out[4], uint64_t imm, uint32_t rd, for (uint32_t hwIndex = 0; hwIndex < 4; hwIndex++, imm >>= 16) { uint32_t hwImm = uint32_t(imm & 0xFFFFu); - if (hwImm == 0) + if (hwImm == 0) { continue; + } out[count++] = op | (hwIndex << 21) | (hwImm << 5) | rd; op = kMovK; @@ -367,8 +397,9 @@ static uint32_t encodeMovSequence64(uint32_t out[4], uint64_t imm, uint32_t rd, for (uint32_t hwIndex = 0; hwIndex < 4; hwIndex++, imm >>= 16) { uint32_t hwImm = uint32_t(imm & 0xFFFFu); - if (hwImm == 0xFFFFu) + if (hwImm == 0xFFFFu) { continue; + } out[count++] = op | (hwIndex << 21) | ((hwImm ^ negMask) << 5) | rd; op = kMovK; @@ -467,7 +498,7 @@ static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const } static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, uint32_t instFlags) noexcept { - return matchSignature(o0, o1, instFlags) && o1.signature() == o2.signature() && o2.signature() == o3.signature();; + return matchSignature(o0, o1, instFlags) && o1.signature() == o2.signature() && o2.signature() == o3.signature(); } // Memory must be either: @@ -475,31 +506,30 @@ static inline bool matchSignature(const Operand_& o0, const Operand_& o1, const // 2. Relative displacement (Label). // 3. Base register + either offset or index. static inline bool armCheckMemBaseIndexRel(const Mem& mem) noexcept { - // Allowed base types (Nothing, Label, and GpX). - constexpr uint32_t kBaseMask = B(0) | - B(RegType::kLabelTag) | - B(RegType::kARM_GpX); - - // Allowed index types (Nothing, GpW, and GpX). - constexpr uint32_t kIndexMask = B(0) | - B(RegType::kARM_GpW) | - B(RegType::kARM_GpX) ; + // Allowed base types (Nothing, Label, and Gp64). + constexpr uint32_t kBaseMask = B(0) | B(RegType::kLabelTag) | B(RegType::kGp64); + // Allowed index types (Nothing, Gp32, and Gp64). + constexpr uint32_t kIndexMask = B(0) | B(RegType::kGp32) | B(RegType::kGp64) ; RegType baseType = mem.baseType(); RegType indexType = mem.indexType(); - if (!Support::bitTest(kBaseMask, baseType)) + if (!Support::bitTest(kBaseMask, baseType)) { return false; + } if (baseType > RegType::kLabelTag) { - // Index allows either GpW or GpX. - if (!Support::bitTest(kIndexMask, indexType)) + // Index allows either Gp32 or Gp64. + if (!Support::bitTest(kIndexMask, indexType)) { return false; + } - if (indexType == RegType::kNone) + if (indexType == RegType::kNone) { return true; - else + } + else { return !mem.hasOffset(); + } } else { // No index register allowed if this is a PC relative address (literal). @@ -526,9 +556,10 @@ static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint if (!reg.hasElementType()) { // Scalar operation [HSD]. - uint32_t sz = diff(reg.type(), RegType::kARM_VecH); - if (sz > 2u || !Support::bitTest(szBits[sHf].sizeMask, sz)) + uint32_t sz = diff(reg.regType(), RegType::kVec16); + if (sz > 2u || !Support::bitTest(szBits[sHf].sizeMask, sz)) { return false; + } opcode->reset(szBits[sHf].mask[sz] ^ sOp); *szOut = sz; @@ -536,11 +567,12 @@ static inline bool pickFpOpcode(const Vec& reg, uint32_t sOp, uint32_t sHf, uint } else { // Vector operation [HSD]. - uint32_t q = diff(reg.type(), RegType::kARM_VecD); - uint32_t sz = reg.elementType() - Vec::kElementTypeH; + uint32_t q = diff(reg.regType(), RegType::kVec64); + uint32_t sz = diff(reg.elementType(), VecElementType::kH); - if (q > 1u || sz > 2u || !Support::bitTest(szBits[vHf].sizeMask, sz)) + if (q > 1u || sz > 2u || !Support::bitTest(szBits[vHf].sizeMask, sz)) { return false; + } opcode->reset(szBits[vHf].mask[sz] ^ (vOp | (q << kQBitIndex))); *szOut = sz; @@ -574,22 +606,21 @@ static inline bool checkSignature(const Operand_& o0, const Operand_& o1, const // Checks whether the register is GP register of the allowed types. // -// Allowed is a 2-bit mask, where the first bits allows GpW and the second bit -// allows GpX. These bits are usually stored within the instruction, but could -// be also hardcoded in the assembler for instructions where GP types are not -// selectable. +// Allowed is a 2-bit mask, where the first bits allows Gp32 and the second bit allows Gp64. These bits are usually +// stored within the instruction, but could be also hardcoded in the assembler for instructions where GP types are +// not selectable. static inline bool checkGpType(const Operand_& op, uint32_t allowed) noexcept { - RegType type = op.as().type(); - return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type); + RegType type = op.as().regType(); + return Support::bitTest(allowed << uint32_t(RegType::kGp32), type); } static inline bool checkGpType(const Operand_& op, uint32_t allowed, uint32_t* x) noexcept { - // NOTE: We set 'x' to one only when GpW is allowed, otherwise the X is part + // NOTE: We set 'x' to one only when Gp32 is allowed, otherwise the X is part // of the opcode and we cannot set it. This is why this works without requiring // additional logic. - RegType type = op.as().type(); - *x = diff(type, RegType::kARM_GpW) & allowed; - return Support::bitTest(allowed << uint32_t(RegType::kARM_GpW), type); + RegType type = op.as().regType(); + *x = diff(type, RegType::kGp32) & allowed; + return Support::bitTest(allowed << uint32_t(RegType::kGp32), type); } static inline bool checkGpType(const Operand_& o0, const Operand_& o1, uint32_t allowed, uint32_t* x) noexcept { @@ -652,7 +683,7 @@ static inline bool checkVecId(const Operand_& o0, const Operand_& o1, const Oper */ static inline bool checkMemBase(const Mem& mem) noexcept { - return mem.baseType() == RegType::kARM_GpX && mem.baseId() <= 31; + return mem.baseType() == RegType::kGp64 && mem.baseId() <= 31; } static inline bool checkEven(const Operand_& o0, const Operand_& o1) noexcept { @@ -677,38 +708,38 @@ static inline bool checkConsecutive(const Operand_& o0, const Operand_& o1, cons // a64::Assembler - CheckReg // ========================= -#define V(index) (index == uint32_t(RegType::kARM_GpW) ? Gp::kIdZr : \ - index == uint32_t(RegType::kARM_GpX) ? Gp::kIdZr : \ - index == uint32_t(RegType::kARM_VecB) ? 31u : \ - index == uint32_t(RegType::kARM_VecH) ? 31u : \ - index == uint32_t(RegType::kARM_VecS) ? 31u : \ - index == uint32_t(RegType::kARM_VecD) ? 31u : \ - index == uint32_t(RegType::kARM_VecV) ? 31u : 0) +#define V(index) (index == uint32_t(RegType::kGp32) ? Gp::kIdZr : \ + index == uint32_t(RegType::kGp64) ? Gp::kIdZr : \ + index == uint32_t(RegType::kVec8) ? 31u : \ + index == uint32_t(RegType::kVec16) ? 31u : \ + index == uint32_t(RegType::kVec32) ? 31u : \ + index == uint32_t(RegType::kVec64) ? 31u : \ + index == uint32_t(RegType::kVec128) ? 31u : 0) static const Support::Array commonHiRegIdOfType = {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}; #undef V static inline bool checkValidRegs(const Operand_& o0) noexcept { - return bool(unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])); + return bool(unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()]))); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()])) & - (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()])) & + (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().regType()]))); } static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) noexcept { - return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().type()])) & - (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().type()])) & - (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().type()])) & - (unsigned(o3.id() < 31) | unsigned(o3.id() == commonHiRegIdOfType[o3.as().type()]))); + return bool((unsigned(o0.id() < 31) | unsigned(o0.id() == commonHiRegIdOfType[o0.as().regType()])) & + (unsigned(o1.id() < 31) | unsigned(o1.id() == commonHiRegIdOfType[o1.as().regType()])) & + (unsigned(o2.id() < 31) | unsigned(o2.id() == commonHiRegIdOfType[o2.as().regType()])) & + (unsigned(o3.id() < 31) | unsigned(o3.id() == commonHiRegIdOfType[o3.as().regType()]))); } // a64::Assembler - Construction & Destruction @@ -716,10 +747,11 @@ static inline bool checkValidRegs(const Operand_& o0, const Operand_& o1, const Assembler::Assembler(CodeHolder* code) noexcept : BaseAssembler() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); - assignEmitterFuncs(this); + initEmitterFuncs(this); - if (code) + if (code) { code->attach(this); + } } Assembler::~Assembler() noexcept {} @@ -760,8 +792,9 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co CondCode instCC = BaseInst::extractARMCondCode(instId); instId = instId & uint32_t(InstIdParts::kRealId); - if (instId >= Inst::_kIdCount) + if (instId >= Inst::_kIdCount) { instId = 0; + } const InstDB::InstInfo* instInfo = &InstDB::_instInfoTable[instId]; uint32_t encodingIndex = instInfo->_encodingDataIndex; @@ -781,21 +814,25 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co uint64_t offsetValue; // Offset value (if known). if (ASMJIT_UNLIKELY(Support::test(options, kRequiresSpecialHandling))) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } // Unknown instruction. - if (ASMJIT_UNLIKELY(instId == 0)) + if (ASMJIT_UNLIKELY(instId == 0)) { goto InvalidInstruction; + } // Condition code can only be used with 'B' instruction. - if (ASMJIT_UNLIKELY(instCC != CondCode::kAL && instId != Inst::kIdB)) + if (ASMJIT_UNLIKELY(instCC != CondCode::kAL && instId != Inst::kIdB)) { goto InvalidInstruction; + } // Grow request, happens rarely. err = writer.ensureSpace(this, 4); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { goto Failed; + } #ifndef ASMJIT_NO_VALIDATION // Strict validation. @@ -803,9 +840,10 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co Operand_ opArray[Globals::kMaxOpCount]; EmitterUtils::opArrayFromEmitArgs(opArray, o0, o1, o2, opExt); - err = _funcs.validate(arch(), BaseInst(instId, options, _extraReg), opArray, Globals::kMaxOpCount, ValidationFlags::kNone); - if (ASMJIT_UNLIKELY(err)) + err = _funcs.validate(BaseInst(instId, options, _extraReg), opArray, Globals::kMaxOpCount, ValidationFlags::kNone); + if (ASMJIT_UNLIKELY(err)) { goto Failed; + } } #endif } @@ -833,6 +871,17 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co break; } + case InstDB::kEncodingBaseOpX16: { + const InstDB::EncodingData::BaseOpX16& opData = InstDB::EncodingData::baseOpX16[encodingIndex]; + + if (isign4 == ENC_OPS1(Reg) && o0.as().isGp64(16)) { + opcode.reset(opData.opcode); + goto EmitOp; + } + + break; + } + case InstDB::kEncodingBaseOpImm: { const InstDB::EncodingData::BaseOpImm& opData = InstDB::EncodingData::baseOpImm[encodingIndex]; @@ -1024,7 +1073,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMov: { // MOV is a pseudo instruction that uses various instructions depending on its signature. - uint32_t x = diff(o0.as().type(), RegType::kARM_GpW); + uint32_t x = diff(o0.as().regType(), RegType::kGp32); if (x > 1) goto InvalidInstruction; @@ -1103,7 +1152,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMovKNZ: { const InstDB::EncodingData::BaseMovKNZ& opData = InstDB::EncodingData::baseMovKNZ[encodingIndex]; - uint32_t x = diff(o0.as().type(), RegType::kARM_GpW); + uint32_t x = diff(o0.as().regType(), RegType::kGp32); if (x > 1) goto InvalidInstruction; @@ -1157,7 +1206,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const InstDB::EncodingData::BaseAdr& opData = InstDB::EncodingData::baseAdr[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Label) || isign4 == ENC_OPS2(Reg, Imm)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -1230,9 +1279,6 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co } if (isign4 == ENC_OPS3(Reg, Reg, Reg) || isign4 == ENC_OPS4(Reg, Reg, Reg, Imm)) { - if (!checkSignature(o1, o2)) - goto InvalidInstruction; - uint32_t opSize = x ? 64 : 32; uint64_t shift = 0; uint32_t sType = uint32_t(ShiftOp::kLSL); @@ -1249,11 +1295,17 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (sType <= uint32_t(ShiftOp::kASR)) { bool hasSP = o0.as().isSP() || o1.as().isSP(); if (!hasSP) { - if (!checkGpId(o0, o1, kZR)) - goto InvalidPhysId; + if (!checkSignature(o1, o2)) { + goto InvalidInstruction; + } - if (shift >= opSize) + if (!checkGpId(o0, o1, kZR)) { + goto InvalidPhysId; + } + + if (shift >= opSize) { goto InvalidImmediate; + } opcode.reset(uint32_t(opData.shiftedOp) << 21); opcode.addImm(x, 31); @@ -1266,8 +1318,10 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co } // SP register can only be used with LSL or Extend. - if (sType != uint32_t(ShiftOp::kLSL)) + if (sType != uint32_t(ShiftOp::kLSL)) { goto InvalidImmediate; + } + sType = x ? uint32_t(ShiftOp::kUXTX) : uint32_t(ShiftOp::kUXTW); } @@ -1275,8 +1329,9 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co opcode.reset(uint32_t(opData.extendedOp) << 21); sType -= uint32_t(ShiftOp::kUXTB); - if (sType > 7 || shift > 4) + if (sType > 7 || shift > 4) { goto InvalidImmediate; + } if (!(opcode.get() & B(29))) { // ADD|SUB (extend) - ZR is not allowed. @@ -1289,6 +1344,11 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co goto InvalidPhysId; } + // Validate whether the register operands match extend option. + if (o2.as().regType() != extendOptionToRegType(sType) || o1.as().regType() < o2.as().regType()) { + goto InvalidInstruction; + } + opcode.addImm(x, 31); opcode.addReg(o2, 16); opcode.addImm(sType, 13); @@ -1414,9 +1474,6 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co } if (isign4 == ENC_OPS2(Reg, Reg) || isign4 == ENC_OPS3(Reg, Reg, Imm)) { - if (!checkSignature(o0, o1)) - goto InvalidInstruction; - uint32_t opSize = x ? 64 : 32; uint32_t sType = 0; uint64_t shift = 0; @@ -1431,8 +1488,13 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co // Shift operation - LSL, LSR, ASR. if (sType <= uint32_t(ShiftOp::kASR)) { if (!hasSP) { - if (shift >= opSize) + if (!checkSignature(o0, o1)) { + goto InvalidInstruction; + } + + if (shift >= opSize) { goto InvalidImmediate; + } opcode.reset(uint32_t(opData.shiftedOp) << 21); opcode.addImm(x, 31); @@ -1453,8 +1515,14 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co // Extend operation - UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX. sType -= uint32_t(ShiftOp::kUXTB); - if (sType > 7 || shift > 4) + if (sType > 7 || shift > 4) { goto InvalidImmediate; + } + + // Validate whether the register operands match extend option. + if (o1.as().regType() != extendOptionToRegType(sType) || o0.as().regType() < o1.as().regType()) { + goto InvalidInstruction; + } opcode.reset(uint32_t(opData.extendedOp) << 21); opcode.addImm(x, 31); @@ -1724,7 +1792,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (!checkGpType(o0, opData.rType, &x)) goto InvalidInstruction; - if (!o1.as().isGpW()) + if (!o1.as().isGp32()) goto InvalidInstruction; if (!checkGpId(o0, o1)) @@ -1998,6 +2066,63 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co break; } + // ------------------------------------------------------------------------ + // [Base - Min/Max] + // ------------------------------------------------------------------------ + + case InstDB::kEncodingBaseMinMax: { + const InstDB::EncodingData::BaseMinMax& opData = InstDB::EncodingData::baseMinMax[encodingIndex]; + + if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { + uint32_t x; + if (!checkGpType(o0, InstDB::kWX, &x)) + goto InvalidInstruction; + + if (!checkSignature(o0, o1, o2)) + goto InvalidInstruction; + + opcode.reset(opData.regOp); + opcode.addImm(x, 31); + opcode.addReg(o2, 16); + opcode.addReg(o1, 5); + opcode.addReg(o0, 0); + goto EmitOp; + } + + if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { + uint32_t x; + if (!checkGpType(o0, InstDB::kWX, &x)) + goto InvalidInstruction; + + if (!checkSignature(o0, o1)) + goto InvalidInstruction; + + uint64_t imm = o2.as().valueAs(); + + if (opData.immOp & (1u << 18)) { + // Zero extend imm. + if (!Support::isUInt8(imm)) { + goto InvalidImmediate; + } + } + else { + // Sign extend imm. + if (!Support::isInt8(int64_t(imm))) { + goto InvalidImmediate; + } + } + + opcode.reset(opData.immOp); + opcode.addImm(x, 31); + opcode.addImm(uint32_t(imm & 0xFFu), 10); + opcode.addReg(o1, 5); + opcode.addReg(o0, 0); + goto EmitOp; + } + + break; + } + // ------------------------------------------------------------------------ // [Base - Special] // ------------------------------------------------------------------------ @@ -2018,7 +2143,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co uint32_t rt = 31; if (o1.isReg()) { - if (!o1.as().isGpX()) + if (!o1.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o1, kZR)) @@ -2037,7 +2162,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMrs: { if (isign4 == ENC_OPS2(Reg, Imm)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -2061,7 +2186,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co case InstDB::kEncodingBaseMsr: { if (isign4 == ENC_OPS2(Imm, Reg)) { - if (!o1.as().isGpX()) + if (!o1.as().isGp64()) goto InvalidInstruction; if (o0.as().valueAs() > 0xFFFFu) @@ -2119,7 +2244,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Operand_& o4 = opExt[EmitterUtils::kOp4]; if (o4.isReg()) { - if (!o4.as().isGpX()) + if (!o4.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o4, kZR)) @@ -2151,7 +2276,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const InstDB::EncodingData::BaseBranchReg& opData = InstDB::EncodingData::baseBranchReg[encodingIndex]; if (isign4 == ENC_OPS1(Reg)) { - if (!o0.as().isGpX()) + if (!o0.as().isGp64()) goto InvalidInstruction; if (!checkGpId(o0, kZR)) @@ -2172,7 +2297,13 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co opcode.reset(opData.opcode); rmRel = &o0; - if (instCC != CondCode::kAL) { + // A variation that uses Cond code (or where Cond code is forced like BC.). + if (instCC != CondCode::kAL || Support::bitTest(opcode.v, 30)) { + if (opcode.hasX()) { + // Condition code cannot be applied when the instruction has X bit set (this would be BL instruction). + goto InvalidInstruction; + } + opcode |= B(30); opcode.addImm(condCodeToOpcodeCond(uint32_t(instCC)), 0); offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2); @@ -2241,6 +2372,86 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co break; } + // ------------------------------------------------------------------------ + // [Base - Prefetch] + // ------------------------------------------------------------------------ + + case InstDB::kEncodingBasePrfm: { + const InstDB::EncodingData::BasePrfm& opData = InstDB::EncodingData::basePrfm[encodingIndex]; + + if (isign4 == ENC_OPS2(Imm, Mem)) { + const Mem& m = o1.as(); + rmRel = &m; + + uint32_t immShift = 3u; + + if (o0.as().valueAs() > 0x1Fu) + goto InvalidImmediate; + + if (!armCheckMemBaseIndexRel(m)) + goto InvalidAddress; + + int64_t offset = m.offset(); + uint32_t prfop = o0.as().valueAs(); + + if (m.hasBaseReg()) { + // [Base {Offset | Index}] + if (m.hasIndex()) { + uint32_t opt = armShiftOpToLdStOptMap[size_t(m.shiftOp())]; + if (opt == 0xFF) + goto InvalidAddress; + + uint32_t shift = m.shift(); + uint32_t s = shift != 0; + + if (s && shift != immShift) + goto InvalidAddressScale; + + opcode.reset(uint32_t(opData.registerOp) << 21); + opcode.addImm(opt, 13); + opcode.addImm(s, 12); + opcode |= B(11); + opcode.addImm(prfop, 0); + goto EmitOp_MemBaseIndex_Rn5_Rm16; + } + + if (!Support::isInt32(offset)) + goto InvalidDisplacement; + + int32_t offset32 = int32_t(offset); + + if (m.isPreOrPost()) + goto InvalidAddress; + + uint32_t imm12 = uint32_t(offset32) >> immShift; + + if (Support::isUInt12(imm12) && (imm12 << immShift) == uint32_t(offset32)) { + opcode.reset(uint32_t(opData.sOffsetOp) << 22); + opcode.addImm(imm12, 10); + opcode.addImm(prfop, 0); + goto EmitOp_MemBase_Rn5; + } + + if (Support::isInt9(offset32)) { + opcode.reset(uint32_t(opData.uOffsetOp) << 21); + opcode.addImm(uint32_t(offset32) & 0x1FFu, 12); + opcode.addImm(prfop, 0); + goto EmitOp_MemBase_Rn5; + } + + goto InvalidAddress; + } + else { + opcode.reset(uint32_t(opData.literalOp) << 24); + opcode.addImm(prfop, 0); + offsetFormat.resetToImmValue(OffsetType::kSignedOffset, 4, 5, 19, 2); + goto EmitOp_Rel; + } + } + + break; + } + // ------------------------------------------------------------------------ // [Base - Load / Store] // ------------------------------------------------------------------------ @@ -2272,7 +2483,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (m.hasBaseReg()) { // [Base {Offset | Index}] if (m.hasIndex()) { - uint32_t opt = armShiftOpToLdStOptMap[m.predicate()]; + uint32_t opt = armShiftOpToLdStOptMap[size_t(m.shiftOp())]; if (opt == 0xFF) goto InvalidAddress; @@ -2357,7 +2568,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co if (!checkGpId(o0, o1, kZR)) goto InvalidPhysId; - if (m.baseType() != RegType::kARM_GpX || m.hasIndex()) + if (m.baseType() != RegType::kGp64 || m.hasIndex()) goto InvalidAddress; if (m.isOffset64Bit()) @@ -2402,7 +2613,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Mem& m = o2.as(); uint32_t x; - if (!o0.as().isGpW() || !checkGpType(o1, opData.rType, &x)) + if (!o0.as().isGp32() || !checkGpType(o1, opData.rType, &x)) goto InvalidInstruction; if (!checkGpId(o0, o1, kZR)) @@ -2452,7 +2663,7 @@ Error Assembler::_emit(InstId instId, const Operand_& o0, const Operand_& o1, co const Mem& m = o3.as(); uint32_t x; - if (!o0.as().isGpW() || !checkGpType(o1, opData.rType, &x) || !checkSignature(o1, o2)) + if (!o0.as().isGp32() || !checkGpType(o1, opData.rType, &x) || !checkSignature(o1, o2)) goto InvalidInstruction; if (!checkGpId(o0, o1, o2, kZR)) @@ -2679,7 +2890,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::FSimdSV& opData = InstDB::EncodingData::fSimdSV[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg)) { - uint32_t q = diff(o1.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o1.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -2689,8 +2900,8 @@ Case_BaseLdurStur: // This operation is only defined for: // hD, vS.{4|8}h (16-bit) // sD, vS.4s (32-bit) - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); - uint32_t elementSz = o1.as().elementType() - Vec::kElementTypeH; + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); + uint32_t elementSz = diff(o1.as().elementType(), VecElementType::kH); // Size greater than 1 means 64-bit elements, not supported. if ((sz | elementSz) > 1 || sz != elementSz) @@ -2761,7 +2972,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, instFlags)) goto InvalidInstruction; - uint32_t q = o1.as().isVecQ(); + uint32_t q = o1.as().isVec128(); uint32_t sz; if (!pickFpOpcode(o0.as(), opData.elementScalarOp(), InstDB::kHF_D, opData.elementVectorOp(), InstDB::kHF_D, &opcode, &sz)) @@ -2808,11 +3019,11 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2) || o0.as().hasElementIndex()) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; - uint32_t sz = o0.as().elementType() - Vec::kElementTypeB; + uint32_t sz = diff(o0.as().elementType(), VecElementType::kB); if (sz == 0 || sz > 3) goto InvalidInstruction; @@ -2838,7 +3049,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdFccmpFccmpe& opData = InstDB::EncodingData::simdFccmpFccmpe[encodingIndex]; if (isign4 == ENC_OPS4(Reg, Reg, Imm, Imm)) { - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; @@ -2900,11 +3111,11 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1)) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; - uint32_t sz = o0.as().elementType() - Vec::kElementTypeB; + uint32_t sz = diff(o0.as().elementType(), VecElementType::kB); if (sz == 0 || sz > 3) goto InvalidInstruction; @@ -2965,7 +3176,7 @@ Case_BaseLdurStur: case InstDB::kEncodingSimdFcmpFcmpe: { const InstDB::EncodingData::SimdFcmpFcmpe& opData = InstDB::EncodingData::simdFcmpFcmpe[encodingIndex]; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; if (sz > 2) @@ -3000,7 +3211,7 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2)) goto InvalidInstruction; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; if (sz > 2 || o0.as().hasElementType()) @@ -3021,8 +3232,8 @@ Case_BaseLdurStur: case InstDB::kEncodingSimdFcvt: { if (isign4 == ENC_OPS2(Reg, Reg)) { - uint32_t dstSz = diff(o0.as().type(), RegType::kARM_VecH); - uint32_t srcSz = diff(o1.as().type(), RegType::kARM_VecH); + uint32_t dstSz = diff(o0.as().regType(), RegType::kVec16); + uint32_t srcSz = diff(o1.as().regType(), RegType::kVec16); if ((dstSz | srcSz) > 3) goto InvalidInstruction; @@ -3065,7 +3276,7 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { // Scalar form - only FCVTXN. - if (o0.as().isVecS() && o1.as().isVecD()) { + if (o0.as().isVec32() && o1.as().isVec64()) { if (!opData.hasScalar()) goto InvalidInstruction; @@ -3082,15 +3293,15 @@ Case_BaseLdurStur: const Vec& rL = (instFlags & InstDB::kInstFlagLong) ? o0.as() : o1.as(); const Vec& rN = (instFlags & InstDB::kInstFlagLong) ? o1.as() : o0.as(); - uint32_t q = diff(rN.type(), RegType::kARM_VecD); + uint32_t q = diff(rN.regType(), RegType::kVec64); if (uint32_t(opcode.hasQ()) != q) goto InvalidInstruction; - if (rL.isVecS4() && rN.elementType() == Vec::kElementTypeH && !opData.isCvtxn()) { + if (rL.isVecS4() && rN.elementType() == VecElementType::kH && !opData.isCvtxn()) { goto EmitOp_Rd0_Rn5; } - if (rL.isVecD2() && rN.elementType() == Vec::kElementTypeS) { + if (rL.isVecD2() && rN.elementType() == VecElementType::kS) { opcode |= B(22); goto EmitOp_Rd0_Rn5; } @@ -3108,8 +3319,8 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { if (oGp.as().isGp() && oVec.as().isVec()) { - uint32_t x = oGp.as().isGpX(); - uint32_t type = diff(oVec.as().type(), RegType::kARM_VecH); + uint32_t x = oGp.as().isGp64(); + uint32_t type = diff(oVec.as().regType(), RegType::kVec16); if (type > 2u) goto InvalidInstruction; @@ -3141,8 +3352,8 @@ Case_BaseLdurStur: goto InvalidInstruction; if (oGp.as().isGp() && oVec.as().isVec()) { - uint32_t x = oGp.as().isGpX(); - uint32_t type = diff(oVec.as().type(), RegType::kARM_VecH); + uint32_t x = oGp.as().isGp64(); + uint32_t type = diff(oVec.as().regType(), RegType::kVec16); uint32_t scaleLimit = 32u << x; if (scale > scaleLimit) @@ -3181,7 +3392,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdFmlal& opData = InstDB::EncodingData::simdFmlal[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); uint32_t qIsOptional = opData.optionalQ(); if (qIsOptional) { @@ -3200,9 +3411,9 @@ Case_BaseLdurStur: q = 0; } - if (uint32_t(o0.as().type()) != uint32_t(o1.as().type()) + qIsOptional || - o0.as().elementType() != opData.tA || - o1.as().elementType() != opData.tB) + if (uint32_t(o0.as().regType()) != uint32_t(o1.as().regType()) + qIsOptional || + uint32_t(o0.as().elementType()) != opData.tA || + uint32_t(o1.as().elementType()) != opData.tB) goto InvalidInstruction; if (!o2.as().hasElementIndex()) { @@ -3214,7 +3425,7 @@ Case_BaseLdurStur: goto EmitOp_Rd0_Rn5_Rm16; } else { - if (o2.as().elementType() != opData.tElement) + if (uint32_t(o2.as().elementType()) != opData.tElement) goto InvalidInstruction; if (o2.as().id() > 15) @@ -3246,8 +3457,8 @@ Case_BaseLdurStur: // FMOV Wd, Sn (sf=0 type=00 rmode=00 op=110) // FMOV Xd, Dn (sf=1 type=11 rmode=00 op=110) // FMOV Xd, Vn.d[1] (sf=1 type=10 rmode=01 op=110) - uint32_t x = o0.as().isGpX(); - uint32_t sz = diff(o1.as().type(), RegType::kARM_VecH); + uint32_t x = o0.as().isGp64(); + uint32_t sz = diff(o1.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; uint32_t rModeOp = 0b00110; @@ -3267,10 +3478,10 @@ Case_BaseLdurStur: if (o1.as().hasElementType()) goto InvalidInstruction; - if (o1.as().isVecS() && x) + if (o1.as().isVec32() && x) goto InvalidInstruction; - if (o1.as().isVecD() && !x) + if (o1.as().isVec64() && !x) goto InvalidInstruction; } @@ -3286,8 +3497,8 @@ Case_BaseLdurStur: // FMOV Sd, Wn (sf=0 type=00 rmode=00 op=111) // FMOV Dd, Xn (sf=1 type=11 rmode=00 op=111) // FMOV Vd.d[1], Xn (sf=1 type=10 rmode=01 op=111) - uint32_t x = o1.as().isGpX(); - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t x = o1.as().isGp64(); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1) & 0x3u; uint32_t rModeOp = 0b00111; @@ -3307,10 +3518,10 @@ Case_BaseLdurStur: if (o0.as().hasElementType()) goto InvalidInstruction; - if (o0.as().isVecS() && x) + if (o0.as().isVec32() && x) goto InvalidInstruction; - if (o0.as().isVecD() && !x) + if (o0.as().isVec64() && !x) goto InvalidInstruction; } @@ -3321,7 +3532,7 @@ Case_BaseLdurStur: } if (checkSignature(o0, o1)) { - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; @@ -3351,7 +3562,7 @@ Case_BaseLdurStur: uint32_t imm8 = Utils::encodeFP64ToImm8(fpValue); if (!o0.as().hasElementType()) { // FMOV (scalar, immediate). - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); uint32_t type = (sz - 1u) & 0x3u; if (sz > 2) @@ -3363,8 +3574,8 @@ Case_BaseLdurStur: goto EmitOp_Rd0; } else { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); - uint32_t sz = o0.as().elementType() - Vec::kElementTypeH; + uint32_t q = diff(o0.as().regType(), RegType::kVec64); + uint32_t sz = diff(o0.as().elementType(), VecElementType::kH); if (q > 1 || sz > 2) goto InvalidInstruction; @@ -3391,14 +3602,14 @@ Case_BaseLdurStur: // hD, vS.2h (16-bit) // sD, vS.2s (32-bit) // dD, vS.2d (64-bit) - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecH); + uint32_t sz = diff(o0.as().regType(), RegType::kVec16); if (sz > 2) goto InvalidInstruction; static const uint32_t szSignatures[3] = { - VecS::kSignature | (Vec::kSignatureElementH), - VecD::kSignature | (Vec::kSignatureElementS), - VecV::kSignature | (Vec::kSignatureElementD) + RegTraits::kSignature | (Vec::kSignatureElementH), + RegTraits::kSignature | (Vec::kSignatureElementS), + RegTraits::kSignature | (Vec::kSignatureElementD) }; if (o1.signature() != szSignatures[sz]) @@ -3414,11 +3625,11 @@ Case_BaseLdurStur: if (!checkSignature(o0, o1, o2)) goto InvalidInstruction; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; - uint32_t sz = o0.as().elementType() - Vec::kElementTypeH; + uint32_t sz = diff(o0.as().elementType(), VecElementType::kH); if (sz > 2) goto InvalidInstruction; @@ -3442,10 +3653,10 @@ Case_BaseLdurStur: if (isign4 == ENC_OPS2(Reg, Reg)) { // The first destination operand is scalar, which matches element-type of source vectors. uint32_t L = (instFlags & InstDB::kInstFlagLong) != 0; - if (diff(o0.as().type(), RegType::kARM_VecB) != o1.as().elementType() - Vec::kElementTypeB + L) + if (diff(o0.as().regType(), RegType::kVec8) != diff(o1.as().elementType(), VecElementType::kB) + L) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3466,7 +3677,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3503,7 +3714,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3538,11 +3749,11 @@ Case_BaseLdurStur: const InstDB::EncodingData::ISimdWWV& opData = InstDB::EncodingData::iSimdWWV[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o2.as().type(), o2.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o2.as().regType(), o2.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; - if (!checkSignature(o0, o1) || !o0.as().isVecV() || o0.as().elementType() != o2.as().elementType() + 1) + if (!checkSignature(o0, o1) || !o0.as().isVec128() || uint32_t(o0.as().elementType()) != uint32_t(o2.as().elementType()) + 1u) goto InvalidInstruction; opcode.reset(opData.opcode()); @@ -3564,7 +3775,7 @@ Case_BaseLdurStur: goto InvalidInstruction; if (!o2.as().hasElementIndex()) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.regularVecType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.regularVecType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3578,7 +3789,7 @@ Case_BaseLdurStur: goto EmitOp_Rd0_Rn5_Rm16; } else { - SizeOp sizeOp = armElementTypeToSizeOp(opData.elementVecType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.elementVecType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3611,7 +3822,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3644,7 +3855,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, o3, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3691,7 +3902,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.opType3, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.opType3, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3712,7 +3923,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_B, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_B, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3722,7 +3933,7 @@ Case_BaseLdurStur: } if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_HS, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_HS, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3778,7 +3989,7 @@ Case_BaseLdurStur: if (!matchSignature(o0, o1, o2, instFlags)) goto InvalidInstruction; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3796,7 +4007,7 @@ Case_BaseLdurStur: if (o2.as().value() != 0) goto InvalidImmediate; - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -3814,7 +4025,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdDot& opData = InstDB::EncodingData::simdDot[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Reg)) { - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); uint32_t size = 2; if (q > 1u) @@ -3824,12 +4035,12 @@ Case_BaseLdurStur: if (!opData.vectorOp) goto InvalidInstruction; - if (o0.as().type() != o1.as().type() || o1.as().type() != o2.as().type()) + if (o0.as().regType() != o1.as().regType() || o1.as().regType() != o2.as().regType()) goto InvalidInstruction; - if (o0.as().elementType() != opData.tA || - o1.as().elementType() != opData.tB || - o2.as().elementType() != opData.tB) + if (uint32_t(o0.as().elementType()) != opData.tA || + uint32_t(o1.as().elementType()) != opData.tB || + uint32_t(o2.as().elementType()) != opData.tB) goto InvalidInstruction; opcode.reset(uint32_t(opData.vectorOp) << 10); @@ -3840,12 +4051,12 @@ Case_BaseLdurStur: if (!opData.elementOp) goto InvalidInstruction; - if (o0.as().type() != o1.as().type() || !o2.as().isVecV()) + if (o0.as().regType() != o1.as().regType() || !o2.as().isVec128()) goto InvalidInstruction; - if (o0.as().elementType() != opData.tA || - o1.as().elementType() != opData.tB || - o2.as().elementType() != opData.tElement) + if (uint32_t(o0.as().elementType()) != opData.tA || + uint32_t(o1.as().elementType()) != opData.tB || + uint32_t(o2.as().elementType()) != opData.tElement) goto InvalidInstruction; uint32_t elementIndex = o2.as().elementIndex(); @@ -3871,22 +4082,22 @@ Case_BaseLdurStur: case InstDB::kEncodingSimdDup: SimdDup: { if (isign4 == ENC_OPS2(Reg, Reg)) { // Truth table of valid encodings of `Q:1|ElementType:3` - uint32_t kValidEncodings = B(Vec::kElementTypeB + 0) | - B(Vec::kElementTypeH + 0) | - B(Vec::kElementTypeS + 0) | - B(Vec::kElementTypeB + 8) | - B(Vec::kElementTypeH + 8) | - B(Vec::kElementTypeS + 8) | - B(Vec::kElementTypeD + 8) ; + uint32_t kValidEncodings = B(uint32_t(VecElementType::kB) + 0) | + B(uint32_t(VecElementType::kH) + 0) | + B(uint32_t(VecElementType::kS) + 0) | + B(uint32_t(VecElementType::kB) + 8) | + B(uint32_t(VecElementType::kH) + 8) | + B(uint32_t(VecElementType::kS) + 8) | + B(uint32_t(VecElementType::kD) + 8) ; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (o1.as().isGp()) { // DUP - Vec (scalar|vector) <- GP register. // // NOTE: This is only scalar for `dup d, x` case, otherwise the value // would be duplicated across all vector elements (1, 2, 4, 8, or 16). - uint32_t elementType = o0.as().elementType(); + uint32_t elementType = uint32_t(o0.as().elementType()); if (q > 1 || !Support::bitTest(kValidEncodings, (q << 3) | elementType)) goto InvalidInstruction; @@ -3905,9 +4116,9 @@ Case_BaseLdurStur: uint32_t dstIndex = o1.as().elementIndex(); if (!o0.as().hasElementType()) { // DUP - Vec (scalar) <- Vec[N]. - uint32_t lsbIndex = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t lsbIndex = diff(o0.as().regType(), RegType::kVec8); - if (lsbIndex != o1.as().elementType() - Vec::kElementTypeB || lsbIndex > 3) + if (lsbIndex != diff(o1.as().elementType(), VecElementType::kB) || lsbIndex > 3) goto InvalidInstruction; uint32_t imm5 = ((dstIndex << 1) | 1u) << lsbIndex; @@ -3920,7 +4131,7 @@ Case_BaseLdurStur: } else { // DUP - Vec (all) <- Vec[N]. - uint32_t elementType = o0.as().elementType(); + uint32_t elementType = uint32_t(o0.as().elementType()); if (q > 1 || !Support::bitTest(kValidEncodings, (q << 3) | elementType)) goto InvalidInstruction; @@ -3941,11 +4152,11 @@ Case_BaseLdurStur: } case InstDB::kEncodingSimdIns: SimdIns: { - if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isVecV()) { + if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isVec128()) { if (!o0.as().hasElementIndex()) goto InvalidInstruction; - uint32_t elementType = o0.as().elementType(); + uint32_t elementType = uint32_t(o0.as().elementType()); uint32_t dstIndex = o0.as().elementIndex(); uint32_t lsbIndex = elementType - 1u; @@ -3959,13 +4170,13 @@ Case_BaseLdurStur: opcode.addImm(imm5, 16); goto EmitOp_Rd0_Rn5; } - else if (o1.as().isVecV() && o1.as().hasElementIndex()) { + else if (o1.as().isVec128() && o1.as().hasElementIndex()) { // INS - Vec[N] <- Vec[M]. if (o0.as().elementType() != o1.as().elementType()) goto InvalidInstruction; uint32_t srcIndex = o1.as().elementIndex(); - if (o0.as().type() != o1.as().type()) + if (o0.as().regType() != o1.as().regType()) goto InvalidInstruction; uint32_t imm4 = srcIndex << lsbIndex; @@ -3997,7 +4208,7 @@ Case_BaseLdurStur: goto InvalidInstruction; // ORR Vd, Vn, Vm - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -4029,7 +4240,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdMoviMvni& opData = InstDB::EncodingData::simdMoviMvni[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Imm) || isign4 == ENC_OPS3(Reg, Imm, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_Any, o0.as().type(), o0.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(InstDB::kVO_V_Any, o0.as().regType(), o0.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4114,7 +4325,6 @@ Case_BaseLdurStur: if (inverted) { imm8 = ~imm8 & 0xFFu; - inverted = 0; } cmode = B(3) | B(2) | B(1); @@ -4147,7 +4357,6 @@ Case_BaseLdurStur: case 3: if (inverted) { imm8 = ~imm8 & 0xFFu; - inverted = 0; } op = 1; @@ -4175,7 +4384,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdShift& opData = InstDB::EncodingData::simdShift[encodingIndex]; const Operand_& sop = significantSimdOp(o0, o1, instFlags); - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().type(), sop.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, sop.as().regType(), sop.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4227,7 +4436,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdShiftES& opData = InstDB::EncodingData::simdShiftES[encodingIndex]; if (isign4 == ENC_OPS3(Reg, Reg, Imm)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4273,14 +4482,14 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdSmovUmov& opData = InstDB::EncodingData::simdSmovUmov[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg) && o0.as().isGp() && o1.as().isVec()) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; if (!o1.as().hasElementIndex()) goto InvalidInstruction; - uint32_t x = o0.as().isGpX(); + uint32_t x = o0.as().isGp64(); uint32_t gpMustBeX = uint32_t(sizeOp.size() >= 3u - opData.isSigned); if (opData.isSigned) { @@ -4313,7 +4522,7 @@ Case_BaseLdurStur: const InstDB::EncodingData::SimdSxtlUxtl& opData = InstDB::EncodingData::simdSxtlUxtl[encodingIndex]; if (isign4 == ENC_OPS2(Reg, Reg)) { - SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().type(), o1.as().elementType()); + SizeOp sizeOp = armElementTypeToSizeOp(opData.vecOpType, o1.as().regType(), o1.as().elementType()); if (!sizeOp.isValid()) goto InvalidInstruction; @@ -4342,7 +4551,7 @@ Case_BaseLdurStur: const Operand_& o4 = opExt[EmitterUtils::kOp4]; const Operand_& o5 = opExt[EmitterUtils::kOp5]; - uint32_t q = diff(o0.as().type(), RegType::kARM_VecD); + uint32_t q = diff(o0.as().regType(), RegType::kVec64); if (q > 1 || o0.as().hasElementIndex()) goto InvalidInstruction; @@ -4421,7 +4630,7 @@ Case_BaseLdurStur: // 32-bit | size==10 | opc == 01 | 010 // 64-bit | size==11 | opc == 01 | 011 // 128-bit| size==00 | opc == 11 | 100 - uint32_t xsz = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t xsz = diff(o0.as().regType(), RegType::kVec8); if (xsz > 4u || o0.as().hasElementIndex()) goto InvalidRegType; @@ -4435,7 +4644,7 @@ Case_BaseLdurStur: if (m.hasBaseReg()) { // [Base {Offset | Index}] if (m.hasIndex()) { - uint32_t opt = armShiftOpToLdStOptMap[m.predicate()]; + uint32_t opt = armShiftOpToLdStOptMap[size_t(m.shiftOp())]; if (opt == 0xFFu) goto InvalidAddress; @@ -4518,7 +4727,7 @@ Case_BaseLdurStur: const Mem& m = o2.as(); rmRel = &m; - uint32_t opc = diff(o0.as().type(), RegType::kARM_VecS); + uint32_t opc = diff(o0.as().regType(), RegType::kVec32); if (opc > 2u || o0.as().hasElementTypeOrIndex()) goto InvalidInstruction; @@ -4528,7 +4737,7 @@ Case_BaseLdurStur: if (!checkVecId(o0, o1)) goto InvalidPhysId; - if (m.baseType() != RegType::kARM_GpX || m.hasIndex()) + if (m.baseType() != RegType::kGp64 || m.hasIndex()) goto InvalidAddress; if (m.isOffset64Bit()) @@ -4574,7 +4783,7 @@ Case_SimdLdurStur: const Mem& m = o1.as(); rmRel = &m; - uint32_t sz = diff(o0.as().type(), RegType::kARM_VecB); + uint32_t sz = diff(o0.as().regType(), RegType::kVec8); if (sz > 4 || o0.as().hasElementTypeOrIndex()) goto InvalidInstruction; @@ -4659,14 +4868,14 @@ Case_SimdLdurStur: uint32_t q = 0; uint32_t rm = 0; uint32_t rn = m.baseId(); - uint32_t sz = v.elementType() - Vec::kElementTypeB; + uint32_t sz = diff(v.elementType(), VecElementType::kB); uint32_t opcSsize = sz; uint32_t offsetPossibility = 0; if (sz > 3) goto InvalidInstruction; - if (m.baseType() != RegType::kARM_GpX) + if (m.baseType() != RegType::kGp64) goto InvalidAddress; // Rn cannot be ZR, but can be SP. @@ -4683,7 +4892,7 @@ Case_SimdLdurStur: if (v.hasElementIndex()) goto InvalidInstruction; - q = diff(v.type(), RegType::kARM_VecD); + q = diff(v.regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -4715,7 +4924,7 @@ Case_SimdLdurStur: // LDx/STx (multiple structures). static const uint8_t opcSsizeByN[] = { 0u, 0x7u << 2, 0xAu << 2, 0x6u << 2, 0x2u << 2 }; - q = diff(v.type(), RegType::kARM_VecD); + q = diff(v.regType(), RegType::kVec64); if (q > 1) goto InvalidInstruction; @@ -4822,11 +5031,13 @@ EmitOp_Multiple: { ASMJIT_ASSERT(multipleOpCount > 0); err = writer.ensureSpace(this, multipleOpCount * 4u); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { goto Failed; + } - for (uint32_t i = 0; i < multipleOpCount; i++) + for (uint32_t i = 0; i < multipleOpCount; i++) { writer.emit32uLE(multipleOpData[i]); + } goto EmitDone; } @@ -4836,28 +5047,33 @@ EmitOp_Multiple: // -------------------------------------------------------------------------- EmitOp_MemBase_Rn5: - if (!checkMemBase(rmRel->as())) + if (!checkMemBase(rmRel->as())) { goto InvalidAddress; + } opcode.addReg(rmRel->as().baseId(), 5); goto EmitOp; EmitOp_MemBaseNoImm_Rn5: - if (!checkMemBase(rmRel->as()) || rmRel->as().hasIndex()) + if (!checkMemBase(rmRel->as()) || rmRel->as().hasIndex()) { goto InvalidAddress; + } - if (rmRel->as().hasOffset()) + if (rmRel->as().hasOffset()) { goto InvalidDisplacement; + } opcode.addReg(rmRel->as().baseId(), 5); goto EmitOp; EmitOp_MemBaseIndex_Rn5_Rm16: - if (!rmRel->as().hasBaseReg()) + if (!rmRel->as().hasBaseReg()) { goto InvalidAddress; + } - if (rmRel->as().indexId() > 30 && rmRel->as().indexId() != Gp::kIdZr) + if (rmRel->as().indexId() > 30 && rmRel->as().indexId() != Gp::kIdZr) { goto InvalidPhysId; + } opcode.addReg(rmRel->as().indexId(), 16); opcode.addReg(rmRel->as().baseId(), 5); @@ -4881,26 +5097,29 @@ EmitOp_Rel: labelOffset = rmRel->as().offset(); } - LabelEntry* label = _code->labelEntry(labelId); - if (ASMJIT_UNLIKELY(!label)) + if (ASMJIT_UNLIKELY(!_code->isLabelValid(labelId))) { goto InvalidLabel; + } + + LabelEntry& le = _code->labelEntry(labelId); if (offsetFormat.type() == OffsetType::kAArch64_ADRP) { // TODO: [ARM] Always create relocation entry. } - if (label->isBoundTo(_section)) { + if (le.isBoundTo(_section)) { // Label bound to the current section. - offsetValue = label->offset() - uint64_t(offset()) + uint64_t(labelOffset); + offsetValue = le.offset() - uint64_t(offset()) + uint64_t(labelOffset); goto EmitOp_DispImm; } else { - // Record non-bound label. + // Create a fixup referencing an non-bound label. size_t codeOffset = writer.offsetFrom(_bufferData); - LabelLink* link = _code->newLabelLink(label, _section->id(), codeOffset, intptr_t(labelOffset), offsetFormat); + Fixup* fixup = _code->newFixup(le, _section->sectionId(), codeOffset, intptr_t(labelOffset), offsetFormat); - if (ASMJIT_UNLIKELY(!link)) + if (ASMJIT_UNLIKELY(!fixup)) { goto OutOfMemory; + } goto EmitOp; } @@ -4913,14 +5132,15 @@ EmitOp_Rel: size_t codeOffset = writer.offsetFrom(_bufferData); - if (baseAddress == Globals::kNoBaseAddress || _section->id() != 0) { + if (baseAddress == Globals::kNoBaseAddress || _section->sectionId() != 0) { // Create a new RelocEntry as we cannot calculate the offset right now. RelocEntry* re; err = _code->newRelocEntry(&re, RelocType::kAbsToRel); - if (err) + if (err) { goto Failed; + } - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = codeOffset; re->_format = offsetFormat; re->_payload = rmRel->as().valueAs() + 4u; @@ -4929,8 +5149,9 @@ EmitOp_Rel: else { uint64_t pc = baseAddress + codeOffset; - if (offsetFormat.type() == OffsetType::kAArch64_ADRP) + if (offsetFormat.type() == OffsetType::kAArch64_ADRP) { pc &= ~uint64_t(4096 - 1); + } offsetValue = targetOffset - pc; goto EmitOp_DispImm; @@ -4941,12 +5162,14 @@ EmitOp_Rel: EmitOp_DispImm: { - if ((offsetValue & Support::lsbMask(offsetFormat.immDiscardLsb())) != 0) + if ((offsetValue & Support::lsbMask(offsetFormat.immDiscardLsb())) != 0) { goto InvalidDisplacement; + } int64_t dispImm64 = int64_t(offsetValue) >> offsetFormat.immDiscardLsb(); - if (!Support::isEncodableOffset64(dispImm64, offsetFormat.immBitCount())) + if (!Support::isEncodableOffset64(dispImm64, offsetFormat.immBitCount())) { goto InvalidDisplacement; + } uint32_t dispImm32 = uint32_t(dispImm64 & Support::lsbMask(offsetFormat.immBitCount())); switch (offsetFormat.type()) { @@ -4984,8 +5207,9 @@ EmitOp: EmitDone: if (Support::test(options, InstOptions::kReserved)) { #ifndef ASMJIT_NO_LOGGING - if (_logger) + if (_logger) { EmitterUtils::logInstructionEmitted(this, BaseInst::composeARMInstId(instId, instCC), options, o0, o1, o2, opExt, 0, 0, writer.cursor()); + } #endif } @@ -5031,21 +5255,26 @@ Failed: Error Assembler::align(AlignMode alignMode, uint32_t alignment) { constexpr uint32_t kNopA64 = 0xD503201Fu; // [11010101|00000011|00100000|00011111]. - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - if (ASMJIT_UNLIKELY(uint32_t(alignMode) > uint32_t(AlignMode::kMaxValue))) + if (ASMJIT_UNLIKELY(uint32_t(alignMode) > uint32_t(AlignMode::kMaxValue))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } - if (alignment <= 1) + if (alignment <= 1) { return kErrorOk; + } - if (ASMJIT_UNLIKELY(alignment > Globals::kMaxAlignment || !Support::isPowerOf2(alignment))) + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(alignment, Globals::kMaxAlignment))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } uint32_t i = uint32_t(Support::alignUpDiff(offset(), alignment)); - if (i == 0) + if (i == 0) { return kErrorOk; + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, i)); @@ -5054,8 +5283,9 @@ Error Assembler::align(AlignMode alignMode, uint32_t alignment) { case AlignMode::kCode: { uint32_t pattern = kNopA64; - if (ASMJIT_UNLIKELY(offset() & 0x3u)) + if (ASMJIT_UNLIKELY(offset() & 0x3u)) { return DebugUtils::errored(kErrorInvalidState); + } while (i >= 4) { writer.emit32uLE(pattern); @@ -5089,12 +5319,16 @@ Error Assembler::align(AlignMode alignMode, uint32_t alignment) { // a64::Assembler - Events // ======================= -Error Assembler::onAttach(CodeHolder* code) noexcept { +Error Assembler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); + + _instructionAlignment = uint8_t(4); + updateEmitterFuncs(this); + return kErrorOk; } -Error Assembler::onDetach(CodeHolder* code) noexcept { +Error Assembler::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } diff --git a/pe-packer/asmjit/arm/a64assembler.h b/pe-packer/asmjit/arm/a64assembler.h index f9eb9d4..e1e154a 100644 --- a/pe-packer/asmjit/arm/a64assembler.h +++ b/pe-packer/asmjit/arm/a64assembler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64ASSEMBLER_H_INCLUDED @@ -21,9 +21,9 @@ class ASMJIT_VIRTAPI Assembler public EmitterExplicitT { public: - typedef BaseAssembler Base; + using Base = BaseAssembler; - //! \name Construction / Destruction + //! \name Construction & Destruction //! \{ ASMJIT_API Assembler(CodeHolder* code = nullptr) noexcept; @@ -31,17 +31,6 @@ public: //! \} - //! \name Accessors - //! \{ - - //! Gets whether the current ARM mode is THUMB (alternative to 32-bit ARM encoding). - ASMJIT_INLINE_NODEBUG bool isInThumbMode() const noexcept { return _environment.isArchThumb(); } - - //! Gets the current code alignment of the current mode (ARM vs THUMB). - ASMJIT_INLINE_NODEBUG uint32_t codeAlignment() const noexcept { return isInThumbMode() ? 2 : 4; } - - //! \} - //! \name Emit //! \{ @@ -59,8 +48,8 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; //! \} }; diff --git a/pe-packer/asmjit/arm/a64builder.cpp b/pe-packer/asmjit/arm/a64builder.cpp index 3a52b2a..7b114cb 100644 --- a/pe-packer/asmjit/arm/a64builder.cpp +++ b/pe-packer/asmjit/arm/a64builder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -17,21 +17,27 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) Builder::Builder(CodeHolder* code) noexcept : BaseBuilder() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); - assignEmitterFuncs(this); + initEmitterFuncs(this); - if (code) + if (code) { code->attach(this); + } } Builder::~Builder() noexcept {} // a64::Builder - Events // ===================== -Error Builder::onAttach(CodeHolder* code) noexcept { - return Base::onAttach(code); +Error Builder::onAttach(CodeHolder& code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + _instructionAlignment = uint8_t(4); + updateEmitterFuncs(this); + + return kErrorOk; } -Error Builder::onDetach(CodeHolder* code) noexcept { +Error Builder::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } diff --git a/pe-packer/asmjit/arm/a64builder.h b/pe-packer/asmjit/arm/a64builder.h index cab1083..9a468ac 100644 --- a/pe-packer/asmjit/arm/a64builder.h +++ b/pe-packer/asmjit/arm/a64builder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64BUILDER_H_INCLUDED @@ -23,7 +23,7 @@ class ASMJIT_VIRTAPI Builder public EmitterExplicitT { public: ASMJIT_NONCOPYABLE(Builder) - typedef BaseBuilder Base; + using Base = BaseBuilder; //! \name Construction & Destruction //! \{ @@ -36,8 +36,8 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; //! \} diff --git a/pe-packer/asmjit/arm/a64compiler.cpp b/pe-packer/asmjit/arm/a64compiler.cpp index d6c4ed2..cc9fc7f 100644 --- a/pe-packer/asmjit/arm/a64compiler.cpp +++ b/pe-packer/asmjit/arm/a64compiler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -18,17 +18,18 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) Compiler::Compiler(CodeHolder* code) noexcept : BaseCompiler() { _archMask = uint64_t(1) << uint32_t(Arch::kAArch64); - assignEmitterFuncs(this); + initEmitterFuncs(this); - if (code) + if (code) { code->attach(this); + } } Compiler::~Compiler() noexcept {} // a64::Compiler - Events // ====================== -Error Compiler::onAttach(CodeHolder* code) noexcept { +Error Compiler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); Error err = addPassT(); @@ -37,13 +38,24 @@ Error Compiler::onAttach(CodeHolder* code) noexcept { return err; } + _instructionAlignment = uint8_t(4); + updateEmitterFuncs(this); + return kErrorOk; } -Error Compiler::onDetach(CodeHolder* code) noexcept { +Error Compiler::onDetach(CodeHolder& code) noexcept { return Base::onDetach(code); } +Error Compiler::onReinit(CodeHolder& code) noexcept { + Error err = Base::onReinit(code); + if (err == kErrorOk) { + err = addPassT(); + } + return err; +} + // a64::Compiler - Finalize // ======================== diff --git a/pe-packer/asmjit/arm/a64compiler.h b/pe-packer/asmjit/arm/a64compiler.h index c0b3867..1b104aa 100644 --- a/pe-packer/asmjit/arm/a64compiler.h +++ b/pe-packer/asmjit/arm/a64compiler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64COMPILER_H_INCLUDED @@ -24,7 +24,7 @@ class ASMJIT_VIRTAPI Compiler public EmitterExplicitT { public: ASMJIT_NONCOPYABLE(Compiler) - typedef BaseCompiler Base; + using Base = BaseCompiler; //! \name Construction & Destruction //! \{ @@ -45,20 +45,27 @@ public: return reg; } + template + ASMJIT_INLINE_NODEBUG RegT _newRegInternal(const Type& type, const char* s) { +#ifndef ASMJIT_NO_LOGGING + RegT reg(Globals::NoInit); + _newReg(®, type, s); + return reg; +#else + DebugUtils::unused(s); + return _newRegInternal(type); +#endif + } + template ASMJIT_INLINE_NODEBUG RegT _newRegInternal(const Type& type, const char* s, Args&&... args) { #ifndef ASMJIT_NO_LOGGING RegT reg(Globals::NoInit); - if (sizeof...(Args) == 0) - _newReg(®, type, s); - else - _newRegFmt(®, type, s, std::forward(args)...); + _newRegFmt(®, type, s, std::forward(args)...); return reg; #else DebugUtils::unused(s, std::forward(args)...); - RegT reg(Globals::NoInit); - _newReg(®, type, nullptr); - return reg; + return _newRegInternal(type); #endif } //! \endcond @@ -92,6 +99,11 @@ public: template ASMJIT_INLINE_NODEBUG Gp newUIntPtr(Args&&... args) { return _newRegInternal(TypeId::kUIntPtr, std::forward(args)...); } + template + ASMJIT_INLINE_NODEBUG Gp newGp32(Args&&... args) { return _newRegInternal(TypeId::kUInt32, std::forward(args)...); } + template + ASMJIT_INLINE_NODEBUG Gp newGp64(Args&&... args) { return _newRegInternal(TypeId::kUInt64, std::forward(args)...); } + template ASMJIT_INLINE_NODEBUG Gp newGpw(Args&&... args) { return _newRegInternal(TypeId::kUInt32, std::forward(args)...); } template @@ -207,9 +219,9 @@ public: //! Return. ASMJIT_INLINE_NODEBUG Error ret() { return addRet(Operand(), Operand()); } //! \overload - ASMJIT_INLINE_NODEBUG Error ret(const BaseReg& o0) { return addRet(o0, Operand()); } + ASMJIT_INLINE_NODEBUG Error ret(const Reg& o0) { return addRet(o0, Operand()); } //! \overload - ASMJIT_INLINE_NODEBUG Error ret(const BaseReg& o0, const BaseReg& o1) { return addRet(o0, o1); } + ASMJIT_INLINE_NODEBUG Error ret(const Reg& o0, const Reg& o1) { return addRet(o0, o1); } //! \} @@ -219,15 +231,16 @@ public: using EmitterExplicitT::br; //! Adds a jump to the given `target` with the provided jump `annotation`. - ASMJIT_INLINE_NODEBUG Error br(const BaseReg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); } + ASMJIT_INLINE_NODEBUG Error br(const Reg& target, JumpAnnotation* annotation) { return emitAnnotatedJump(Inst::kIdBr, target, annotation); } //! \} //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} diff --git a/pe-packer/asmjit/arm/a64emithelper.cpp b/pe-packer/asmjit/arm/a64emithelper.cpp index 1e8da61..3537761 100644 --- a/pe-packer/asmjit/arm/a64emithelper.cpp +++ b/pe-packer/asmjit/arm/a64emithelper.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -54,14 +54,17 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove( return emitter->ldr(dst.as().x(), src); default: { - if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) + if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) { return emitter->ldr(dst.as().s(), src); + } - if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) + if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) { return emitter->ldr(dst.as().d(), src); + } - if (TypeUtils::isVec128(typeId)) + if (TypeUtils::isVec128(typeId)) { return emitter->ldr(dst.as().q(), src); + } break; } @@ -90,14 +93,17 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove( return emitter->str(src.as().x(), dst); default: { - if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) + if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) { return emitter->str(src.as().s(), dst); + } - if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) + if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) { return emitter->str(src.as().d(), dst); + } - if (TypeUtils::isVec128(typeId)) + if (TypeUtils::isVec128(typeId)) { return emitter->str(src.as().q(), dst); + } break; } @@ -120,14 +126,17 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove( return emitter->mov(dst.as().x(), src.as().x()); default: { - if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) + if (TypeUtils::isFloat32(typeId) || TypeUtils::isVec32(typeId)) { return emitter->fmov(dst.as().s(), src.as().s()); + } - if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) + if (TypeUtils::isFloat64(typeId) || TypeUtils::isVec64(typeId)) { return emitter->mov(dst.as().b8(), src.as().b8()); + } - if (TypeUtils::isVec128(typeId)) + if (TypeUtils::isVec128(typeId)) { return emitter->mov(dst.as().b16(), src.as().b16()); + } break; } @@ -139,8 +148,8 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitRegMove( } Error EmitHelper::emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment) { + const Reg& a, + const Reg& b, const char* comment) { DebugUtils::unused(a, b, comment); return DebugUtils::errored(kErrorInvalidState); @@ -148,13 +157,12 @@ Error EmitHelper::emitRegSwap( // TODO: [ARM] EmitArgMove is unfinished. Error EmitHelper::emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, + const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { // Deduce optional `dstTypeId`, which may be `TypeId::kVoid` in some cases. if (dstTypeId == TypeId::kVoid) { - const ArchTraits& archTraits = ArchTraits::byArch(_emitter->arch()); - dstTypeId = archTraits.regTypeToTypeId(dst_.type()); + dstTypeId = RegUtils::typeIdOf(dst_.regType()); } // Invalid or abstract TypeIds are not allowed. @@ -169,9 +177,9 @@ Error EmitHelper::emitArgMove( if (TypeUtils::isInt(dstTypeId)) { if (TypeUtils::isInt(srcTypeId)) { - uint32_t x = dstSize == 8; + uint32_t x = uint32_t(dstSize == 8); - dst.setSignature(OperandSignature{x ? uint32_t(GpX::kSignature) : uint32_t(GpW::kSignature)}); + dst.setSignature(OperandSignature{x ? RegTraits::kSignature : RegTraits::kSignature}); _emitter->setInlineComment(comment); if (src.isReg()) { @@ -186,7 +194,7 @@ Error EmitHelper::emitArgMove( case TypeId::kInt16: instId = Inst::kIdLdrsh; break; case TypeId::kUInt16: instId = Inst::kIdLdrh; break; case TypeId::kInt32: instId = x ? Inst::kIdLdrsw : Inst::kIdLdr; break; - case TypeId::kUInt32: instId = Inst::kIdLdr; x = 0; break; + case TypeId::kUInt32: instId = Inst::kIdLdr; break; case TypeId::kInt64: instId = Inst::kIdLdr; break; case TypeId::kUInt64: instId = Inst::kIdLdr; break; default: @@ -200,10 +208,10 @@ Error EmitHelper::emitArgMove( if (TypeUtils::isFloat(dstTypeId) || TypeUtils::isVec(dstTypeId)) { if (TypeUtils::isFloat(srcTypeId) || TypeUtils::isVec(srcTypeId)) { switch (srcSize) { - case 2: dst.as().setSignature(OperandSignature{VecH::kSignature}); break; - case 4: dst.as().setSignature(OperandSignature{VecS::kSignature}); break; - case 8: dst.as().setSignature(OperandSignature{VecD::kSignature}); break; - case 16: dst.as().setSignature(OperandSignature{VecV::kSignature}); break; + case 2: dst.as().setSignature(RegTraits::kSignature); break; + case 4: dst.as().setSignature(RegTraits::kSignature); break; + case 8: dst.as().setSignature(RegTraits::kSignature); break; + case 16: dst.as().setSignature(RegTraits::kSignature); break; default: return DebugUtils::errored(kErrorInvalidState); } @@ -286,7 +294,7 @@ struct PrologEpilogInfo { } if (n == 1) { - pairs[pairCount].ids[1] = uint8_t(BaseReg::kIdBad); + pairs[pairCount].ids[1] = uint8_t(Reg::kIdBad); pairs[pairCount].offset = uint16_t(offset); offset += slotSize * 2; pairCount++; @@ -312,6 +320,11 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) { { Inst::kIdStr_v, Inst::kIdStp_v } }}; + // Emit: 'bti {jc}' (indirect branch protection). + if (frame.hasIndirectBranchProtection()) { + ASMJIT_PROPAGATE(emitter->bti(Predicate::BTI::kJC)); + } + uint32_t adjustInitialOffset = pei.sizeTotal; for (RegGroup group : Support::EnumValues{}) { @@ -334,12 +347,14 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitProlog(const FuncFrame& frame) { mem.makePreIndex(); } - if (pair.ids[1] == BaseReg::kIdBad) + if (pair.ids[1] == Reg::kIdBad) { ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem)); - else + } + else { ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem)); + } - mem.resetToFixedOffset(); + mem.resetOffsetMode(); if (i == 0 && frame.hasPreservedFP()) { ASMJIT_PROPAGATE(emitter->mov(x29, sp)); @@ -416,12 +431,14 @@ ASMJIT_FAVOR_SIZE Error EmitHelper::emitEpilog(const FuncFrame& frame) { mem.makePostIndex(); } - if (pair.ids[1] == BaseReg::kIdBad) + if (pair.ids[1] == Reg::kIdBad) { ASMJIT_PROPAGATE(emitter->emit(insts.singleInstId, regs[0], mem)); - else + } + else { ASMJIT_PROPAGATE(emitter->emit(insts.pairInstId, regs[0], regs[1], mem)); + } - mem.resetToFixedOffset(); + mem.resetOffsetMode(); } } @@ -445,7 +462,7 @@ static Error ASMJIT_CDECL Emitter_emitArgsAssignment(BaseEmitter* emitter, const return emitHelper.emitArgsAssignment(frame, args); } -void assignEmitterFuncs(BaseEmitter* emitter) { +void initEmitterFuncs(BaseEmitter* emitter) { emitter->_funcs.emitProlog = Emitter_emitProlog; emitter->_funcs.emitEpilog = Emitter_emitEpilog; emitter->_funcs.emitArgsAssignment = Emitter_emitArgsAssignment; diff --git a/pe-packer/asmjit/arm/a64emithelper_p.h b/pe-packer/asmjit/arm/a64emithelper_p.h index 6e10f9e..1b034fe 100644 --- a/pe-packer/asmjit/arm/a64emithelper_p.h +++ b/pe-packer/asmjit/arm/a64emithelper_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64EMITHELPER_P_H_INCLUDED @@ -24,23 +24,28 @@ public: ASMJIT_INLINE_NODEBUG explicit EmitHelper(BaseEmitter* emitter = nullptr) noexcept : BaseEmitHelper(emitter) {} + ASMJIT_INLINE_NODEBUG virtual ~EmitHelper() noexcept = default; + Error emitRegMove( const Operand_& dst_, const Operand_& src_, TypeId typeId, const char* comment = nullptr) override; Error emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment = nullptr) override; + const Reg& a, + const Reg& b, const char* comment = nullptr) override; Error emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, + const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) override; Error emitProlog(const FuncFrame& frame); Error emitEpilog(const FuncFrame& frame); }; -void assignEmitterFuncs(BaseEmitter* emitter); +void initEmitterFuncs(BaseEmitter* emitter); + +[[maybe_unused]] +static inline void updateEmitterFuncs(BaseEmitter* emitter) noexcept { DebugUtils::unused(emitter); } //! \} //! \endcond diff --git a/pe-packer/asmjit/arm/a64emitter.h b/pe-packer/asmjit/arm/a64emitter.h index 22717d3..876af77 100644 --- a/pe-packer/asmjit/arm/a64emitter.h +++ b/pe-packer/asmjit/arm/a64emitter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64EMITTER_H_INCLUDED @@ -71,7 +71,7 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) //! ARM emitter. //! //! NOTE: This class cannot be instantiated, you can only cast to it and use it as emitter that emits to either -//! \ref Assembler, \ref Builder, or \ref Compiler (use withcaution with \ref Compiler as it expects virtual +//! \ref Assembler, \ref Builder, or \ref Compiler (use with caution with \ref Compiler as it expects virtual //! registers to be used). template struct EmitterExplicitT { @@ -79,11 +79,22 @@ struct EmitterExplicitT { // These two are unfortunately reported by the sanitizer. We know what we do, however, the sanitizer doesn't. // I have tried to use reinterpret_cast instead, but that would generate bad code when compiled by MSC. - ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF inline This* _emitter() noexcept { return static_cast(this); } - ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF inline const This* _emitter() const noexcept { return static_cast(this); } + ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF ASMJIT_INLINE_NODEBUG This* _emitter() noexcept { return static_cast(this); } + ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF ASMJIT_INLINE_NODEBUG const This* _emitter() const noexcept { return static_cast(this); } //! \endcond + + //! \name Native Registers + //! \{ + + //! Returns either 32-bit or 64-bit GP register of the given `id` depending on the emitter's architecture. + inline Gp gpz(uint32_t id) const noexcept { return Gp(_emitter()->_gpSignature, id); } + //! Clones the given `reg` to either 32-bit or 64-bit GP register depending on the emitter's architecture. + inline Gp gpz(const Gp& reg) const noexcept { return Gp(_emitter()->_gpSignature, reg.id()); } + + //! \} + //! \name General Purpose Instructions //! \{ @@ -215,6 +226,9 @@ struct EmitterExplicitT { ASMJIT_INST_2x(mvn, Mvn, Gp, Gp) ASMJIT_INST_3x(mvn, Mvn, Gp, Gp, Imm) + ASMJIT_INST_2x(mvn_, Mvn, Gp, Gp) + ASMJIT_INST_3x(mvn_, Mvn, Gp, Gp, Imm) + ASMJIT_INST_2x(neg, Neg, Gp, Gp) ASMJIT_INST_3x(neg, Neg, Gp, Gp, Imm) ASMJIT_INST_2x(negs, Negs, Gp, Gp) @@ -300,27 +314,9 @@ struct EmitterExplicitT { ASMJIT_INST_1x(dcps1, Dcps1, Imm) ASMJIT_INST_1x(dcps2, Dcps2, Imm) ASMJIT_INST_1x(dcps3, Dcps3, Imm) - ASMJIT_INST_0x(dgh, Dgh) ASMJIT_INST_0x(pssbb, Pssbb) ASMJIT_INST_0x(ssbb, Ssbb) ASMJIT_INST_1x(udf, Udf, Imm) - ASMJIT_INST_1x(setf8, Setf8, Gp) - ASMJIT_INST_1x(setf16, Setf16, Gp) - - //! \} - - //! \name ARMv8.4 Instructions - //! \{ - - ASMJIT_INST_0x(cfinv, Cfinv) - - //! \} - - //! \name ARMv8.5 Instructions - //! \{ - - ASMJIT_INST_0x(axflag, Axflag) - ASMJIT_INST_0x(xaflag, Xaflag) //! \} @@ -514,6 +510,8 @@ struct EmitterExplicitT { ASMJIT_INST_2x(ldxrb, Ldxrb, Gp, Mem) ASMJIT_INST_2x(ldxrh, Ldxrh, Gp, Mem) + ASMJIT_INST_2x(prfm, Prfm, Imm, Mem) + ASMJIT_INST_2x(stadd, Stadd, Gp, Mem) ASMJIT_INST_2x(staddb, Staddb, Gp, Mem) ASMJIT_INST_2x(staddh, Staddh, Gp, Mem) @@ -618,6 +616,27 @@ struct EmitterExplicitT { ASMJIT_INST_3x(swplh, Swplh, Gp, Gp, Mem) //! \} + //! \name BTI Instructions + //! \{ + + ASMJIT_INST_1x(bti, Bti, Imm); + + //! \} + + //! \name CHK Instructions + //! \{ + + ASMJIT_INST_1x(chkfeat, Chkfeat, Gp); + + //! \} + + //! \name CLRBHB Instructions + //! \{ + + ASMJIT_INST_0x(clrbhb, Clrbhb); + + //! \} + //! \name CRC Instructions (ARMv8.1-A, optional in ARMv8.0-A) //! \{ @@ -633,6 +652,55 @@ struct EmitterExplicitT { //! \} + //! \name CSSC Instructions + //! \{ + + ASMJIT_INST_2x(abs, Abs, Gp, Gp); + ASMJIT_INST_2x(cnt, Cnt, Gp, Gp); + ASMJIT_INST_2x(ctz, Ctz, Gp, Gp); + ASMJIT_INST_3x(smax, Smax, Gp, Gp, Gp); + ASMJIT_INST_3x(smax, Smax, Gp, Gp, Imm); + ASMJIT_INST_3x(smin, Smin, Gp, Gp, Gp); + ASMJIT_INST_3x(smin, Smin, Gp, Gp, Imm); + ASMJIT_INST_3x(umax, Umax, Gp, Gp, Gp); + ASMJIT_INST_3x(umax, Umax, Gp, Gp, Imm); + ASMJIT_INST_3x(umin, Umin, Gp, Gp, Gp); + ASMJIT_INST_3x(umin, Umin, Gp, Gp, Imm); + + //! \} + + //! \name DGH Instructions + //! \{ + + ASMJIT_INST_0x(dgh, Dgh) + + //! \} + + //! \name FLAGM Instructions + //! \{ + + ASMJIT_INST_0x(cfinv, Cfinv) + ASMJIT_INST_1x(setf8, Setf8, Gp) + ASMJIT_INST_1x(setf16, Setf16, Gp) + + //! \} + + //! \name FLAGM2 Instructions + //! \{ + + ASMJIT_INST_0x(axflag, Axflag) + ASMJIT_INST_0x(xaflag, Xaflag) + + //! \} + + //! \name HBC Instructions + //! \{ + + ASMJIT_INST_1cc(bc, Bc, Imm) + ASMJIT_INST_1cc(bc, Bc, Label) + + //! \} + //! \name MTE Instructions //! \{ @@ -861,6 +929,7 @@ struct EmitterExplicitT { ASMJIT_INST_3x(movi, Movi_v, Vec, Imm, Imm); ASMJIT_INST_3x(mul, Mul_v, Vec, Vec, Vec); ASMJIT_INST_2x(mvn, Mvn_v, Vec, Vec); + ASMJIT_INST_2x(mvn_, Mvn_v, Vec, Vec); ASMJIT_INST_2x(mvni, Mvni_v, Vec, Imm); ASMJIT_INST_3x(mvni, Mvni_v, Vec, Imm, Imm); ASMJIT_INST_2x(neg, Neg_v, Vec, Vec); @@ -1200,7 +1269,7 @@ struct EmitterExplicitT { //! \} }; -//! Emitter (ARM). +//! Emitter (AArch64). //! //! \note This class cannot be instantiated, you can only cast to it and use it as emitter that emits to either //! `a64::Assembler`, `a64::Builder`, or `a64::Compiler` (use with caution with `a64::Compiler` as it requires diff --git a/pe-packer/asmjit/arm/a64formatter.cpp b/pe-packer/asmjit/arm/a64formatter.cpp index 8cd4a06..069f6af 100644 --- a/pe-packer/asmjit/arm/a64formatter.cpp +++ b/pe-packer/asmjit/arm/a64formatter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -24,19 +24,23 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) ASMJIT_FAVOR_SIZE Error FormatterInternal::formatInstruction( String& sb, - FormatFlags flags, + FormatFlags formatFlags, const BaseEmitter* emitter, Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept { - DebugUtils::unused(arch); - // Format instruction options and instruction mnemonic. InstId instId = inst.realId(); - if (instId < Inst::_kIdCount) - ASMJIT_PROPAGATE(InstInternal::instIdToString(arch, instId, sb)); - else + if (instId != Inst::kIdNone && instId < Inst::_kIdCount) { + InstStringifyOptions stringifyOptions = + Support::test(formatFlags, FormatFlags::kShowAliases) + ? InstStringifyOptions::kAliases + : InstStringifyOptions::kNone; + ASMJIT_PROPAGATE(InstInternal::instIdToString(instId, stringifyOptions, sb)); + } + else { ASMJIT_PROPAGATE(sb.appendFormat("[InstId=#%u]", unsigned(instId))); + } CondCode cc = inst.armCondCode(); if (cc != CondCode::kAL) { @@ -46,11 +50,12 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatInstruction( for (uint32_t i = 0; i < opCount; i++) { const Operand_& op = operands[i]; - if (op.isNone()) + if (op.isNone()) { break; + } ASMJIT_PROPAGATE(sb.append(i == 0 ? " " : ", ")); - ASMJIT_PROPAGATE(formatOperand(sb, flags, emitter, arch, op)); + ASMJIT_PROPAGATE(formatOperand(sb, formatFlags, emitter, arch, op)); } return kErrorOk; diff --git a/pe-packer/asmjit/arm/a64formatter_p.h b/pe-packer/asmjit/arm/a64formatter_p.h index d0adde3..1da4936 100644 --- a/pe-packer/asmjit/arm/a64formatter_p.h +++ b/pe-packer/asmjit/arm/a64formatter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64FORMATTER_P_H_INCLUDED diff --git a/pe-packer/asmjit/arm/a64func.cpp b/pe-packer/asmjit/arm/a64func.cpp index 55e3f2e..d4a54d6 100644 --- a/pe-packer/asmjit/arm/a64func.cpp +++ b/pe-packer/asmjit/arm/a64func.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -13,7 +13,7 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) namespace FuncInternal { -static inline bool shouldThreatAsCDecl(CallConvId ccId) noexcept { +static inline bool shouldTreatAsCDecl(CallConvId ccId) noexcept { return ccId == CallConvId::kCDecl || ccId == CallConvId::kStdCall || ccId == CallConvId::kFastCall || @@ -25,34 +25,41 @@ static inline bool shouldThreatAsCDecl(CallConvId ccId) noexcept { } static RegType regTypeFromFpOrVecTypeId(TypeId typeId) noexcept { - if (typeId == TypeId::kFloat32) - return RegType::kARM_VecS; - else if (typeId == TypeId::kFloat64) - return RegType::kARM_VecD; - else if (TypeUtils::isVec32(typeId)) - return RegType::kARM_VecS; - else if (TypeUtils::isVec64(typeId)) - return RegType::kARM_VecD; - else if (TypeUtils::isVec128(typeId)) - return RegType::kARM_VecV; - else + if (typeId == TypeId::kFloat32) { + return RegType::kVec32; + } + else if (typeId == TypeId::kFloat64) { + return RegType::kVec64; + } + else if (TypeUtils::isVec32(typeId)) { + return RegType::kVec32; + } + else if (TypeUtils::isVec64(typeId)) { + return RegType::kVec64; + } + else if (TypeUtils::isVec128(typeId)) { + return RegType::kVec128; + } + else { return RegType::kNone; + } } ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept { cc.setArch(environment.arch()); + cc.setStrategy(environment.isDarwin() ? CallConvStrategy::kAArch64Apple : CallConvStrategy::kDefault); cc.setSaveRestoreRegSize(RegGroup::kGp, 8); cc.setSaveRestoreRegSize(RegGroup::kVec, 8); cc.setSaveRestoreAlignment(RegGroup::kGp, 16); cc.setSaveRestoreAlignment(RegGroup::kVec, 16); - cc.setSaveRestoreAlignment(RegGroup::kExtraVirt2, 1); + cc.setSaveRestoreAlignment(RegGroup::kMask, 1); cc.setSaveRestoreAlignment(RegGroup::kExtraVirt3, 1); cc.setPassedOrder(RegGroup::kGp, 0, 1, 2, 3, 4, 5, 6, 7); cc.setPassedOrder(RegGroup::kVec, 0, 1, 2, 3, 4, 5, 6, 7); cc.setNaturalStackAlignment(16); - if (shouldThreatAsCDecl(ccId)) { + if (shouldTreatAsCDecl(ccId)) { // ARM doesn't have that many calling conventions as we can find in X86 world, treat most conventions as __cdecl. cc.setId(CallConvId::kCDecl); cc.setPreservedRegs(RegGroup::kGp, Support::bitMask(Gp::kIdOs, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30)); @@ -68,7 +75,7 @@ ASMJIT_FAVOR_SIZE Error initCallConv(CallConv& cc, CallConvId ccId, const Enviro return kErrorOk; } -ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept { +ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& signature) noexcept { DebugUtils::unused(signature); const CallConv& cc = func.callConv(); @@ -77,6 +84,13 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si uint32_t i; uint32_t argCount = func.argCount(); + // Minimum stack size of a single argument passed via stack. The standard AArch64 calling convention + // specifies 8 bytes, so each function argument would occupy at least 8 bytes even if it needs less. + // However, Apple has decided to not follow this rule and function argument can occupy less, for + // example two consecutive 32-bit arguments would occupy 8 bytes total, instead of 16 as specified + // by ARM. + uint32_t minStackArgSize = cc.strategy() == CallConvStrategy::kAArch64Apple ? 4u : 8u; + if (func.hasRet()) { for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) { TypeId typeId = func._rets[valueIndex].typeId(); @@ -89,27 +103,28 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si case TypeId::kInt8: case TypeId::kInt16: case TypeId::kInt32: { - func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kInt32); + func._rets[valueIndex].initReg(RegType::kGp32, valueIndex, TypeId::kInt32); break; } case TypeId::kUInt8: case TypeId::kUInt16: case TypeId::kUInt32: { - func._rets[valueIndex].initReg(RegType::kARM_GpW, valueIndex, TypeId::kUInt32); + func._rets[valueIndex].initReg(RegType::kGp32, valueIndex, TypeId::kUInt32); break; } case TypeId::kInt64: case TypeId::kUInt64: { - func._rets[valueIndex].initReg(RegType::kARM_GpX, valueIndex, typeId); + func._rets[valueIndex].initReg(RegType::kGp64, valueIndex, typeId); break; } default: { RegType regType = regTypeFromFpOrVecTypeId(typeId); - if (regType == RegType::kNone) + if (regType == RegType::kNone) { return DebugUtils::errored(kErrorInvalidRegType); + } func._rets[valueIndex].initReg(regType, valueIndex, typeId); break; @@ -119,7 +134,8 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si } switch (cc.strategy()) { - case CallConvStrategy::kDefault: { + case CallConvStrategy::kDefault: + case CallConvStrategy::kAArch64Apple: { uint32_t gpzPos = 0; uint32_t vecPos = 0; @@ -128,19 +144,23 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si TypeId typeId = arg.typeId(); if (TypeUtils::isInt(typeId)) { - uint32_t regId = BaseReg::kIdBad; + uint32_t regId = Reg::kIdBad; - if (gpzPos < CallConv::kMaxRegArgsPerGroup) + if (gpzPos < CallConv::kMaxRegArgsPerGroup) { regId = cc._passedOrder[RegGroup::kGp].id[gpzPos]; + } - if (regId != BaseReg::kIdBad) { - RegType regType = typeId <= TypeId::kUInt32 ? RegType::kARM_GpW : RegType::kARM_GpX; + if (regId != Reg::kIdBad) { + RegType regType = typeId <= TypeId::kUInt32 ? RegType::kGp32 : RegType::kGp64; arg.assignRegData(regType, regId); func.addUsedRegs(RegGroup::kGp, Support::bitMask(regId)); gpzPos++; } else { - uint32_t size = Support::max(TypeUtils::sizeOf(typeId), registerSize); + uint32_t size = Support::max(TypeUtils::sizeOf(typeId), minStackArgSize); + if (size >= 8) { + stackOffset = Support::alignUp(stackOffset, 8); + } arg.assignStackOffset(int32_t(stackOffset)); stackOffset += size; } @@ -148,15 +168,17 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si } if (TypeUtils::isFloat(typeId) || TypeUtils::isVec(typeId)) { - uint32_t regId = BaseReg::kIdBad; + uint32_t regId = Reg::kIdBad; - if (vecPos < CallConv::kMaxRegArgsPerGroup) + if (vecPos < CallConv::kMaxRegArgsPerGroup) { regId = cc._passedOrder[RegGroup::kVec].id[vecPos]; + } - if (regId != BaseReg::kIdBad) { + if (regId != Reg::kIdBad) { RegType regType = regTypeFromFpOrVecTypeId(typeId); - if (regType == RegType::kNone) + if (regType == RegType::kNone) { return DebugUtils::errored(kErrorInvalidRegType); + } arg.initTypeId(typeId); arg.assignRegData(regType, regId); @@ -164,7 +186,10 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si vecPos++; } else { - uint32_t size = TypeUtils::sizeOf(typeId); + uint32_t size = Support::max(TypeUtils::sizeOf(typeId), minStackArgSize); + if (size >= 8) { + stackOffset = Support::alignUp(stackOffset, 8); + } arg.assignStackOffset(int32_t(stackOffset)); stackOffset += size; } @@ -178,7 +203,7 @@ ASMJIT_FAVOR_SIZE Error initFuncDetail(FuncDetail& func, const FuncSignature& si return DebugUtils::errored(kErrorInvalidState); } - func._argStackSize = stackOffset; + func._argStackSize = Support::alignUp(stackOffset, 8u); return kErrorOk; } diff --git a/pe-packer/asmjit/arm/a64func_p.h b/pe-packer/asmjit/arm/a64func_p.h index 9f531fc..2239f65 100644 --- a/pe-packer/asmjit/arm/a64func_p.h +++ b/pe-packer/asmjit/arm/a64func_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64FUNC_P_H_INCLUDED @@ -21,7 +21,7 @@ namespace FuncInternal { Error initCallConv(CallConv& cc, CallConvId ccId, const Environment& environment) noexcept; //! Initialize `FuncDetail` (AArch64 specific). -Error initFuncDetail(FuncDetail& func, const FuncSignature& signature, uint32_t registerSize) noexcept; +Error initFuncDetail(FuncDetail& func, const FuncSignature& signature) noexcept; } // {FuncInternal} diff --git a/pe-packer/asmjit/arm/a64globals.h b/pe-packer/asmjit/arm/a64globals.h index 103414f..e7d2a26 100644 --- a/pe-packer/asmjit/arm/a64globals.h +++ b/pe-packer/asmjit/arm/a64globals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64GLOBALS_H_INCLUDED @@ -15,20 +15,18 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) -// a64 uses everything from arm namespace and adds into it. -using namespace arm; - //! \addtogroup asmjit_a64 //! \{ //! AArch64 instruction. //! //! \note Only used to hold ARM-specific enumerations and static functions. -struct Inst { +namespace Inst { //! Instruction id. enum Id : uint32_t { // ${InstId:Begin} kIdNone = 0, //!< Instruction ''. + kIdAbs, //!< Instruction 'abs'. kIdAdc, //!< Instruction 'adc'. kIdAdcs, //!< Instruction 'adcs'. kIdAdd, //!< Instruction 'add'. @@ -57,6 +55,7 @@ struct Inst { kIdAutizb, //!< Instruction 'autizb'. kIdAxflag, //!< Instruction 'axflag'. kIdB, //!< Instruction 'b'. + kIdBc, //!< Instruction 'bc'. kIdBfc, //!< Instruction 'bfc'. kIdBfi, //!< Instruction 'bfi'. kIdBfm, //!< Instruction 'bfm'. @@ -67,6 +66,7 @@ struct Inst { kIdBlr, //!< Instruction 'blr'. kIdBr, //!< Instruction 'br'. kIdBrk, //!< Instruction 'brk'. + kIdBti, //!< Instruction 'bti'. kIdCas, //!< Instruction 'cas'. kIdCasa, //!< Instruction 'casa'. kIdCasab, //!< Instruction 'casab'. @@ -88,8 +88,10 @@ struct Inst { kIdCcmn, //!< Instruction 'ccmn'. kIdCcmp, //!< Instruction 'ccmp'. kIdCfinv, //!< Instruction 'cfinv'. + kIdChkfeat, //!< Instruction 'chkfeat'. kIdCinc, //!< Instruction 'cinc'. kIdCinv, //!< Instruction 'cinv'. + kIdClrbhb, //!< Instruction 'clrbhb'. kIdClrex, //!< Instruction 'clrex'. kIdCls, //!< Instruction 'cls'. kIdClz, //!< Instruction 'clz'. @@ -97,6 +99,7 @@ struct Inst { kIdCmp, //!< Instruction 'cmp'. kIdCmpp, //!< Instruction 'cmpp'. kIdCneg, //!< Instruction 'cneg'. + kIdCnt, //!< Instruction 'cnt'. kIdCrc32b, //!< Instruction 'crc32b'. kIdCrc32cb, //!< Instruction 'crc32cb'. kIdCrc32ch, //!< Instruction 'crc32ch'. @@ -112,6 +115,7 @@ struct Inst { kIdCsinc, //!< Instruction 'csinc'. kIdCsinv, //!< Instruction 'csinv'. kIdCsneg, //!< Instruction 'csneg'. + kIdCtz, //!< Instruction 'ctz'. kIdDc, //!< Instruction 'dc'. kIdDcps1, //!< Instruction 'dcps1'. kIdDcps2, //!< Instruction 'dcps2'. @@ -293,6 +297,7 @@ struct Inst { kIdPacdza, //!< Instruction 'pacdza'. kIdPacdzb, //!< Instruction 'pacdzb'. kIdPacga, //!< Instruction 'pacga'. + kIdPrfm, //!< Instruction 'prfm'. kIdPssbb, //!< Instruction 'pssbb'. kIdRbit, //!< Instruction 'rbit'. kIdRet, //!< Instruction 'ret'. @@ -313,7 +318,9 @@ struct Inst { kIdSev, //!< Instruction 'sev'. kIdSevl, //!< Instruction 'sevl'. kIdSmaddl, //!< Instruction 'smaddl'. + kIdSmax, //!< Instruction 'smax'. kIdSmc, //!< Instruction 'smc'. + kIdSmin, //!< Instruction 'smin'. kIdSmnegl, //!< Instruction 'smnegl'. kIdSmsubl, //!< Instruction 'smsubl'. kIdSmulh, //!< Instruction 'smulh'. @@ -431,6 +438,8 @@ struct Inst { kIdUdf, //!< Instruction 'udf'. kIdUdiv, //!< Instruction 'udiv'. kIdUmaddl, //!< Instruction 'umaddl'. + kIdUmax, //!< Instruction 'umax'. + kIdUmin, //!< Instruction 'umin'. kIdUmnegl, //!< Instruction 'umnegl'. kIdUmull, //!< Instruction 'umull'. kIdUmulh, //!< Instruction 'umulh'. @@ -804,7 +813,7 @@ namespace Predicate { //! Address translate options (AT). namespace AT { - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { return (op1 << 11) | (cRn << 7) | (cRm << 3) | (op2 << 0); } @@ -826,6 +835,17 @@ namespace AT { }; } +//! Branch target identification targets (BTI). +namespace BTI { + //! Branch target identification targets + enum Value : uint32_t { + kNone = 0u, + kC = 1u, + kJ = 2u, + kJC = 3u + }; +} + //! Data barrier options (DMB/DSB). namespace DB { //! Data barrier immediate values. @@ -862,7 +882,7 @@ namespace DB { //! Data cache maintenance options. namespace DC { - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { return (op1 << 11) | (cRn << 7) | (cRm << 3) | (op2 << 0); } @@ -901,7 +921,7 @@ namespace DC { //! Instruction cache maintenance options. namespace IC { - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { return (op1 << 11) | (cRn << 7) | (cRm << 3) | (op2 << 0); } @@ -955,7 +975,7 @@ namespace PSB { } namespace TLBI { - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { return (op1 << 11) | (cRn << 7) | (cRm << 3) | (op2 << 0); } @@ -1054,7 +1074,7 @@ namespace TSB { //! Processor state access through MSR. namespace PState { //! Encodes a pstate from `op0` and `op1`. - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op0, uint32_t op1) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op0, uint32_t op1) noexcept { return (op0 << 3) | (op1 << 0); } @@ -1083,17 +1103,17 @@ namespace SysReg { }; //! Encodes a system register from `op0`, `op1`, `cRn`, `cRm`, and `op2` fields. - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(uint32_t op0, uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(uint32_t op0, uint32_t op1, uint32_t cRn, uint32_t cRm, uint32_t op2) noexcept { return (op0 << 14) | (op1 << 11) | (cRn << 7) | (cRm << 3) | (op2 << 0); } //! Encodes a system register from `fields`. - static ASMJIT_INLINE_NODEBUG constexpr uint32_t encode(const Fields& fields) noexcept { + static ASMJIT_INLINE_CONSTEXPR uint32_t encode(const Fields& fields) noexcept { return encode(fields.op0, fields.op1, fields.cRn, fields.cRm, fields.op2); } //! Decodes a system register to \ref Fields. - static ASMJIT_INLINE_NODEBUG constexpr Fields decode(uint32_t id) noexcept { + static ASMJIT_INLINE_CONSTEXPR Fields decode(uint32_t id) noexcept { return Fields { uint8_t((id >> 14) & 0x3u), uint8_t((id >> 11) & 0x7u), diff --git a/pe-packer/asmjit/arm/a64instapi.cpp b/pe-packer/asmjit/arm/a64instapi.cpp index 82bed0e..003db32 100644 --- a/pe-packer/asmjit/arm/a64instapi.cpp +++ b/pe-packer/asmjit/arm/a64instapi.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -15,65 +15,35 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) +namespace InstInternal { + // a64::InstInternal - Text // ======================== #ifndef ASMJIT_NO_TEXT -Error InstInternal::instIdToString(Arch arch, InstId instId, String& output) noexcept { +Error instIdToString(InstId instId, InstStringifyOptions options, String& output) noexcept { uint32_t realId = instId & uint32_t(InstIdParts::kRealId); - DebugUtils::unused(arch); - - if (ASMJIT_UNLIKELY(!Inst::isDefinedId(realId))) + if (ASMJIT_UNLIKELY(!Inst::isDefinedId(realId))) { return DebugUtils::errored(kErrorInvalidInstruction); - - - char nameData[32]; - size_t nameSize = Support::decodeInstName(nameData, InstDB::_instNameIndexTable[realId], InstDB::_instNameStringTable); - - return output.append(nameData, nameSize); -} - -InstId InstInternal::stringToInstId(Arch arch, const char* s, size_t len) noexcept { - DebugUtils::unused(arch); - - if (ASMJIT_UNLIKELY(!s)) - return Inst::kIdNone; - - if (len == SIZE_MAX) - len = strlen(s); - - if (ASMJIT_UNLIKELY(len == 0 || len > InstDB::kMaxNameSize)) - return Inst::kIdNone; - - uint32_t prefix = uint32_t(s[0]) - 'a'; - if (ASMJIT_UNLIKELY(prefix > 'z' - 'a')) - return Inst::kIdNone; - - size_t base = InstDB::instNameIndex[prefix].start; - size_t end = InstDB::instNameIndex[prefix].end; - - if (ASMJIT_UNLIKELY(!base)) - return Inst::kIdNone; - - char nameData[32]; - for (size_t lim = end - base; lim != 0; lim >>= 1) { - size_t instId = base + (lim >> 1); - size_t nameSize = Support::decodeInstName(nameData, InstDB::_instNameIndexTable[instId], InstDB::_instNameStringTable); - - int result = Support::compareStringViews(s, len, nameData, nameSize); - if (result < 0) - continue; - - if (result > 0) { - base = instId + 1; - lim--; - continue; - } - - return InstId(instId); } - return Inst::kIdNone; + return InstNameUtils::decode(InstDB::_instNameIndexTable[realId], options, InstDB::_instNameStringTable, output); +} + +InstId stringToInstId(const char* s, size_t len) noexcept { + if (ASMJIT_UNLIKELY(!s)) { + return BaseInst::kIdNone; + } + + if (len == SIZE_MAX) { + len = strlen(s); + } + + if (len == 0u || len > InstDB::instNameIndex.maxNameLength) { + return BaseInst::kIdNone; + } + + return InstNameUtils::findInstruction(s, len, InstDB::_instNameIndexTable, InstDB::_instNameStringTable, InstDB::instNameIndex); } #endif // !ASMJIT_NO_TEXT @@ -81,9 +51,9 @@ InstId InstInternal::stringToInstId(Arch arch, const char* s, size_t len) noexce // ============================ #ifndef ASMJIT_NO_VALIDATION -ASMJIT_FAVOR_SIZE Error InstInternal::validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept { +ASMJIT_FAVOR_SIZE Error validate(const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept { // TODO: - DebugUtils::unused(arch, inst, operands, opCount, validationFlags); + DebugUtils::unused(inst, operands, opCount, validationFlags); return kErrorOk; } #endif // !ASMJIT_NO_VALIDATION @@ -127,18 +97,13 @@ static const InstRWInfoData instRWInfoData[] = { static const uint8_t elementTypeSize[8] = { 0, 1, 2, 4, 8, 4, 4, 0 }; -Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept { - // Unused in Release configuration as the assert is not compiled in. - DebugUtils::unused(arch); - - // Only called when `arch` matches X86 family. - ASMJIT_ASSERT(Environment::isFamilyARM(arch)); - +Error queryRWInfo(const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept { // Get the instruction data. uint32_t realId = inst.id() & uint32_t(InstIdParts::kRealId); - if (ASMJIT_UNLIKELY(!Inst::isDefinedId(realId))) + if (ASMJIT_UNLIKELY(!Inst::isDefinedId(realId))) { return DebugUtils::errored(kErrorInvalidInstruction); + } out->_instFlags = InstRWFlags::kNone; out->_opCount = uint8_t(opCount); @@ -163,7 +128,7 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* OpRWFlags rwFlags = i < opCount - 1 ? (OpRWFlags)rwInfo.rwx[0] : (OpRWFlags)rwInfo.rwx[1]; op._opFlags = rwFlags & ~(OpRWFlags::kZExt); - op._physId = BaseReg::kIdBad; + op._physId = Reg::kIdBad; op._rmSize = 0; op._resetReserved(); @@ -176,21 +141,25 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* op._consecutiveLeadCount = 0; if (srcOp.isReg()) { - if (i == 0) + if (i == 0) { op._consecutiveLeadCount = uint8_t(opCount - 1); - else + } + else { op.addOpFlags(OpRWFlags::kConsecutive); + } } else { const Mem& memOp = srcOp.as(); if (memOp.hasBase()) { op.addOpFlags(OpRWFlags::kMemBaseRead); + if ((memOp.hasIndex() || memOp.hasOffset()) && memOp.isPreOrPost()) { + op.addOpFlags(OpRWFlags::kMemBaseWrite); + } } if (memOp.hasIndex()) { op.addOpFlags(OpRWFlags::kMemIndexRead); - op.addOpFlags(memOp.isPreOrPost() ? OpRWFlags::kMemIndexWrite : OpRWFlags::kNone); } } } @@ -208,7 +177,7 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* OpRWFlags rwFlags = (OpRWFlags)rwInfo.rwx[i]; op._opFlags = rwFlags & ~(OpRWFlags::kZExt); - op._physId = BaseReg::kIdBad; + op._physId = Reg::kIdBad; op._rmSize = 0; op._resetReserved(); @@ -223,10 +192,10 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* if (srcOp.isReg()) { if (srcOp.as().hasElementIndex()) { // Only part of the vector is accessed if element index [] is used. - uint32_t elementType = srcOp.as().elementType(); + VecElementType elementType = srcOp.as().elementType(); uint32_t elementIndex = srcOp.as().elementIndex(); - uint32_t elementSize = elementTypeSize[elementType]; + uint32_t elementSize = elementTypeSize[size_t(elementType)]; uint64_t accessMask = uint64_t(Support::lsbMask(elementSize)) << (elementIndex * elementSize); op._readByteMask &= accessMask; @@ -240,11 +209,13 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* if (memOp.hasBase()) { op.addOpFlags(OpRWFlags::kMemBaseRead); + if ((memOp.hasIndex() || memOp.hasOffset()) && memOp.isPreOrPost()) { + op.addOpFlags(OpRWFlags::kMemBaseWrite); + } } if (memOp.hasIndex()) { op.addOpFlags(OpRWFlags::kMemIndexRead); - op.addOpFlags(memOp.isPreOrPost() ? OpRWFlags::kMemIndexWrite : OpRWFlags::kNone); } } } @@ -258,13 +229,15 @@ Error InstInternal::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* // ================================= #ifndef ASMJIT_NO_INTROSPECTION -Error InstInternal::queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept { +Error queryFeatures(const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept { // TODO: [ARM] QueryFeatures not implemented yet. - DebugUtils::unused(arch, inst, operands, opCount, out); + DebugUtils::unused(inst, operands, opCount, out); return kErrorOk; } #endif // !ASMJIT_NO_INTROSPECTION +} // {InstInternal} + // a64::InstInternal - Unit // ======================== diff --git a/pe-packer/asmjit/arm/a64instapi_p.h b/pe-packer/asmjit/arm/a64instapi_p.h index 320a3e8..772bb42 100644 --- a/pe-packer/asmjit/arm/a64instapi_p.h +++ b/pe-packer/asmjit/arm/a64instapi_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTAPI_P_H_INCLUDED @@ -18,17 +18,17 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) namespace InstInternal { #ifndef ASMJIT_NO_TEXT -Error ASMJIT_CDECL instIdToString(Arch arch, InstId instId, String& output) noexcept; -InstId ASMJIT_CDECL stringToInstId(Arch arch, const char* s, size_t len) noexcept; +Error ASMJIT_CDECL instIdToString(InstId instId, InstStringifyOptions options, String& output) noexcept; +InstId ASMJIT_CDECL stringToInstId(const char* s, size_t len) noexcept; #endif // !ASMJIT_NO_TEXT #ifndef ASMJIT_NO_VALIDATION -Error ASMJIT_CDECL validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept; +Error ASMJIT_CDECL validate(const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept; #endif // !ASMJIT_NO_VALIDATION #ifndef ASMJIT_NO_INTROSPECTION -Error ASMJIT_CDECL queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept; -Error ASMJIT_CDECL queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept; +Error ASMJIT_CDECL queryRWInfo(const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept; +Error ASMJIT_CDECL queryFeatures(const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept; #endif // !ASMJIT_NO_INTROSPECTION } // {InstInternal} diff --git a/pe-packer/asmjit/arm/a64instdb.cpp b/pe-packer/asmjit/arm/a64instdb.cpp index 5d04b71..4b2170b 100644 --- a/pe-packer/asmjit/arm/a64instdb.cpp +++ b/pe-packer/asmjit/arm/a64instdb.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -43,9 +43,6 @@ ERETAA, ERETAB: Exception Return, with pointer authentication. LDAPxxx PACIA, PACIA1716, PACIASP, PACIAZ, PACIZA: Pointer Authentication Code for Instruction address, using key A. PACIB, PACIB1716, PACIBSP, PACIBZ, PACIZB: Pointer Authentication Code for Instruction address, using key B. -PRFM (immediate): Prefetch Memory (immediate). -PRFM (literal): Prefetch Memory (literal). -PRFM (register): Prefetch Memory (register). PRFUM: Prefetch Memory (unscaled offset). RETAA, RETAB: Return from subroutine, with pointer authentication. RMIF: Rotate, Mask Insert Flags. @@ -59,769 +56,781 @@ const InstInfo _instInfoTable[] = { // +------------------+---------------------+--------------------------------------------------------------------------------------+-----------+---------------------------+----+ // ${InstInfo:Begin} INST(None , None , (_) , 0 , 0 , 0 ), // #0 - INST(Adc , BaseRRR , (0b0001101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #1 - INST(Adcs , BaseRRR , (0b0011101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #2 - INST(Add , BaseAddSub , (0b0001011000, 0b0001011001, 0b0010001) , kRWI_W , 0 , 0 ), // #3 - INST(Addg , BaseRRII , (0b1001000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 0 ), // #4 - INST(Adds , BaseAddSub , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_W , 0 , 1 ), // #5 - INST(Adr , BaseAdr , (0b0001000000000000000000, OffsetType::kAArch64_ADR) , kRWI_W , 0 , 0 ), // #6 - INST(Adrp , BaseAdr , (0b1001000000000000000000, OffsetType::kAArch64_ADRP) , kRWI_W , 0 , 1 ), // #7 - INST(And , BaseLogical , (0b0001010000, 0b00100100, 0) , kRWI_W , 0 , 0 ), // #8 - INST(Ands , BaseLogical , (0b1101010000, 0b11100100, 0) , kRWI_W , 0 , 1 ), // #9 - INST(Asr , BaseShift , (0b0001101011000000001010, 0b0001001100000000011111, 0) , kRWI_W , 0 , 0 ), // #10 - INST(Asrv , BaseShift , (0b0001101011000000001010, 0b0000000000000000000000, 0) , kRWI_W , 0 , 1 ), // #11 - INST(At , BaseAtDcIcTlbi , (0b00011111110000, 0b00001111000000, true) , kRWI_RX , 0 , 0 ), // #12 - INST(Autda , BaseRR , (0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 0 ), // #13 - INST(Autdza , BaseR , (0b11011010110000010011101111100000, kX, kZR, 0) , kRWI_X , 0 , 0 ), // #14 - INST(Autdb , BaseRR , (0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 1 ), // #15 - INST(Autdzb , BaseR , (0b11011010110000010011111111100000, kX, kZR, 0) , kRWI_X , 0 , 1 ), // #16 - INST(Autia , BaseRR , (0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 2 ), // #17 - INST(Autia1716 , BaseOp , (0b11010101000000110010000110011111) , 0 , 0 , 0 ), // #18 - INST(Autiasp , BaseOp , (0b11010101000000110010001110111111) , 0 , 0 , 1 ), // #19 - INST(Autiaz , BaseOp , (0b11010101000000110010001110011111) , 0 , 0 , 2 ), // #20 - INST(Autib , BaseRR , (0b11011010110000010001010000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 3 ), // #21 - INST(Autib1716 , BaseOp , (0b11010101000000110010000111011111) , 0 , 0 , 3 ), // #22 - INST(Autibsp , BaseOp , (0b11010101000000110010001111111111) , 0 , 0 , 4 ), // #23 - INST(Autibz , BaseOp , (0b11010101000000110010001111011111) , 0 , 0 , 5 ), // #24 - INST(Autiza , BaseR , (0b11011010110000010011001111100000, kX, kZR, 0) , kRWI_X , 0 , 2 ), // #25 - INST(Autizb , BaseR , (0b11011010110000010011011111100000, kX, kZR, 0) , kRWI_X , 0 , 3 ), // #26 - INST(Axflag , BaseOp , (0b11010101000000000100000001011111) , 0 , 0 , 6 ), // #27 - INST(B , BaseBranchRel , (0b00010100000000000000000000000000) , 0 , F(Cond) , 0 ), // #28 - INST(Bfc , BaseBfc , (0b00110011000000000000001111100000) , kRWI_X , 0 , 0 ), // #29 - INST(Bfi , BaseBfi , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #30 - INST(Bfm , BaseBfm , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #31 - INST(Bfxil , BaseBfx , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #32 - INST(Bic , BaseLogical , (0b0001010001, 0b00100100, 1) , kRWI_W , 0 , 2 ), // #33 - INST(Bics , BaseLogical , (0b1101010001, 0b11100100, 1) , kRWI_W , 0 , 3 ), // #34 - INST(Bl , BaseBranchRel , (0b10010100000000000000000000000000) , 0 , 0 , 1 ), // #35 - INST(Blr , BaseBranchReg , (0b11010110001111110000000000000000) , kRWI_R , 0 , 0 ), // #36 - INST(Br , BaseBranchReg , (0b11010110000111110000000000000000) , kRWI_R , 0 , 1 ), // #37 - INST(Brk , BaseOpImm , (0b11010100001000000000000000000000, 16, 5) , 0 , 0 , 0 ), // #38 - INST(Cas , BaseAtomicOp , (0b1000100010100000011111, kWX, 30, 0) , kRWI_XRX , 0 , 0 ), // #39 - INST(Casa , BaseAtomicOp , (0b1000100011100000011111, kWX, 30, 1) , kRWI_XRX , 0 , 1 ), // #40 - INST(Casab , BaseAtomicOp , (0b0000100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 2 ), // #41 - INST(Casah , BaseAtomicOp , (0b0100100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 3 ), // #42 - INST(Casal , BaseAtomicOp , (0b1000100011100000111111, kWX, 30, 1) , kRWI_XRX , 0 , 4 ), // #43 - INST(Casalb , BaseAtomicOp , (0b0000100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 5 ), // #44 - INST(Casalh , BaseAtomicOp , (0b0100100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 6 ), // #45 - INST(Casb , BaseAtomicOp , (0b0000100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 7 ), // #46 - INST(Cash , BaseAtomicOp , (0b0100100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 8 ), // #47 - INST(Casl , BaseAtomicOp , (0b1000100010100000111111, kWX, 30, 0) , kRWI_XRX , 0 , 9 ), // #48 - INST(Caslb , BaseAtomicOp , (0b0000100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 10 ), // #49 - INST(Caslh , BaseAtomicOp , (0b0100100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 11 ), // #50 - INST(Casp , BaseAtomicCasp , (0b0000100000100000011111, kWX, 30) , kRWI_XXRRX, 0 , 0 ), // #51 - INST(Caspa , BaseAtomicCasp , (0b0000100001100000011111, kWX, 30) , kRWI_XXRRX, 0 , 1 ), // #52 - INST(Caspal , BaseAtomicCasp , (0b0000100001100000111111, kWX, 30) , kRWI_XXRRX, 0 , 2 ), // #53 - INST(Caspl , BaseAtomicCasp , (0b0000100000100000111111, kWX, 30) , kRWI_XXRRX, 0 , 3 ), // #54 - INST(Cbnz , BaseBranchCmp , (0b00110101000000000000000000000000) , kRWI_R , 0 , 0 ), // #55 - INST(Cbz , BaseBranchCmp , (0b00110100000000000000000000000000) , kRWI_R , 0 , 1 ), // #56 - INST(Ccmn , BaseCCmp , (0b00111010010000000000000000000000) , kRWI_R , 0 , 0 ), // #57 - INST(Ccmp , BaseCCmp , (0b01111010010000000000000000000000) , kRWI_R , 0 , 1 ), // #58 - INST(Cfinv , BaseOp , (0b11010101000000000100000000011111) , 0 , 0 , 7 ), // #59 - INST(Cinc , BaseCInc , (0b00011010100000000000010000000000) , kRWI_W , 0 , 0 ), // #60 - INST(Cinv , BaseCInc , (0b01011010100000000000000000000000) , kRWI_W , 0 , 1 ), // #61 - INST(Clrex , BaseOpImm , (0b11010101000000110011000001011111, 4, 8) , 0 , 0 , 1 ), // #62 - INST(Cls , BaseRR , (0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 4 ), // #63 - INST(Clz , BaseRR , (0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 5 ), // #64 - INST(Cmn , BaseCmpCmn , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_R , 0 , 0 ), // #65 - INST(Cmp , BaseCmpCmn , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_R , 0 , 1 ), // #66 - INST(Cmpp , BaseRR , (0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true) , kRWI_R , 0 , 6 ), // #67 - INST(Cneg , BaseCInc , (0b01011010100000000000010000000000) , kRWI_W , 0 , 2 ), // #68 - INST(Crc32b , BaseRRR , (0b0001101011000000010000, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 2 ), // #69 - INST(Crc32cb , BaseRRR , (0b0001101011000000010100, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 3 ), // #70 - INST(Crc32ch , BaseRRR , (0b0001101011000000010101, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 4 ), // #71 - INST(Crc32cw , BaseRRR , (0b0001101011000000010110, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 5 ), // #72 - INST(Crc32cx , BaseRRR , (0b1001101011000000010111, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 6 ), // #73 - INST(Crc32h , BaseRRR , (0b0001101011000000010001, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 7 ), // #74 - INST(Crc32w , BaseRRR , (0b0001101011000000010010, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 8 ), // #75 - INST(Crc32x , BaseRRR , (0b1001101011000000010011, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 9 ), // #76 - INST(Csdb , BaseOp , (0b11010101000000110010001010011111) , 0 , 0 , 8 ), // #77 - INST(Csel , BaseCSel , (0b00011010100000000000000000000000) , kRWI_W , 0 , 0 ), // #78 - INST(Cset , BaseCSet , (0b00011010100111110000011111100000) , kRWI_W , 0 , 0 ), // #79 - INST(Csetm , BaseCSet , (0b01011010100111110000001111100000) , kRWI_W , 0 , 1 ), // #80 - INST(Csinc , BaseCSel , (0b00011010100000000000010000000000) , kRWI_W , 0 , 1 ), // #81 - INST(Csinv , BaseCSel , (0b01011010100000000000000000000000) , kRWI_W , 0 , 2 ), // #82 - INST(Csneg , BaseCSel , (0b01011010100000000000010000000000) , kRWI_W , 0 , 3 ), // #83 - INST(Dc , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, true) , kRWI_RX , 0 , 1 ), // #84 - INST(Dcps1 , BaseOpImm , (0b11010100101000000000000000000001, 16, 5) , 0 , 0 , 2 ), // #85 - INST(Dcps2 , BaseOpImm , (0b11010100101000000000000000000010, 16, 5) , 0 , 0 , 3 ), // #86 - INST(Dcps3 , BaseOpImm , (0b11010100101000000000000000000011, 16, 5) , 0 , 0 , 4 ), // #87 - INST(Dgh , BaseOp , (0b11010101000000110010000011011111) , 0 , 0 , 9 ), // #88 - INST(Dmb , BaseOpImm , (0b11010101000000110011000010111111, 4, 8) , 0 , 0 , 5 ), // #89 - INST(Drps , BaseOp , (0b11010110101111110000001111100000) , 0 , 0 , 10 ), // #90 - INST(Dsb , BaseOpImm , (0b11010101000000110011000010011111, 4, 8) , 0 , 0 , 6 ), // #91 - INST(Eon , BaseLogical , (0b1001010001, 0b10100100, 1) , kRWI_W , 0 , 4 ), // #92 - INST(Eor , BaseLogical , (0b1001010000, 0b10100100, 0) , kRWI_W , 0 , 5 ), // #93 - INST(Esb , BaseOp , (0b11010101000000110010001000011111) , 0 , 0 , 11 ), // #94 - INST(Extr , BaseExtract , (0b00010011100000000000000000000000) , kRWI_W , 0 , 0 ), // #95 - INST(Eret , BaseOp , (0b11010110100111110000001111100000) , 0 , 0 , 12 ), // #96 - INST(Gmi , BaseRRR , (0b1001101011000000000101, kX , kZR, kX , kSP, kX , kZR, true) , kRWI_W , 0 , 10 ), // #97 - INST(Hint , BaseOpImm , (0b11010101000000110010000000011111, 7, 5) , 0 , 0 , 7 ), // #98 - INST(Hlt , BaseOpImm , (0b11010100010000000000000000000000, 16, 5) , 0 , 0 , 8 ), // #99 - INST(Hvc , BaseOpImm , (0b11010100000000000000000000000010, 16, 5) , 0 , 0 , 9 ), // #100 - INST(Ic , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, false) , kRWI_RX , 0 , 2 ), // #101 - INST(Isb , BaseOpImm , (0b11010101000000110011000011011111, 4, 8) , 0 , 0 , 10 ), // #102 - INST(Ldadd , BaseAtomicOp , (0b1011100000100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 12 ), // #103 - INST(Ldadda , BaseAtomicOp , (0b1011100010100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 13 ), // #104 - INST(Ldaddab , BaseAtomicOp , (0b0011100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 14 ), // #105 - INST(Ldaddah , BaseAtomicOp , (0b0111100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 15 ), // #106 - INST(Ldaddal , BaseAtomicOp , (0b1011100011100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 16 ), // #107 - INST(Ldaddalb , BaseAtomicOp , (0b0011100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 17 ), // #108 - INST(Ldaddalh , BaseAtomicOp , (0b0111100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 18 ), // #109 - INST(Ldaddb , BaseAtomicOp , (0b0011100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 19 ), // #110 - INST(Ldaddh , BaseAtomicOp , (0b0111100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 20 ), // #111 - INST(Ldaddl , BaseAtomicOp , (0b1011100001100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 21 ), // #112 - INST(Ldaddlb , BaseAtomicOp , (0b0011100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 22 ), // #113 - INST(Ldaddlh , BaseAtomicOp , (0b0111100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 23 ), // #114 - INST(Ldar , BaseRM_NoImm , (0b1000100011011111111111, kWX, kZR, 30) , kRWI_W , 0 , 0 ), // #115 - INST(Ldarb , BaseRM_NoImm , (0b0000100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 1 ), // #116 - INST(Ldarh , BaseRM_NoImm , (0b0100100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 2 ), // #117 - INST(Ldaxp , BaseLdxp , (0b1000100001111111100000, kWX, 30) , kRWI_WW , 0 , 0 ), // #118 - INST(Ldaxr , BaseRM_NoImm , (0b1000100001011111111111, kWX, kZR, 30) , kRWI_W , 0 , 3 ), // #119 - INST(Ldaxrb , BaseRM_NoImm , (0b0000100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 4 ), // #120 - INST(Ldaxrh , BaseRM_NoImm , (0b0100100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 5 ), // #121 - INST(Ldclr , BaseAtomicOp , (0b1011100000100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 24 ), // #122 - INST(Ldclra , BaseAtomicOp , (0b1011100010100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 25 ), // #123 - INST(Ldclrab , BaseAtomicOp , (0b0011100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 26 ), // #124 - INST(Ldclrah , BaseAtomicOp , (0b0111100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 27 ), // #125 - INST(Ldclral , BaseAtomicOp , (0b1011100011100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 28 ), // #126 - INST(Ldclralb , BaseAtomicOp , (0b0011100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 29 ), // #127 - INST(Ldclralh , BaseAtomicOp , (0b0111100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 30 ), // #128 - INST(Ldclrb , BaseAtomicOp , (0b0011100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 31 ), // #129 - INST(Ldclrh , BaseAtomicOp , (0b0111100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 32 ), // #130 - INST(Ldclrl , BaseAtomicOp , (0b1011100001100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 33 ), // #131 - INST(Ldclrlb , BaseAtomicOp , (0b0011100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 34 ), // #132 - INST(Ldclrlh , BaseAtomicOp , (0b0111100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 35 ), // #133 - INST(Ldeor , BaseAtomicOp , (0b1011100000100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 36 ), // #134 - INST(Ldeora , BaseAtomicOp , (0b1011100010100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 37 ), // #135 - INST(Ldeorab , BaseAtomicOp , (0b0011100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 38 ), // #136 - INST(Ldeorah , BaseAtomicOp , (0b0111100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 39 ), // #137 - INST(Ldeoral , BaseAtomicOp , (0b1011100011100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 40 ), // #138 - INST(Ldeoralb , BaseAtomicOp , (0b0011100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 41 ), // #139 - INST(Ldeoralh , BaseAtomicOp , (0b0111100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 42 ), // #140 - INST(Ldeorb , BaseAtomicOp , (0b0011100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 43 ), // #141 - INST(Ldeorh , BaseAtomicOp , (0b0111100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 44 ), // #142 - INST(Ldeorl , BaseAtomicOp , (0b1011100001100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 45 ), // #143 - INST(Ldeorlb , BaseAtomicOp , (0b0011100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 46 ), // #144 - INST(Ldeorlh , BaseAtomicOp , (0b0111100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 47 ), // #145 - INST(Ldg , BaseRM_SImm9 , (0b1101100101100000000000, 0b0000000000000000000000, kX , kZR, 0, 4) , kRWI_W , 0 , 0 ), // #146 - INST(Ldgm , BaseRM_NoImm , (0b1101100111100000000000, kX , kZR, 0 ) , kRWI_W , 0 , 6 ), // #147 - INST(Ldlar , BaseRM_NoImm , (0b1000100011011111011111, kWX, kZR, 30) , kRWI_W , 0 , 7 ), // #148 - INST(Ldlarb , BaseRM_NoImm , (0b0000100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 8 ), // #149 - INST(Ldlarh , BaseRM_NoImm , (0b0100100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 9 ), // #150 - INST(Ldnp , BaseLdpStp , (0b0010100001, 0 , kWX, 31, 2) , kRWI_WW , 0 , 0 ), // #151 - INST(Ldp , BaseLdpStp , (0b0010100101, 0b0010100011, kWX, 31, 2) , kRWI_W , 0 , 1 ), // #152 - INST(Ldpsw , BaseLdpStp , (0b0110100101, 0b0110100011, kX , 0 , 2) , kRWI_WW , 0 , 2 ), // #153 - INST(Ldr , BaseLdSt , (0b1011100101, 0b10111000010, 0b10111000011, 0b00011000, kWX, 30, 2, Inst::kIdLdur) , kRWI_W , 0 , 0 ), // #154 - INST(Ldraa , BaseRM_SImm10 , (0b1111100000100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 0 ), // #155 - INST(Ldrab , BaseRM_SImm10 , (0b1111100010100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 1 ), // #156 - INST(Ldrb , BaseLdSt , (0b0011100101, 0b00111000010, 0b00111000011, 0 , kW , 0 , 0, Inst::kIdLdurb) , kRWI_W , 0 , 1 ), // #157 - INST(Ldrh , BaseLdSt , (0b0111100101, 0b01111000010, 0b01111000011, 0 , kW , 0 , 1, Inst::kIdLdurh) , kRWI_W , 0 , 2 ), // #158 - INST(Ldrsb , BaseLdSt , (0b0011100111, 0b00111000100, 0b00111000111, 0 , kWX, 22, 0, Inst::kIdLdursb) , kRWI_W , 0 , 3 ), // #159 - INST(Ldrsh , BaseLdSt , (0b0111100111, 0b01111000100, 0b01111000111, 0 , kWX, 22, 1, Inst::kIdLdursh) , kRWI_W , 0 , 4 ), // #160 - INST(Ldrsw , BaseLdSt , (0b1011100110, 0b10111000100, 0b10111000101, 0b10011000, kX , 0 , 2, Inst::kIdLdursw) , kRWI_W , 0 , 5 ), // #161 - INST(Ldset , BaseAtomicOp , (0b1011100000100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 48 ), // #162 - INST(Ldseta , BaseAtomicOp , (0b1011100010100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 49 ), // #163 - INST(Ldsetab , BaseAtomicOp , (0b0011100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 50 ), // #164 - INST(Ldsetah , BaseAtomicOp , (0b0111100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 51 ), // #165 - INST(Ldsetal , BaseAtomicOp , (0b1011100011100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 52 ), // #166 - INST(Ldsetalb , BaseAtomicOp , (0b0011100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 53 ), // #167 - INST(Ldsetalh , BaseAtomicOp , (0b0111100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 54 ), // #168 - INST(Ldsetb , BaseAtomicOp , (0b0011100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 55 ), // #169 - INST(Ldseth , BaseAtomicOp , (0b0111100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 56 ), // #170 - INST(Ldsetl , BaseAtomicOp , (0b1011100001100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 57 ), // #171 - INST(Ldsetlb , BaseAtomicOp , (0b0011100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 58 ), // #172 - INST(Ldsetlh , BaseAtomicOp , (0b0111100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 59 ), // #173 - INST(Ldsmax , BaseAtomicOp , (0b1011100000100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 60 ), // #174 - INST(Ldsmaxa , BaseAtomicOp , (0b1011100010100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 61 ), // #175 - INST(Ldsmaxab , BaseAtomicOp , (0b0011100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 62 ), // #176 - INST(Ldsmaxah , BaseAtomicOp , (0b0111100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 63 ), // #177 - INST(Ldsmaxal , BaseAtomicOp , (0b1011100011100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 64 ), // #178 - INST(Ldsmaxalb , BaseAtomicOp , (0b0011100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 65 ), // #179 - INST(Ldsmaxalh , BaseAtomicOp , (0b0111100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 66 ), // #180 - INST(Ldsmaxb , BaseAtomicOp , (0b0011100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 67 ), // #181 - INST(Ldsmaxh , BaseAtomicOp , (0b0111100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 68 ), // #182 - INST(Ldsmaxl , BaseAtomicOp , (0b1011100001100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 69 ), // #183 - INST(Ldsmaxlb , BaseAtomicOp , (0b0011100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 70 ), // #184 - INST(Ldsmaxlh , BaseAtomicOp , (0b0111100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 71 ), // #185 - INST(Ldsmin , BaseAtomicOp , (0b1011100000100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 72 ), // #186 - INST(Ldsmina , BaseAtomicOp , (0b1011100010100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 73 ), // #187 - INST(Ldsminab , BaseAtomicOp , (0b0011100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 74 ), // #188 - INST(Ldsminah , BaseAtomicOp , (0b0111100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 75 ), // #189 - INST(Ldsminal , BaseAtomicOp , (0b1011100011100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 76 ), // #190 - INST(Ldsminalb , BaseAtomicOp , (0b0011100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 77 ), // #191 - INST(Ldsminalh , BaseAtomicOp , (0b0111100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 78 ), // #192 - INST(Ldsminb , BaseAtomicOp , (0b0011100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 79 ), // #193 - INST(Ldsminh , BaseAtomicOp , (0b0111100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 80 ), // #194 - INST(Ldsminl , BaseAtomicOp , (0b1011100001100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 81 ), // #195 - INST(Ldsminlb , BaseAtomicOp , (0b0011100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 82 ), // #196 - INST(Ldsminlh , BaseAtomicOp , (0b0111100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 83 ), // #197 - INST(Ldtr , BaseRM_SImm9 , (0b1011100001000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 1 ), // #198 - INST(Ldtrb , BaseRM_SImm9 , (0b0011100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 2 ), // #199 - INST(Ldtrh , BaseRM_SImm9 , (0b0111100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 3 ), // #200 - INST(Ldtrsb , BaseRM_SImm9 , (0b0011100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 4 ), // #201 - INST(Ldtrsh , BaseRM_SImm9 , (0b0111100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 5 ), // #202 - INST(Ldtrsw , BaseRM_SImm9 , (0b1011100010000000000010, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 6 ), // #203 - INST(Ldumax , BaseAtomicOp , (0b1011100000100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 84 ), // #204 - INST(Ldumaxa , BaseAtomicOp , (0b1011100010100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 85 ), // #205 - INST(Ldumaxab , BaseAtomicOp , (0b0011100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 86 ), // #206 - INST(Ldumaxah , BaseAtomicOp , (0b0111100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 87 ), // #207 - INST(Ldumaxal , BaseAtomicOp , (0b1011100011100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 88 ), // #208 - INST(Ldumaxalb , BaseAtomicOp , (0b0011100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 89 ), // #209 - INST(Ldumaxalh , BaseAtomicOp , (0b0111100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 90 ), // #210 - INST(Ldumaxb , BaseAtomicOp , (0b0011100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 91 ), // #211 - INST(Ldumaxh , BaseAtomicOp , (0b0111100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 92 ), // #212 - INST(Ldumaxl , BaseAtomicOp , (0b1011100001100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 93 ), // #213 - INST(Ldumaxlb , BaseAtomicOp , (0b0011100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 94 ), // #214 - INST(Ldumaxlh , BaseAtomicOp , (0b0111100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 95 ), // #215 - INST(Ldumin , BaseAtomicOp , (0b1011100000100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 96 ), // #216 - INST(Ldumina , BaseAtomicOp , (0b1011100010100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 97 ), // #217 - INST(Lduminab , BaseAtomicOp , (0b0011100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 98 ), // #218 - INST(Lduminah , BaseAtomicOp , (0b0111100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 99 ), // #219 - INST(Lduminal , BaseAtomicOp , (0b1011100011100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 100), // #220 - INST(Lduminalb , BaseAtomicOp , (0b0011100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 101), // #221 - INST(Lduminalh , BaseAtomicOp , (0b0111100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 102), // #222 - INST(Lduminb , BaseAtomicOp , (0b0011100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 103), // #223 - INST(Lduminh , BaseAtomicOp , (0b0111100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 104), // #224 - INST(Lduminl , BaseAtomicOp , (0b1011100001100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 105), // #225 - INST(Lduminlb , BaseAtomicOp , (0b0011100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 106), // #226 - INST(Lduminlh , BaseAtomicOp , (0b0111100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 107), // #227 - INST(Ldur , BaseRM_SImm9 , (0b1011100001000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 7 ), // #228 - INST(Ldurb , BaseRM_SImm9 , (0b0011100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 8 ), // #229 - INST(Ldurh , BaseRM_SImm9 , (0b0111100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 9 ), // #230 - INST(Ldursb , BaseRM_SImm9 , (0b0011100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 10 ), // #231 - INST(Ldursh , BaseRM_SImm9 , (0b0111100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 11 ), // #232 - INST(Ldursw , BaseRM_SImm9 , (0b1011100010000000000000, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 12 ), // #233 - INST(Ldxp , BaseLdxp , (0b1000100001111111000000, kWX, 30) , kRWI_WW , 0 , 1 ), // #234 - INST(Ldxr , BaseRM_NoImm , (0b1000100001011111011111, kWX, kZR, 30) , kRWI_W , 0 , 10 ), // #235 - INST(Ldxrb , BaseRM_NoImm , (0b0000100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 11 ), // #236 - INST(Ldxrh , BaseRM_NoImm , (0b0100100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 12 ), // #237 - INST(Lsl , BaseShift , (0b0001101011000000001000, 0b0101001100000000000000, 0) , kRWI_W , 0 , 2 ), // #238 - INST(Lslv , BaseShift , (0b0001101011000000001000, 0b0000000000000000000000, 0) , kRWI_W , 0 , 3 ), // #239 - INST(Lsr , BaseShift , (0b0001101011000000001001, 0b0101001100000000011111, 0) , kRWI_W , 0 , 4 ), // #240 - INST(Lsrv , BaseShift , (0b0001101011000000001001, 0b0000000000000000000000, 0) , kRWI_W , 0 , 5 ), // #241 - INST(Madd , BaseRRRR , (0b0001101100000000000000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #242 - INST(Mneg , BaseRRR , (0b0001101100000000111111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 11 ), // #243 - INST(Mov , BaseMov , (_) , kRWI_W , 0 , 0 ), // #244 - INST(Movk , BaseMovKNZ , (0b01110010100000000000000000000000) , kRWI_X , 0 , 0 ), // #245 - INST(Movn , BaseMovKNZ , (0b00010010100000000000000000000000) , kRWI_W , 0 , 1 ), // #246 - INST(Movz , BaseMovKNZ , (0b01010010100000000000000000000000) , kRWI_W , 0 , 2 ), // #247 - INST(Mrs , BaseMrs , (_) , kRWI_W , 0 , 0 ), // #248 - INST(Msr , BaseMsr , (_) , kRWI_W , 0 , 0 ), // #249 - INST(Msub , BaseRRRR , (0b0001101100000000100000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #250 - INST(Mul , BaseRRR , (0b0001101100000000011111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 12 ), // #251 - INST(Mvn , BaseMvnNeg , (0b00101010001000000000001111100000) , kRWI_W , 0 , 0 ), // #252 - INST(Neg , BaseMvnNeg , (0b01001011000000000000001111100000) , kRWI_W , 0 , 1 ), // #253 - INST(Negs , BaseMvnNeg , (0b01101011000000000000001111100000) , kRWI_W , 0 , 2 ), // #254 - INST(Ngc , BaseRR , (0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 7 ), // #255 - INST(Ngcs , BaseRR , (0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 8 ), // #256 - INST(Nop , BaseOp , (0b11010101000000110010000000011111) , 0 , 0 , 13 ), // #257 - INST(Orn , BaseLogical , (0b0101010001, 0b01100100, 1) , kRWI_W , 0 , 6 ), // #258 - INST(Orr , BaseLogical , (0b0101010000, 0b01100100, 0) , kRWI_W , 0 , 7 ), // #259 - INST(Pacda , BaseRR , (0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 9 ), // #260 - INST(Pacdb , BaseRR , (0b11011010110000010000110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 10 ), // #261 - INST(Pacdza , BaseR , (0b11011010110000010010101111100000, kX, kZR, 0) , kRWI_X , 0 , 4 ), // #262 - INST(Pacdzb , BaseR , (0b11011010110000010010111111100000, kX, kZR, 0) , kRWI_X , 0 , 5 ), // #263 - INST(Pacga , BaseRRR , (0b1001101011000000001100, kX, kZR, kX, kZR, kX, kSP, false) , kRWI_W , 0 , 13 ), // #264 - INST(Pssbb , BaseOp , (0b11010101000000110011010010011111) , 0 , 0 , 14 ), // #265 - INST(Rbit , BaseRR , (0b01011010110000000000000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 11 ), // #266 - INST(Ret , BaseBranchReg , (0b11010110010111110000000000000000) , kRWI_R , 0 , 2 ), // #267 - INST(Rev , BaseRev , (_) , kRWI_W , 0 , 0 ), // #268 - INST(Rev16 , BaseRR , (0b01011010110000000000010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 12 ), // #269 - INST(Rev32 , BaseRR , (0b11011010110000000000100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 13 ), // #270 - INST(Rev64 , BaseRR , (0b11011010110000000000110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 14 ), // #271 - INST(Ror , BaseShift , (0b0001101011000000001011, 0b0001001110000000000000, 1) , kRWI_W , 0 , 6 ), // #272 - INST(Rorv , BaseShift , (0b0001101011000000001011, 0b0000000000000000000000, 1) , kRWI_W , 0 , 7 ), // #273 - INST(Sbc , BaseRRR , (0b0101101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 14 ), // #274 - INST(Sbcs , BaseRRR , (0b0111101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 15 ), // #275 - INST(Sbfiz , BaseBfi , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #276 - INST(Sbfm , BaseBfm , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #277 - INST(Sbfx , BaseBfx , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #278 - INST(Sdiv , BaseRRR , (0b0001101011000000000011, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 16 ), // #279 - INST(Setf8 , BaseR , (0b00111010000000000000100000001101, kW, kZR, 5) , 0 , 0 , 6 ), // #280 - INST(Setf16 , BaseR , (0b00111010000000000100100000001101, kW, kZR, 5) , 0 , 0 , 7 ), // #281 - INST(Sev , BaseOp , (0b11010101000000110010000010011111) , 0 , 0 , 15 ), // #282 - INST(Sevl , BaseOp , (0b11010101000000110010000010111111) , 0 , 0 , 16 ), // #283 - INST(Smaddl , BaseRRRR , (0b1001101100100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 2 ), // #284 - INST(Smc , BaseOpImm , (0b11010100000000000000000000000011, 16, 5) , 0 , 0 , 11 ), // #285 - INST(Smnegl , BaseRRR , (0b1001101100100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 17 ), // #286 - INST(Smsubl , BaseRRRR , (0b1001101100100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 3 ), // #287 - INST(Smulh , BaseRRR , (0b1001101101000000011111, kX , kZR, kX , kZR, kX , kZR, true) , kRWI_W , 0 , 18 ), // #288 - INST(Smull , BaseRRR , (0b1001101100100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 19 ), // #289 - INST(Ssbb , BaseOp , (0b11010101000000110011000010011111) , 0 , 0 , 17 ), // #290 - INST(St2g , BaseRM_SImm9 , (0b1101100110100000000010, 0b1101100110100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 13 ), // #291 - INST(Stadd , BaseAtomicSt , (0b1011100000100000000000, kWX, 30) , kRWI_RX , 0 , 0 ), // #292 - INST(Staddl , BaseAtomicSt , (0b1011100001100000000000, kWX, 30) , kRWI_RX , 0 , 1 ), // #293 - INST(Staddb , BaseAtomicSt , (0b0011100000100000000000, kW , 0 ) , kRWI_RX , 0 , 2 ), // #294 - INST(Staddlb , BaseAtomicSt , (0b0011100001100000000000, kW , 0 ) , kRWI_RX , 0 , 3 ), // #295 - INST(Staddh , BaseAtomicSt , (0b0111100000100000000000, kW , 0 ) , kRWI_RX , 0 , 4 ), // #296 - INST(Staddlh , BaseAtomicSt , (0b0111100001100000000000, kW , 0 ) , kRWI_RX , 0 , 5 ), // #297 - INST(Stclr , BaseAtomicSt , (0b1011100000100000000100, kWX, 30) , kRWI_RX , 0 , 6 ), // #298 - INST(Stclrl , BaseAtomicSt , (0b1011100001100000000100, kWX, 30) , kRWI_RX , 0 , 7 ), // #299 - INST(Stclrb , BaseAtomicSt , (0b0011100000100000000100, kW , 0 ) , kRWI_RX , 0 , 8 ), // #300 - INST(Stclrlb , BaseAtomicSt , (0b0011100001100000000100, kW , 0 ) , kRWI_RX , 0 , 9 ), // #301 - INST(Stclrh , BaseAtomicSt , (0b0111100000100000000100, kW , 0 ) , kRWI_RX , 0 , 10 ), // #302 - INST(Stclrlh , BaseAtomicSt , (0b0111100001100000000100, kW , 0 ) , kRWI_RX , 0 , 11 ), // #303 - INST(Steor , BaseAtomicSt , (0b1011100000100000001000, kWX, 30) , kRWI_RX , 0 , 12 ), // #304 - INST(Steorl , BaseAtomicSt , (0b1011100001100000001000, kWX, 30) , kRWI_RX , 0 , 13 ), // #305 - INST(Steorb , BaseAtomicSt , (0b0011100000100000001000, kW , 0 ) , kRWI_RX , 0 , 14 ), // #306 - INST(Steorlb , BaseAtomicSt , (0b0011100001100000001000, kW , 0 ) , kRWI_RX , 0 , 15 ), // #307 - INST(Steorh , BaseAtomicSt , (0b0111100000100000001000, kW , 0 ) , kRWI_RX , 0 , 16 ), // #308 - INST(Steorlh , BaseAtomicSt , (0b0111100001100000001000, kW , 0 ) , kRWI_RX , 0 , 17 ), // #309 - INST(Stg , BaseRM_SImm9 , (0b1101100100100000000010, 0b1101100100100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 14 ), // #310 - INST(Stgm , BaseRM_NoImm , (0b1101100110100000000000, kX , kZR, 0 ) , kRWI_RW , 0 , 13 ), // #311 - INST(Stgp , BaseLdpStp , (0b0110100100, 0b0110100010, kX, 0, 4) , kRWI_RRW , 0 , 3 ), // #312 - INST(Stllr , BaseRM_NoImm , (0b1000100010011111011111, kWX, kZR, 30) , kRWI_RW , 0 , 14 ), // #313 - INST(Stllrb , BaseRM_NoImm , (0b0000100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 15 ), // #314 - INST(Stllrh , BaseRM_NoImm , (0b0100100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 16 ), // #315 - INST(Stlr , BaseRM_NoImm , (0b1000100010011111111111, kWX, kZR, 30) , kRWI_RW , 0 , 17 ), // #316 - INST(Stlrb , BaseRM_NoImm , (0b0000100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 18 ), // #317 - INST(Stlrh , BaseRM_NoImm , (0b0100100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 19 ), // #318 - INST(Stlxp , BaseStxp , (0b1000100000100000100000, kWX, 30) , kRWI_WRRX , 0 , 0 ), // #319 - INST(Stlxr , BaseAtomicOp , (0b1000100000000000111111, kWX, 30, 1) , kRWI_WRX , 0 , 108), // #320 - INST(Stlxrb , BaseAtomicOp , (0b0000100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 109), // #321 - INST(Stlxrh , BaseAtomicOp , (0b0100100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 110), // #322 - INST(Stnp , BaseLdpStp , (0b0010100000, 0 , kWX, 31, 2) , kRWI_RRW , 0 , 4 ), // #323 - INST(Stp , BaseLdpStp , (0b0010100100, 0b0010100010, kWX, 31, 2) , kRWI_RRW , 0 , 5 ), // #324 - INST(Str , BaseLdSt , (0b1011100100, 0b10111000000, 0b10111000001, 0 , kWX, 30, 2, Inst::kIdStur) , kRWI_RW , 0 , 6 ), // #325 - INST(Strb , BaseLdSt , (0b0011100100, 0b00111000000, 0b00111000001, 0 , kW , 30, 0, Inst::kIdSturb) , kRWI_RW , 0 , 7 ), // #326 - INST(Strh , BaseLdSt , (0b0111100100, 0b01111000000, 0b01111000001, 0 , kWX, 30, 1, Inst::kIdSturh) , kRWI_RW , 0 , 8 ), // #327 - INST(Stset , BaseAtomicSt , (0b1011100000100000001100, kWX, 30) , kRWI_RX , 0 , 18 ), // #328 - INST(Stsetl , BaseAtomicSt , (0b1011100001100000001100, kWX, 30) , kRWI_RX , 0 , 19 ), // #329 - INST(Stsetb , BaseAtomicSt , (0b0011100000100000001100, kW , 0 ) , kRWI_RX , 0 , 20 ), // #330 - INST(Stsetlb , BaseAtomicSt , (0b0011100001100000001100, kW , 0 ) , kRWI_RX , 0 , 21 ), // #331 - INST(Stseth , BaseAtomicSt , (0b0111100000100000001100, kW , 0 ) , kRWI_RX , 0 , 22 ), // #332 - INST(Stsetlh , BaseAtomicSt , (0b0111100001100000001100, kW , 0 ) , kRWI_RX , 0 , 23 ), // #333 - INST(Stsmax , BaseAtomicSt , (0b1011100000100000010000, kWX, 30) , kRWI_RX , 0 , 24 ), // #334 - INST(Stsmaxl , BaseAtomicSt , (0b1011100001100000010000, kWX, 30) , kRWI_RX , 0 , 25 ), // #335 - INST(Stsmaxb , BaseAtomicSt , (0b0011100000100000010000, kW , 0 ) , kRWI_RX , 0 , 26 ), // #336 - INST(Stsmaxlb , BaseAtomicSt , (0b0011100001100000010000, kW , 0 ) , kRWI_RX , 0 , 27 ), // #337 - INST(Stsmaxh , BaseAtomicSt , (0b0111100000100000010000, kW , 0 ) , kRWI_RX , 0 , 28 ), // #338 - INST(Stsmaxlh , BaseAtomicSt , (0b0111100001100000010000, kW , 0 ) , kRWI_RX , 0 , 29 ), // #339 - INST(Stsmin , BaseAtomicSt , (0b1011100000100000010100, kWX, 30) , kRWI_RX , 0 , 30 ), // #340 - INST(Stsminl , BaseAtomicSt , (0b1011100001100000010100, kWX, 30) , kRWI_RX , 0 , 31 ), // #341 - INST(Stsminb , BaseAtomicSt , (0b0011100000100000010100, kW , 0 ) , kRWI_RX , 0 , 32 ), // #342 - INST(Stsminlb , BaseAtomicSt , (0b0011100001100000010100, kW , 0 ) , kRWI_RX , 0 , 33 ), // #343 - INST(Stsminh , BaseAtomicSt , (0b0111100000100000010100, kW , 0 ) , kRWI_RX , 0 , 34 ), // #344 - INST(Stsminlh , BaseAtomicSt , (0b0111100001100000010100, kW , 0 ) , kRWI_RX , 0 , 35 ), // #345 - INST(Sttr , BaseRM_SImm9 , (0b1011100000000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 15 ), // #346 - INST(Sttrb , BaseRM_SImm9 , (0b0011100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 16 ), // #347 - INST(Sttrh , BaseRM_SImm9 , (0b0111100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 17 ), // #348 - INST(Stumax , BaseAtomicSt , (0b1011100000100000011000, kWX, 30) , kRWI_RX , 0 , 36 ), // #349 - INST(Stumaxl , BaseAtomicSt , (0b1011100001100000011000, kWX, 30) , kRWI_RX , 0 , 37 ), // #350 - INST(Stumaxb , BaseAtomicSt , (0b0011100000100000011000, kW , 0 ) , kRWI_RX , 0 , 38 ), // #351 - INST(Stumaxlb , BaseAtomicSt , (0b0011100001100000011000, kW , 0 ) , kRWI_RX , 0 , 39 ), // #352 - INST(Stumaxh , BaseAtomicSt , (0b0111100000100000011000, kW , 0 ) , kRWI_RX , 0 , 40 ), // #353 - INST(Stumaxlh , BaseAtomicSt , (0b0111100001100000011000, kW , 0 ) , kRWI_RX , 0 , 41 ), // #354 - INST(Stumin , BaseAtomicSt , (0b1011100000100000011100, kWX, 30) , kRWI_RX , 0 , 42 ), // #355 - INST(Stuminl , BaseAtomicSt , (0b1011100001100000011100, kWX, 30) , kRWI_RX , 0 , 43 ), // #356 - INST(Stuminb , BaseAtomicSt , (0b0011100000100000011100, kW , 0 ) , kRWI_RX , 0 , 44 ), // #357 - INST(Stuminlb , BaseAtomicSt , (0b0011100001100000011100, kW , 0 ) , kRWI_RX , 0 , 45 ), // #358 - INST(Stuminh , BaseAtomicSt , (0b0111100000100000011100, kW , 0 ) , kRWI_RX , 0 , 46 ), // #359 - INST(Stuminlh , BaseAtomicSt , (0b0111100001100000011100, kW , 0 ) , kRWI_RX , 0 , 47 ), // #360 - INST(Stur , BaseRM_SImm9 , (0b1011100000000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 18 ), // #361 - INST(Sturb , BaseRM_SImm9 , (0b0011100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 19 ), // #362 - INST(Sturh , BaseRM_SImm9 , (0b0111100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 20 ), // #363 - INST(Stxp , BaseStxp , (0b1000100000100000000000, kWX, 30) , kRWI_WRRW , 0 , 1 ), // #364 - INST(Stxr , BaseStx , (0b1000100000000000011111, kWX, 30) , kRWI_WRW , 0 , 0 ), // #365 - INST(Stxrb , BaseStx , (0b0000100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 1 ), // #366 - INST(Stxrh , BaseStx , (0b0100100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 2 ), // #367 - INST(Stz2g , BaseRM_SImm9 , (0b1101100111100000000010, 0b1101100111100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 21 ), // #368 - INST(Stzg , BaseRM_SImm9 , (0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 22 ), // #369 - INST(Stzgm , BaseRM_NoImm , (0b1101100100100000000000, kX , kZR, 0) , kRWI_RW , 0 , 20 ), // #370 - INST(Sub , BaseAddSub , (0b1001011000, 0b1001011001, 0b1010001) , kRWI_X , 0 , 2 ), // #371 - INST(Subg , BaseRRII , (0b1101000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 1 ), // #372 - INST(Subp , BaseRRR , (0b1001101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 20 ), // #373 - INST(Subps , BaseRRR , (0b1011101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 21 ), // #374 - INST(Subs , BaseAddSub , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_X , 0 , 3 ), // #375 - INST(Svc , BaseOpImm , (0b11010100000000000000000000000001, 16, 5) , 0 , 0 , 12 ), // #376 - INST(Swp , BaseAtomicOp , (0b1011100000100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 111), // #377 - INST(Swpa , BaseAtomicOp , (0b1011100010100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 112), // #378 - INST(Swpab , BaseAtomicOp , (0b0011100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 113), // #379 - INST(Swpah , BaseAtomicOp , (0b0111100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 114), // #380 - INST(Swpal , BaseAtomicOp , (0b1011100011100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 115), // #381 - INST(Swpalb , BaseAtomicOp , (0b0011100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 116), // #382 - INST(Swpalh , BaseAtomicOp , (0b0111100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 117), // #383 - INST(Swpb , BaseAtomicOp , (0b0011100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 118), // #384 - INST(Swph , BaseAtomicOp , (0b0111100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 119), // #385 - INST(Swpl , BaseAtomicOp , (0b1011100001100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 120), // #386 - INST(Swplb , BaseAtomicOp , (0b0011100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 121), // #387 - INST(Swplh , BaseAtomicOp , (0b0111100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 122), // #388 - INST(Sxtb , BaseExtend , (0b0001001100000000000111, kWX, 0) , kRWI_W , 0 , 0 ), // #389 - INST(Sxth , BaseExtend , (0b0001001100000000001111, kWX, 0) , kRWI_W , 0 , 1 ), // #390 - INST(Sxtw , BaseExtend , (0b1001001101000000011111, kX , 0) , kRWI_W , 0 , 2 ), // #391 - INST(Sys , BaseSys , (_) , kRWI_W , 0 , 0 ), // #392 - INST(Tlbi , BaseAtDcIcTlbi , (0b00011110000000, 0b00010000000000, false) , kRWI_RX , 0 , 3 ), // #393 - INST(Tst , BaseTst , (0b1101010000, 0b111001000) , kRWI_R , 0 , 0 ), // #394 - INST(Tbnz , BaseBranchTst , (0b00110111000000000000000000000000) , kRWI_R , 0 , 0 ), // #395 - INST(Tbz , BaseBranchTst , (0b00110110000000000000000000000000) , kRWI_R , 0 , 1 ), // #396 - INST(Ubfiz , BaseBfi , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #397 - INST(Ubfm , BaseBfm , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #398 - INST(Ubfx , BaseBfx , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #399 - INST(Udf , BaseOpImm , (0b00000000000000000000000000000000, 16, 0) , 0 , 0 , 13 ), // #400 - INST(Udiv , BaseRRR , (0b0001101011000000000010, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 22 ), // #401 - INST(Umaddl , BaseRRRR , (0b1001101110100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 4 ), // #402 - INST(Umnegl , BaseRRR , (0b1001101110100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 23 ), // #403 - INST(Umull , BaseRRR , (0b1001101110100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 24 ), // #404 - INST(Umulh , BaseRRR , (0b1001101111000000011111, kX , kZR, kX , kZR, kX , kZR, false) , kRWI_W , 0 , 25 ), // #405 - INST(Umsubl , BaseRRRR , (0b1001101110100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 5 ), // #406 - INST(Uxtb , BaseExtend , (0b0101001100000000000111, kW, 1) , kRWI_W , 0 , 3 ), // #407 - INST(Uxth , BaseExtend , (0b0101001100000000001111, kW, 1) , kRWI_W , 0 , 4 ), // #408 - INST(Wfe , BaseOp , (0b11010101000000110010000001011111) , 0 , 0 , 18 ), // #409 - INST(Wfi , BaseOp , (0b11010101000000110010000001111111) , 0 , 0 , 19 ), // #410 - INST(Xaflag , BaseOp , (0b11010101000000000100000000111111) , 0 , 0 , 20 ), // #411 - INST(Xpacd , BaseR , (0b11011010110000010100011111100000, kX, kZR, 0) , kRWI_X , 0 , 8 ), // #412 - INST(Xpaci , BaseR , (0b11011010110000010100001111100000, kX, kZR, 0) , kRWI_X , 0 , 9 ), // #413 - INST(Xpaclri , BaseOp , (0b11010101000000110010000011111111) , kRWI_X , 0 , 21 ), // #414 - INST(Yield , BaseOp , (0b11010101000000110010000000111111) , 0 , 0 , 22 ), // #415 - INST(Abs_v , ISimdVV , (0b0000111000100000101110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #416 - INST(Add_v , ISimdVVV , (0b0000111000100000100001, kVO_V_Any) , kRWI_W , 0 , 0 ), // #417 - INST(Addhn_v , ISimdVVV , (0b0000111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 1 ), // #418 - INST(Addhn2_v , ISimdVVV , (0b0100111000100000010000, kVO_V_B16H8S4) , kRWI_W , F(Narrow) , 2 ), // #419 - INST(Addp_v , ISimdPair , (0b0101111000110001101110, 0b0000111000100000101111, kVO_V_Any) , kRWI_W , F(Pair) , 0 ), // #420 - INST(Addv_v , ISimdSV , (0b0000111000110001101110, kVO_V_BH_4S) , kRWI_W , 0 , 0 ), // #421 - INST(Aesd_v , ISimdVVx , (0b0100111000101000010110, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 0 ), // #422 - INST(Aese_v , ISimdVVx , (0b0100111000101000010010, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 1 ), // #423 - INST(Aesimc_v , ISimdVVx , (0b0100111000101000011110, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 2 ), // #424 - INST(Aesmc_v , ISimdVVx , (0b0100111000101000011010, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 3 ), // #425 - INST(And_v , ISimdVVV , (0b0000111000100000000111, kVO_V_B) , kRWI_W , 0 , 3 ), // #426 - INST(Bcax_v , ISimdVVVV , (0b1100111000100000000000, kVO_V_B16) , kRWI_W , 0 , 0 ), // #427 - INST(Bfcvt_v , ISimdVVx , (0b0001111001100011010000, kOp_H, kOp_S) , kRWI_W , 0 , 4 ), // #428 - INST(Bfcvtn_v , ISimdVVx , (0b0000111010100001011010, kOp_V4H, kOp_V4S) , kRWI_W , F(Narrow) , 5 ), // #429 - INST(Bfcvtn2_v , ISimdVVx , (0b0100111010100001011010, kOp_V8H, kOp_V4S) , kRWI_W , F(Narrow) , 6 ), // #430 - INST(Bfdot_v , SimdDot , (0b0010111001000000111111, 0b0000111101000000111100, kET_S, kET_H, kET_2H) , kRWI_X , 0 , 0 ), // #431 - INST(Bfmlalb_v , SimdFmlal , (0b0010111011000000111111, 0b0000111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 0 ), // #432 - INST(Bfmlalt_v , SimdFmlal , (0b0110111011000000111111, 0b0100111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 1 ), // #433 - INST(Bfmmla_v , ISimdVVVx , (0b0110111001000000111011, kOp_V4S, kOp_V8H, kOp_V8H) , kRWI_X , F(Long) , 0 ), // #434 - INST(Bic_v , SimdBicOrr , (0b0000111001100000000111, 0b0010111100000000000001) , kRWI_W , 0 , 0 ), // #435 - INST(Bif_v , ISimdVVV , (0b0010111011100000000111, kVO_V_B) , kRWI_X , 0 , 4 ), // #436 - INST(Bit_v , ISimdVVV , (0b0010111010100000000111, kVO_V_B) , kRWI_X , 0 , 5 ), // #437 - INST(Bsl_v , ISimdVVV , (0b0010111001100000000111, kVO_V_B) , kRWI_X , 0 , 6 ), // #438 - INST(Cls_v , ISimdVV , (0b0000111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 1 ), // #439 - INST(Clz_v , ISimdVV , (0b0010111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 2 ), // #440 - INST(Cmeq_v , SimdCmp , (0b0010111000100000100011, 0b0000111000100000100110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #441 - INST(Cmge_v , SimdCmp , (0b0000111000100000001111, 0b0010111000100000100010, kVO_V_Any) , kRWI_W , 0 , 1 ), // #442 - INST(Cmgt_v , SimdCmp , (0b0000111000100000001101, 0b0000111000100000100010, kVO_V_Any) , kRWI_W , 0 , 2 ), // #443 - INST(Cmhi_v , SimdCmp , (0b0010111000100000001101, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 3 ), // #444 - INST(Cmhs_v , SimdCmp , (0b0010111000100000001111, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 4 ), // #445 - INST(Cmle_v , SimdCmp , (0b0000000000000000000000, 0b0010111000100000100110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #446 - INST(Cmlt_v , SimdCmp , (0b0000000000000000000000, 0b0000111000100000101010, kVO_V_Any) , kRWI_W , 0 , 6 ), // #447 - INST(Cmtst_v , ISimdVVV , (0b0000111000100000100011, kVO_V_Any) , kRWI_W , 0 , 7 ), // #448 - INST(Cnt_v , ISimdVV , (0b0000111000100000010110, kVO_V_B) , kRWI_W , 0 , 3 ), // #449 - INST(Dup_v , SimdDup , (_) , kRWI_W , 0 , 0 ), // #450 - INST(Eor_v , ISimdVVV , (0b0010111000100000000111, kVO_V_B) , kRWI_W , 0 , 8 ), // #451 - INST(Eor3_v , ISimdVVVV , (0b1100111000000000000000, kVO_V_B16) , kRWI_W , 0 , 1 ), // #452 - INST(Ext_v , ISimdVVVI , (0b0010111000000000000000, kVO_V_B, 4, 11, 1) , kRWI_W , 0 , 0 ), // #453 - INST(Fabd_v , FSimdVVV , (0b0111111010100000110101, kHF_C, 0b0010111010100000110101, kHF_C) , kRWI_W , 0 , 0 ), // #454 - INST(Fabs_v , FSimdVV , (0b0001111000100000110000, kHF_A, 0b0000111010100000111110, kHF_B) , kRWI_W , 0 , 0 ), // #455 - INST(Facge_v , FSimdVVV , (0b0111111000100000111011, kHF_C, 0b0010111000100000111011, kHF_C) , kRWI_W , 0 , 1 ), // #456 - INST(Facgt_v , FSimdVVV , (0b0111111010100000111011, kHF_C, 0b0010111010100000111011, kHF_C) , kRWI_W , 0 , 2 ), // #457 - INST(Fadd_v , FSimdVVV , (0b0001111000100000001010, kHF_A, 0b0000111000100000110101, kHF_C) , kRWI_W , 0 , 3 ), // #458 - INST(Faddp_v , FSimdPair , (0b0111111000110000110110, 0b0010111000100000110101) , kRWI_W , 0 , 0 ), // #459 - INST(Fcadd_v , SimdFcadd , (0b0010111000000000111001) , kRWI_W , 0 , 0 ), // #460 - INST(Fccmp_v , SimdFccmpFccmpe , (0b00011110001000000000010000000000) , kRWI_R , 0 , 0 ), // #461 - INST(Fccmpe_v , SimdFccmpFccmpe , (0b00011110001000000000010000010000) , kRWI_R , 0 , 1 ), // #462 - INST(Fcmeq_v , SimdFcm , (0b0000111000100000111001, kHF_C, 0b0000111010100000110110) , kRWI_W , 0 , 0 ), // #463 - INST(Fcmge_v , SimdFcm , (0b0010111000100000111001, kHF_C, 0b0010111010100000110010) , kRWI_W , 0 , 1 ), // #464 - INST(Fcmgt_v , SimdFcm , (0b0010111010100000111001, kHF_C, 0b0000111010100000110010) , kRWI_W , 0 , 2 ), // #465 - INST(Fcmla_v , SimdFcmla , (0b0010111000000000110001, 0b0010111100000000000100) , kRWI_X , 0 , 0 ), // #466 - INST(Fcmle_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0010111010100000110110) , kRWI_W , 0 , 3 ), // #467 - INST(Fcmlt_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0000111010100000111010) , kRWI_W , 0 , 4 ), // #468 - INST(Fcmp_v , SimdFcmpFcmpe , (0b00011110001000000010000000000000) , kRWI_R , 0 , 0 ), // #469 - INST(Fcmpe_v , SimdFcmpFcmpe , (0b00011110001000000010000000010000) , kRWI_R , 0 , 1 ), // #470 - INST(Fcsel_v , SimdFcsel , (_) , kRWI_W , 0 , 0 ), // #471 - INST(Fcvt_v , SimdFcvt , (_) , kRWI_W , 0 , 0 ), // #472 - INST(Fcvtas_v , SimdFcvtSV , (0b0000111000100001110010, 0b0000000000000000000000, 0b0001111000100100000000, 1) , kRWI_W , 0 , 0 ), // #473 - INST(Fcvtau_v , SimdFcvtSV , (0b0010111000100001110010, 0b0000000000000000000000, 0b0001111000100101000000, 1) , kRWI_W , 0 , 1 ), // #474 - INST(Fcvtl_v , SimdFcvtLN , (0b0000111000100001011110, 0, 0) , kRWI_W , F(Long) , 0 ), // #475 - INST(Fcvtl2_v , SimdFcvtLN , (0b0100111000100001011110, 0, 0) , kRWI_W , F(Long) , 1 ), // #476 - INST(Fcvtms_v , SimdFcvtSV , (0b0000111000100001101110, 0b0000000000000000000000, 0b0001111000110000000000, 1) , kRWI_W , 0 , 2 ), // #477 - INST(Fcvtmu_v , SimdFcvtSV , (0b0010111000100001101110, 0b0000000000000000000000, 0b0001111000110001000000, 1) , kRWI_W , 0 , 3 ), // #478 - INST(Fcvtn_v , SimdFcvtLN , (0b0000111000100001011010, 0, 0) , kRWI_W , F(Narrow) , 2 ), // #479 - INST(Fcvtn2_v , SimdFcvtLN , (0b0100111000100001011010, 0, 0) , kRWI_X , F(Narrow) , 3 ), // #480 - INST(Fcvtns_v , SimdFcvtSV , (0b0000111000100001101010, 0b0000000000000000000000, 0b0001111000100000000000, 1) , kRWI_W , 0 , 4 ), // #481 - INST(Fcvtnu_v , SimdFcvtSV , (0b0010111000100001101010, 0b0000000000000000000000, 0b0001111000100001000000, 1) , kRWI_W , 0 , 5 ), // #482 - INST(Fcvtps_v , SimdFcvtSV , (0b0000111010100001101010, 0b0000000000000000000000, 0b0001111000101000000000, 1) , kRWI_W , 0 , 6 ), // #483 - INST(Fcvtpu_v , SimdFcvtSV , (0b0010111010100001101010, 0b0000000000000000000000, 0b0001111000101001000000, 1) , kRWI_W , 0 , 7 ), // #484 - INST(Fcvtxn_v , SimdFcvtLN , (0b0010111000100001011010, 1, 1) , kRWI_W , F(Narrow) , 4 ), // #485 - INST(Fcvtxn2_v , SimdFcvtLN , (0b0110111000100001011010, 1, 0) , kRWI_X , F(Narrow) , 5 ), // #486 - INST(Fcvtzs_v , SimdFcvtSV , (0b0000111010100001101110, 0b0000111100000000111111, 0b0001111000111000000000, 1) , kRWI_W , 0 , 8 ), // #487 - INST(Fcvtzu_v , SimdFcvtSV , (0b0010111010100001101110, 0b0010111100000000111111, 0b0001111000111001000000, 1) , kRWI_W , 0 , 9 ), // #488 - INST(Fdiv_v , FSimdVVV , (0b0001111000100000000110, kHF_A, 0b0010111000100000111111, kHF_C) , kRWI_W , 0 , 4 ), // #489 - INST(Fjcvtzs_v , ISimdVVx , (0b0001111001111110000000, kOp_GpW, kOp_D) , kRWI_W , 0 , 7 ), // #490 - INST(Fmadd_v , FSimdVVVV , (0b0001111100000000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 0 ), // #491 - INST(Fmax_v , FSimdVVV , (0b0001111000100000010010, kHF_A, 0b0000111000100000111101, kHF_C) , kRWI_W , 0 , 5 ), // #492 - INST(Fmaxnm_v , FSimdVVV , (0b0001111000100000011010, kHF_A, 0b0000111000100000110001, kHF_C) , kRWI_W , 0 , 6 ), // #493 - INST(Fmaxnmp_v , FSimdPair , (0b0111111000110000110010, 0b0010111000100000110001) , kRWI_W , 0 , 1 ), // #494 - INST(Fmaxnmv_v , FSimdSV , (0b0010111000110000110010) , kRWI_W , 0 , 0 ), // #495 - INST(Fmaxp_v , FSimdPair , (0b0111111000110000111110, 0b0010111000100000111101) , kRWI_W , 0 , 2 ), // #496 - INST(Fmaxv_v , FSimdSV , (0b0010111000110000111110) , kRWI_W , 0 , 1 ), // #497 - INST(Fmin_v , FSimdVVV , (0b0001111000100000010110, kHF_A, 0b0000111010100000111101, kHF_C) , kRWI_W , 0 , 7 ), // #498 - INST(Fminnm_v , FSimdVVV , (0b0001111000100000011110, kHF_A, 0b0000111010100000110001, kHF_C) , kRWI_W , 0 , 8 ), // #499 - INST(Fminnmp_v , FSimdPair , (0b0111111010110000110010, 0b0010111010100000110001) , kRWI_W , 0 , 3 ), // #500 - INST(Fminnmv_v , FSimdSV , (0b0010111010110000110010) , kRWI_W , 0 , 2 ), // #501 - INST(Fminp_v , FSimdPair , (0b0111111010110000111110, 0b0010111010100000111101) , kRWI_W , 0 , 4 ), // #502 - INST(Fminv_v , FSimdSV , (0b0010111010110000111110) , kRWI_W , 0 , 3 ), // #503 - INST(Fmla_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111000100000110011, 0b0000111110000000000100) , kRWI_X , F(VH0_15) , 0 ), // #504 - INST(Fmlal_v , SimdFmlal , (0b0000111000100000111011, 0b0000111110000000000000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 2 ), // #505 - INST(Fmlal2_v , SimdFmlal , (0b0010111000100000110011, 0b0010111110000000100000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 3 ), // #506 - INST(Fmls_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111010100000110011, 0b0000111110000000010100) , kRWI_X , F(VH0_15) , 1 ), // #507 - INST(Fmlsl_v , SimdFmlal , (0b0000111010100000111011, 0b0000111110000000010000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 4 ), // #508 - INST(Fmlsl2_v , SimdFmlal , (0b0010111010100000110011, 0b0010111110000000110000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 5 ), // #509 - INST(Fmov_v , SimdFmov , (_) , kRWI_W , 0 , 0 ), // #510 - INST(Fmsub_v , FSimdVVVV , (0b0001111100000000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 1 ), // #511 - INST(Fmul_v , FSimdVVVe , (0b0001111000100000000010, kHF_A, 0b0010111000100000110111, 0b0000111110000000100100) , kRWI_W , F(VH0_15) , 2 ), // #512 - INST(Fmulx_v , FSimdVVVe , (0b0101111000100000110111, kHF_C, 0b0000111000100000110111, 0b0010111110000000100100) , kRWI_W , F(VH0_15) , 3 ), // #513 - INST(Fneg_v , FSimdVV , (0b0001111000100001010000, kHF_A, 0b0010111010100000111110, kHF_B) , kRWI_W , 0 , 1 ), // #514 - INST(Fnmadd_v , FSimdVVVV , (0b0001111100100000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 2 ), // #515 - INST(Fnmsub_v , FSimdVVVV , (0b0001111100100000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #516 - INST(Fnmul_v , FSimdVVV , (0b0001111000100000100010, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 9 ), // #517 - INST(Frecpe_v , FSimdVV , (0b0101111010100001110110, kHF_B, 0b0000111010100001110110, kHF_B) , kRWI_W , 0 , 2 ), // #518 - INST(Frecps_v , FSimdVVV , (0b0101111000100000111111, kHF_C, 0b0000111000100000111111, kHF_C) , kRWI_W , 0 , 10 ), // #519 - INST(Frecpx_v , FSimdVV , (0b0101111010100001111110, kHF_B, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #520 - INST(Frint32x_v , FSimdVV , (0b0001111000101000110000, kHF_N, 0b0010111000100001111010, kHF_N) , kRWI_W , 0 , 4 ), // #521 - INST(Frint32z_v , FSimdVV , (0b0001111000101000010000, kHF_N, 0b0000111000100001111010, kHF_N) , kRWI_W , 0 , 5 ), // #522 - INST(Frint64x_v , FSimdVV , (0b0001111000101001110000, kHF_N, 0b0010111000100001111110, kHF_N) , kRWI_W , 0 , 6 ), // #523 - INST(Frint64z_v , FSimdVV , (0b0001111000101001010000, kHF_N, 0b0000111000100001111110, kHF_N) , kRWI_W , 0 , 7 ), // #524 - INST(Frinta_v , FSimdVV , (0b0001111000100110010000, kHF_A, 0b0010111000100001100010, kHF_B) , kRWI_W , 0 , 8 ), // #525 - INST(Frinti_v , FSimdVV , (0b0001111000100111110000, kHF_A, 0b0010111010100001100110, kHF_B) , kRWI_W , 0 , 9 ), // #526 - INST(Frintm_v , FSimdVV , (0b0001111000100101010000, kHF_A, 0b0000111000100001100110, kHF_B) , kRWI_W , 0 , 10 ), // #527 - INST(Frintn_v , FSimdVV , (0b0001111000100100010000, kHF_A, 0b0000111000100001100010, kHF_B) , kRWI_W , 0 , 11 ), // #528 - INST(Frintp_v , FSimdVV , (0b0001111000100100110000, kHF_A, 0b0000111010100001100010, kHF_B) , kRWI_W , 0 , 12 ), // #529 - INST(Frintx_v , FSimdVV , (0b0001111000100111010000, kHF_A, 0b0010111000100001100110, kHF_B) , kRWI_W , 0 , 13 ), // #530 - INST(Frintz_v , FSimdVV , (0b0001111000100101110000, kHF_A, 0b0000111010100001100110, kHF_B) , kRWI_W , 0 , 14 ), // #531 - INST(Frsqrte_v , FSimdVV , (0b0111111010100001110110, kHF_B, 0b0010111010100001110110, kHF_B) , kRWI_W , 0 , 15 ), // #532 - INST(Frsqrts_v , FSimdVVV , (0b0101111010100000111111, kHF_C, 0b0000111010100000111111, kHF_C) , kRWI_W , 0 , 11 ), // #533 - INST(Fsqrt_v , FSimdVV , (0b0001111000100001110000, kHF_A, 0b0010111010100001111110, kHF_B) , kRWI_W , 0 , 16 ), // #534 - INST(Fsub_v , FSimdVVV , (0b0001111000100000001110, kHF_A, 0b0000111010100000110101, kHF_C) , kRWI_W , 0 , 12 ), // #535 - INST(Ins_v , SimdIns , (_) , kRWI_X , 0 , 0 ), // #536 - INST(Ld1_v , SimdLdNStN , (0b0000110101000000000000, 0b0000110001000000001000, 1, 0) , kRWI_LDn , F(Consecutive) , 0 ), // #537 - INST(Ld1r_v , SimdLdNStN , (0b0000110101000000110000, 0b0000000000000000000000, 1, 1) , kRWI_LDn , F(Consecutive) , 1 ), // #538 - INST(Ld2_v , SimdLdNStN , (0b0000110101100000000000, 0b0000110001000000100000, 2, 0) , kRWI_LDn , F(Consecutive) , 2 ), // #539 - INST(Ld2r_v , SimdLdNStN , (0b0000110101100000110000, 0b0000000000000000000000, 2, 1) , kRWI_LDn , F(Consecutive) , 3 ), // #540 - INST(Ld3_v , SimdLdNStN , (0b0000110101000000001000, 0b0000110001000000010000, 3, 0) , kRWI_LDn , F(Consecutive) , 4 ), // #541 - INST(Ld3r_v , SimdLdNStN , (0b0000110101000000111000, 0b0000000000000000000000, 3, 1) , kRWI_LDn , F(Consecutive) , 5 ), // #542 - INST(Ld4_v , SimdLdNStN , (0b0000110101100000001000, 0b0000110001000000000000, 4, 0) , kRWI_LDn , F(Consecutive) , 6 ), // #543 - INST(Ld4r_v , SimdLdNStN , (0b0000110101100000111000, 0b0000000000000000000000, 4, 1) , kRWI_LDn , F(Consecutive) , 7 ), // #544 - INST(Ldnp_v , SimdLdpStp , (0b0010110001, 0b0000000000) , kRWI_WW , 0 , 0 ), // #545 - INST(Ldp_v , SimdLdpStp , (0b0010110101, 0b0010110011) , kRWI_WW , 0 , 1 ), // #546 - INST(Ldr_v , SimdLdSt , (0b0011110101, 0b00111100010, 0b00111100011, 0b00011100, Inst::kIdLdur_v) , kRWI_W , 0 , 0 ), // #547 - INST(Ldur_v , SimdLdurStur , (0b0011110001000000000000) , kRWI_W , 0 , 0 ), // #548 - INST(Mla_v , ISimdVVVe , (0b0000111000100000100101, kVO_V_BHS, 0b0010111100000000000000, kVO_V_HS) , kRWI_X , F(VH0_15) , 0 ), // #549 - INST(Mls_v , ISimdVVVe , (0b0010111000100000100101, kVO_V_BHS, 0b0010111100000000010000, kVO_V_HS) , kRWI_X , F(VH0_15) , 1 ), // #550 - INST(Mov_v , SimdMov , (_) , kRWI_W , 0 , 0 ), // #551 - INST(Movi_v , SimdMoviMvni , (0b0000111100000000000001, 0) , kRWI_W , 0 , 0 ), // #552 - INST(Mul_v , ISimdVVVe , (0b0000111000100000100111, kVO_V_BHS, 0b0000111100000000100000, kVO_V_HS) , kRWI_W , F(VH0_15) , 2 ), // #553 - INST(Mvn_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 4 ), // #554 - INST(Mvni_v , SimdMoviMvni , (0b0000111100000000000001, 1) , kRWI_W , 0 , 1 ), // #555 - INST(Neg_v , ISimdVV , (0b0010111000100000101110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #556 - INST(Not_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 6 ), // #557 - INST(Orn_v , ISimdVVV , (0b0000111011100000000111, kVO_V_B) , kRWI_W , 0 , 9 ), // #558 - INST(Orr_v , SimdBicOrr , (0b0000111010100000000111, 0b0000111100000000000001) , kRWI_W , 0 , 1 ), // #559 - INST(Pmul_v , ISimdVVV , (0b0010111000100000100111, kVO_V_B) , kRWI_W , 0 , 10 ), // #560 - INST(Pmull_v , ISimdVVV , (0b0000111000100000111000, kVO_V_B8D1) , kRWI_W , F(Long) , 11 ), // #561 - INST(Pmull2_v , ISimdVVV , (0b0100111000100000111000, kVO_V_B16D2) , kRWI_W , F(Long) , 12 ), // #562 - INST(Raddhn_v , ISimdVVV , (0b0010111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #563 - INST(Raddhn2_v , ISimdVVV , (0b0110111000100000010000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #564 - INST(Rax1_v , ISimdVVV , (0b1100111001100000100011, kVO_V_D2) , kRWI_W , 0 , 15 ), // #565 - INST(Rbit_v , ISimdVV , (0b0010111001100000010110, kVO_V_B) , kRWI_W , 0 , 7 ), // #566 - INST(Rev16_v , ISimdVV , (0b0000111000100000000110, kVO_V_B) , kRWI_W , 0 , 8 ), // #567 - INST(Rev32_v , ISimdVV , (0b0010111000100000000010, kVO_V_BH) , kRWI_W , 0 , 9 ), // #568 - INST(Rev64_v , ISimdVV , (0b0000111000100000000010, kVO_V_BHS) , kRWI_W , 0 , 10 ), // #569 - INST(Rshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100011, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 0 ), // #570 - INST(Rshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 1 ), // #571 - INST(Rsubhn_v , ISimdVVV , (0b0010111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 16 ), // #572 - INST(Rsubhn2_v , ISimdVVV , (0b0110111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 17 ), // #573 - INST(Saba_v , ISimdVVV , (0b0000111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 18 ), // #574 - INST(Sabal_v , ISimdVVV , (0b0000111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 19 ), // #575 - INST(Sabal2_v , ISimdVVV , (0b0100111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 20 ), // #576 - INST(Sabd_v , ISimdVVV , (0b0000111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 21 ), // #577 - INST(Sabdl_v , ISimdVVV , (0b0000111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #578 - INST(Sabdl2_v , ISimdVVV , (0b0100111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #579 - INST(Sadalp_v , ISimdVV , (0b0000111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 11 ), // #580 - INST(Saddl_v , ISimdVVV , (0b0000111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 24 ), // #581 - INST(Saddl2_v , ISimdVVV , (0b0100111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 25 ), // #582 - INST(Saddlp_v , ISimdVV , (0b0000111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 12 ), // #583 - INST(Saddlv_v , ISimdSV , (0b0000111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 1 ), // #584 - INST(Saddw_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 0 ), // #585 - INST(Saddw2_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 1 ), // #586 - INST(Scvtf_v , SimdFcvtSV , (0b0000111000100001110110, 0b0000111100000000111001, 0b0001111000100010000000, 0) , kRWI_W , 0 , 10 ), // #587 - INST(Sdot_v , SimdDot , (0b0000111010000000100101, 0b0000111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 1 ), // #588 - INST(Sha1c_v , ISimdVVVx , (0b0101111000000000000000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 1 ), // #589 - INST(Sha1h_v , ISimdVVx , (0b0101111000101000000010, kOp_S, kOp_S) , kRWI_W , 0 , 8 ), // #590 - INST(Sha1m_v , ISimdVVVx , (0b0101111000000000001000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 2 ), // #591 - INST(Sha1p_v , ISimdVVVx , (0b0101111000000000000100, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 3 ), // #592 - INST(Sha1su0_v , ISimdVVVx , (0b0101111000000000001100, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 4 ), // #593 - INST(Sha1su1_v , ISimdVVx , (0b0101111000101000000110, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 9 ), // #594 - INST(Sha256h_v , ISimdVVVx , (0b0101111000000000010000, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 5 ), // #595 - INST(Sha256h2_v , ISimdVVVx , (0b0101111000000000010100, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 6 ), // #596 - INST(Sha256su0_v , ISimdVVx , (0b0101111000101000001010, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 10 ), // #597 - INST(Sha256su1_v , ISimdVVVx , (0b0101111000000000011000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 7 ), // #598 - INST(Sha512h_v , ISimdVVVx , (0b1100111001100000100000, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 8 ), // #599 - INST(Sha512h2_v , ISimdVVVx , (0b1100111001100000100001, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 9 ), // #600 - INST(Sha512su0_v , ISimdVVx , (0b1100111011000000100000, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 11 ), // #601 - INST(Sha512su1_v , ISimdVVVx , (0b1100111001100000100010, kOp_V2D, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 10 ), // #602 - INST(Shadd_v , ISimdVVV , (0b0000111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 26 ), // #603 - INST(Shl_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000010101, 0, kVO_V_Any) , kRWI_W , 0 , 2 ), // #604 - INST(Shll_v , SimdShiftES , (0b0010111000100001001110, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #605 - INST(Shll2_v , SimdShiftES , (0b0110111000100001001110, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #606 - INST(Shrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100001, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 3 ), // #607 - INST(Shrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 4 ), // #608 - INST(Shsub_v , ISimdVVV , (0b0000111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 27 ), // #609 - INST(Sli_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010101, 0, kVO_V_Any) , kRWI_X , 0 , 5 ), // #610 - INST(Sm3partw1_v , ISimdVVVx , (0b1100111001100000110000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 11 ), // #611 - INST(Sm3partw2_v , ISimdVVVx , (0b1100111001100000110001, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #612 - INST(Sm3ss1_v , ISimdVVVVx , (0b1100111001000000000000, kOp_V4S, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_W , 0 , 0 ), // #613 - INST(Sm3tt1a_v , SimdSm3tt , (0b1100111001000000100000) , kRWI_X , 0 , 0 ), // #614 - INST(Sm3tt1b_v , SimdSm3tt , (0b1100111001000000100001) , kRWI_X , 0 , 1 ), // #615 - INST(Sm3tt2a_v , SimdSm3tt , (0b1100111001000000100010) , kRWI_X , 0 , 2 ), // #616 - INST(Sm3tt2b_v , SimdSm3tt , (0b1100111001000000100011) , kRWI_X , 0 , 3 ), // #617 - INST(Sm4e_v , ISimdVVx , (0b1100111011000000100001, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #618 - INST(Sm4ekey_v , ISimdVVVx , (0b1100111001100000110010, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 13 ), // #619 - INST(Smax_v , ISimdVVV , (0b0000111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 28 ), // #620 - INST(Smaxp_v , ISimdVVV , (0b0000111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 29 ), // #621 - INST(Smaxv_v , ISimdSV , (0b0000111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 2 ), // #622 - INST(Smin_v , ISimdVVV , (0b0000111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 30 ), // #623 - INST(Sminp_v , ISimdVVV , (0b0000111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 31 ), // #624 - INST(Sminv_v , ISimdSV , (0b0000111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 3 ), // #625 - INST(Smlal_v , ISimdVVVe , (0b0000111000100000100000, kVO_V_B8H4S2, 0b0000111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 3 ), // #626 - INST(Smlal2_v , ISimdVVVe , (0b0100111000100000100000, kVO_V_B16H8S4, 0b0100111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 4 ), // #627 - INST(Smlsl_v , ISimdVVVe , (0b0000111000100000101000, kVO_V_B8H4S2, 0b0000111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 5 ), // #628 - INST(Smlsl2_v , ISimdVVVe , (0b0100111000100000101000, kVO_V_B16H8S4, 0b0100111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 6 ), // #629 - INST(Smmla_v , ISimdVVVx , (0b0100111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 14 ), // #630 - INST(Smov_v , SimdSmovUmov , (0b0000111000000000001011, kVO_V_BHS, 1) , kRWI_W , 0 , 0 ), // #631 - INST(Smull_v , ISimdVVVe , (0b0000111000100000110000, kVO_V_B8H4S2, 0b0000111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 7 ), // #632 - INST(Smull2_v , ISimdVVVe , (0b0100111000100000110000, kVO_V_B16H8S4, 0b0100111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 8 ), // #633 - INST(Sqabs_v , ISimdVV , (0b0000111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 13 ), // #634 - INST(Sqadd_v , ISimdVVV , (0b0000111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 32 ), // #635 - INST(Sqdmlal_v , ISimdVVVe , (0b0000111000100000100100, kVO_SV_BHS, 0b0000111100000000001100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 9 ), // #636 - INST(Sqdmlal2_v , ISimdVVVe , (0b0100111000100000100100, kVO_V_B16H8S4, 0b0100111100000000001100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 10 ), // #637 - INST(Sqdmlsl_v , ISimdVVVe , (0b0000111000100000101100, kVO_SV_BHS, 0b0000111100000000011100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 11 ), // #638 - INST(Sqdmlsl2_v , ISimdVVVe , (0b0100111000100000101100, kVO_V_B16H8S4, 0b0100111100000000011100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 12 ), // #639 - INST(Sqdmulh_v , ISimdVVVe , (0b0000111000100000101101, kVO_SV_HS, 0b0000111100000000110000, kVO_SV_HS) , kRWI_W , F(VH0_15) , 13 ), // #640 - INST(Sqdmull_v , ISimdVVVe , (0b0000111000100000110100, kVO_SV_BHS, 0b0000111100000000101100, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 14 ), // #641 - INST(Sqdmull2_v , ISimdVVVe , (0b0100111000100000110100, kVO_V_B16H8S4, 0b0100111100000000101100, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 15 ), // #642 - INST(Sqneg_v , ISimdVV , (0b0010111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 14 ), // #643 - INST(Sqrdmlah_v , ISimdVVVe , (0b0010111000000000100001, kVO_SV_HS, 0b0010111100000000110100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 16 ), // #644 - INST(Sqrdmlsh_v , ISimdVVVe , (0b0010111000000000100011, kVO_SV_HS, 0b0010111100000000111100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 17 ), // #645 - INST(Sqrdmulh_v , ISimdVVVe , (0b0010111000100000101101, kVO_SV_HS, 0b0000111100000000110100, kVO_SV_HS) , kRWI_W , F(VH0_15) , 18 ), // #646 - INST(Sqrshl_v , SimdShift , (0b0000111000100000010111, 0b0000000000000000000000, 1, kVO_SV_Any) , kRWI_W , 0 , 6 ), // #647 - INST(Sqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 7 ), // #648 - INST(Sqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 8 ), // #649 - INST(Sqrshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100011, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 9 ), // #650 - INST(Sqrshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 10 ), // #651 - INST(Sqshl_v , SimdShift , (0b0000111000100000010011, 0b0000111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 11 ), // #652 - INST(Sqshlu_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000011001, 0, kVO_SV_Any) , kRWI_W , 0 , 12 ), // #653 - INST(Sqshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #654 - INST(Sqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #655 - INST(Sqshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100001, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #656 - INST(Sqshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #657 - INST(Sqsub_v , ISimdVVV , (0b0000111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 33 ), // #658 - INST(Sqxtn_v , ISimdVV , (0b0000111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #659 - INST(Sqxtn2_v , ISimdVV , (0b0100111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #660 - INST(Sqxtun_v , ISimdVV , (0b0010111000100001001010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 17 ), // #661 - INST(Sqxtun2_v , ISimdVV , (0b0110111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 18 ), // #662 - INST(Srhadd_v , ISimdVVV , (0b0000111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 34 ), // #663 - INST(Sri_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010001, 1, kVO_V_Any) , kRWI_W , 0 , 17 ), // #664 - INST(Srshl_v , SimdShift , (0b0000111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 18 ), // #665 - INST(Srshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 19 ), // #666 - INST(Srsra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 20 ), // #667 - INST(Sshl_v , SimdShift , (0b0000111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 21 ), // #668 - INST(Sshll_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #669 - INST(Sshll2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #670 - INST(Sshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 24 ), // #671 - INST(Ssra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 25 ), // #672 - INST(Ssubl_v , ISimdVVV , (0b0000111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 35 ), // #673 - INST(Ssubl2_v , ISimdVVV , (0b0100111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 36 ), // #674 - INST(Ssubw_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 2 ), // #675 - INST(Ssubw2_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B16H8S4) , kRWI_X , 0 , 3 ), // #676 - INST(St1_v , SimdLdNStN , (0b0000110100000000000000, 0b0000110000000000001000, 1, 0) , kRWI_STn , F(Consecutive) , 8 ), // #677 - INST(St2_v , SimdLdNStN , (0b0000110100100000000000, 0b0000110000000000100000, 2, 0) , kRWI_STn , F(Consecutive) , 9 ), // #678 - INST(St3_v , SimdLdNStN , (0b0000110100000000001000, 0b0000110000000000010000, 3, 0) , kRWI_STn , F(Consecutive) , 10 ), // #679 - INST(St4_v , SimdLdNStN , (0b0000110100100000001000, 0b0000110000000000000000, 4, 0) , kRWI_STn , F(Consecutive) , 11 ), // #680 - INST(Stnp_v , SimdLdpStp , (0b0010110000, 0b0000000000) , kRWI_RRW , 0 , 2 ), // #681 - INST(Stp_v , SimdLdpStp , (0b0010110100, 0b0010110010) , kRWI_RRW , 0 , 3 ), // #682 - INST(Str_v , SimdLdSt , (0b0011110100, 0b00111100000, 0b00111100001, 0b00000000, Inst::kIdStur_v) , kRWI_RW , 0 , 1 ), // #683 - INST(Stur_v , SimdLdurStur , (0b0011110000000000000000) , kRWI_RW , 0 , 1 ), // #684 - INST(Sub_v , ISimdVVV , (0b0010111000100000100001, kVO_V_Any) , kRWI_W , 0 , 37 ), // #685 - INST(Subhn_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 38 ), // #686 - INST(Subhn2_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 39 ), // #687 - INST(Sudot_v , SimdDot , (0b0000000000000000000000, 0b0000111100000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 2 ), // #688 - INST(Suqadd_v , ISimdVV , (0b0000111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 19 ), // #689 - INST(Sxtl_v , SimdSxtlUxtl , (0b0000111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #690 - INST(Sxtl2_v , SimdSxtlUxtl , (0b0100111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #691 - INST(Tbl_v , SimdTblTbx , (0b0000111000000000000000) , kRWI_W , 0 , 0 ), // #692 - INST(Tbx_v , SimdTblTbx , (0b0000111000000000000100) , kRWI_W , 0 , 1 ), // #693 - INST(Trn1_v , ISimdVVV , (0b0000111000000000001010, kVO_V_BHS_D2) , kRWI_W , 0 , 40 ), // #694 - INST(Trn2_v , ISimdVVV , (0b0000111000000000011010, kVO_V_BHS_D2) , kRWI_W , 0 , 41 ), // #695 - INST(Uaba_v , ISimdVVV , (0b0010111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 42 ), // #696 - INST(Uabal_v , ISimdVVV , (0b0010111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 43 ), // #697 - INST(Uabal2_v , ISimdVVV , (0b0110111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 44 ), // #698 - INST(Uabd_v , ISimdVVV , (0b0010111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 45 ), // #699 - INST(Uabdl_v , ISimdVVV , (0b0010111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 46 ), // #700 - INST(Uabdl2_v , ISimdVVV , (0b0110111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 47 ), // #701 - INST(Uadalp_v , ISimdVV , (0b0010111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 20 ), // #702 - INST(Uaddl_v , ISimdVVV , (0b0010111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 48 ), // #703 - INST(Uaddl2_v , ISimdVVV , (0b0110111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 49 ), // #704 - INST(Uaddlp_v , ISimdVV , (0b0010111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 21 ), // #705 - INST(Uaddlv_v , ISimdSV , (0b0010111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 4 ), // #706 - INST(Uaddw_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 4 ), // #707 - INST(Uaddw2_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 5 ), // #708 - INST(Ucvtf_v , SimdFcvtSV , (0b0010111000100001110110, 0b0010111100000000111001, 0b0001111000100011000000, 0) , kRWI_W , 0 , 11 ), // #709 - INST(Udot_v , SimdDot , (0b0010111010000000100101, 0b0010111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 3 ), // #710 - INST(Uhadd_v , ISimdVVV , (0b0010111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 50 ), // #711 - INST(Uhsub_v , ISimdVVV , (0b0010111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 51 ), // #712 - INST(Umax_v , ISimdVVV , (0b0010111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 52 ), // #713 - INST(Umaxp_v , ISimdVVV , (0b0010111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 53 ), // #714 - INST(Umaxv_v , ISimdSV , (0b0010111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 5 ), // #715 - INST(Umin_v , ISimdVVV , (0b0010111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 54 ), // #716 - INST(Uminp_v , ISimdVVV , (0b0010111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 55 ), // #717 - INST(Uminv_v , ISimdSV , (0b0010111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 6 ), // #718 - INST(Umlal_v , ISimdVVVe , (0b0010111000100000100000, kVO_V_B8H4S2, 0b0010111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 19 ), // #719 - INST(Umlal2_v , ISimdVVVe , (0b0110111000100000100000, kVO_V_B16H8S4, 0b0010111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 20 ), // #720 - INST(Umlsl_v , ISimdVVVe , (0b0010111000100000101000, kVO_V_B8H4S2, 0b0010111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 21 ), // #721 - INST(Umlsl2_v , ISimdVVVe , (0b0110111000100000101000, kVO_V_B16H8S4, 0b0110111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 22 ), // #722 - INST(Ummla_v , ISimdVVVx , (0b0110111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 15 ), // #723 - INST(Umov_v , SimdSmovUmov , (0b0000111000000000001111, kVO_V_Any, 0) , kRWI_W , 0 , 1 ), // #724 - INST(Umull_v , ISimdVVVe , (0b0010111000100000110000, kVO_V_B8H4S2, 0b0010111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 23 ), // #725 - INST(Umull2_v , ISimdVVVe , (0b0110111000100000110000, kVO_V_B16H8S4, 0b0110111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 24 ), // #726 - INST(Uqadd_v , ISimdVVV , (0b0010111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 56 ), // #727 - INST(Uqrshl_v , SimdShift , (0b0010111000100000010111, 0b0000000000000000000000, 0, kVO_SV_Any) , kRWI_W , 0 , 26 ), // #728 - INST(Uqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #729 - INST(Uqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #730 - INST(Uqshl_v , SimdShift , (0b0010111000100000010011, 0b0010111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 29 ), // #731 - INST(Uqshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 30 ), // #732 - INST(Uqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 31 ), // #733 - INST(Uqsub_v , ISimdVVV , (0b0010111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 57 ), // #734 - INST(Uqxtn_v , ISimdVV , (0b0010111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 22 ), // #735 - INST(Uqxtn2_v , ISimdVV , (0b0110111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 23 ), // #736 - INST(Urecpe_v , ISimdVV , (0b0000111010100001110010, kVO_V_S) , kRWI_W , 0 , 24 ), // #737 - INST(Urhadd_v , ISimdVVV , (0b0010111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 58 ), // #738 - INST(Urshl_v , SimdShift , (0b0010111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 32 ), // #739 - INST(Urshr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 33 ), // #740 - INST(Ursqrte_v , ISimdVV , (0b0010111010100001110010, kVO_V_S) , kRWI_W , 0 , 25 ), // #741 - INST(Ursra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 34 ), // #742 - INST(Usdot_v , SimdDot , (0b0000111010000000100111, 0b0000111110000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 4 ), // #743 - INST(Ushl_v , SimdShift , (0b0010111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 35 ), // #744 - INST(Ushll_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 36 ), // #745 - INST(Ushll2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 37 ), // #746 - INST(Ushr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 38 ), // #747 - INST(Usmmla_v , ISimdVVVx , (0b0100111010000000101011, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 16 ), // #748 - INST(Usqadd_v , ISimdVV , (0b0010111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 26 ), // #749 - INST(Usra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 39 ), // #750 - INST(Usubl_v , ISimdVVV , (0b0010111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 59 ), // #751 - INST(Usubl2_v , ISimdVVV , (0b0110111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 60 ), // #752 - INST(Usubw_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 6 ), // #753 - INST(Usubw2_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B16H8S4) , kRWI_W , 0 , 7 ), // #754 - INST(Uxtl_v , SimdSxtlUxtl , (0b0010111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 2 ), // #755 - INST(Uxtl2_v , SimdSxtlUxtl , (0b0110111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 3 ), // #756 - INST(Uzp1_v , ISimdVVV , (0b0000111000000000000110, kVO_V_BHS_D2) , kRWI_W , 0 , 61 ), // #757 - INST(Uzp2_v , ISimdVVV , (0b0000111000000000010110, kVO_V_BHS_D2) , kRWI_W , 0 , 62 ), // #758 - INST(Xar_v , ISimdVVVI , (0b1100111001100000100011, kVO_V_D2, 6, 10, 0) , kRWI_W , 0 , 1 ), // #759 - INST(Xtn_v , ISimdVV , (0b0000111000100001001010, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #760 - INST(Xtn2_v , ISimdVV , (0b0100111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #761 - INST(Zip1_v , ISimdVVV , (0b0000111000000000001110, kVO_V_BHS_D2) , kRWI_W , 0 , 63 ), // #762 - INST(Zip2_v , ISimdVVV , (0b0000111000000000011110, kVO_V_BHS_D2) , kRWI_W , 0 , 64 ) // #763 + INST(Abs , BaseRR , (0b01011010110000000010000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 0 ), // #1 + INST(Adc , BaseRRR , (0b0001101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #2 + INST(Adcs , BaseRRR , (0b0011101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #3 + INST(Add , BaseAddSub , (0b0001011000, 0b0001011001, 0b0010001) , kRWI_W , 0 , 0 ), // #4 + INST(Addg , BaseRRII , (0b1001000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 0 ), // #5 + INST(Adds , BaseAddSub , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_W , 0 , 1 ), // #6 + INST(Adr , BaseAdr , (0b0001000000000000000000, OffsetType::kAArch64_ADR) , kRWI_W , 0 , 0 ), // #7 + INST(Adrp , BaseAdr , (0b1001000000000000000000, OffsetType::kAArch64_ADRP) , kRWI_W , 0 , 1 ), // #8 + INST(And , BaseLogical , (0b0001010000, 0b00100100, 0) , kRWI_W , 0 , 0 ), // #9 + INST(Ands , BaseLogical , (0b1101010000, 0b11100100, 0) , kRWI_W , 0 , 1 ), // #10 + INST(Asr , BaseShift , (0b0001101011000000001010, 0b0001001100000000011111, 0) , kRWI_W , 0 , 0 ), // #11 + INST(Asrv , BaseShift , (0b0001101011000000001010, 0b0000000000000000000000, 0) , kRWI_W , 0 , 1 ), // #12 + INST(At , BaseAtDcIcTlbi , (0b00011111110000, 0b00001111000000, true) , kRWI_RX , 0 , 0 ), // #13 + INST(Autda , BaseRR , (0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 1 ), // #14 + INST(Autdza , BaseR , (0b11011010110000010011101111100000, kX, kZR, 0) , kRWI_X , 0 , 0 ), // #15 + INST(Autdb , BaseRR , (0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 2 ), // #16 + INST(Autdzb , BaseR , (0b11011010110000010011111111100000, kX, kZR, 0) , kRWI_X , 0 , 1 ), // #17 + INST(Autia , BaseRR , (0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 3 ), // #18 + INST(Autia1716 , BaseOp , (0b11010101000000110010000110011111) , 0 , 0 , 0 ), // #19 + INST(Autiasp , BaseOp , (0b11010101000000110010001110111111) , 0 , 0 , 1 ), // #20 + INST(Autiaz , BaseOp , (0b11010101000000110010001110011111) , 0 , 0 , 2 ), // #21 + INST(Autib , BaseRR , (0b11011010110000010001010000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 4 ), // #22 + INST(Autib1716 , BaseOp , (0b11010101000000110010000111011111) , 0 , 0 , 3 ), // #23 + INST(Autibsp , BaseOp , (0b11010101000000110010001111111111) , 0 , 0 , 4 ), // #24 + INST(Autibz , BaseOp , (0b11010101000000110010001111011111) , 0 , 0 , 5 ), // #25 + INST(Autiza , BaseR , (0b11011010110000010011001111100000, kX, kZR, 0) , kRWI_X , 0 , 2 ), // #26 + INST(Autizb , BaseR , (0b11011010110000010011011111100000, kX, kZR, 0) , kRWI_X , 0 , 3 ), // #27 + INST(Axflag , BaseOp , (0b11010101000000000100000001011111) , 0 , 0 , 6 ), // #28 + INST(B , BaseBranchRel , (0b00010100000000000000000000000000) , 0 , F(Cond) , 0 ), // #29 + INST(Bc , BaseBranchRel , (0b00010100000000000000000000010000) , 0 , F(Cond) , 1 ), // #30 + INST(Bfc , BaseBfc , (0b00110011000000000000001111100000) , kRWI_X , 0 , 0 ), // #31 + INST(Bfi , BaseBfi , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #32 + INST(Bfm , BaseBfm , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #33 + INST(Bfxil , BaseBfx , (0b00110011000000000000000000000000) , kRWI_X , 0 , 0 ), // #34 + INST(Bic , BaseLogical , (0b0001010001, 0b00100100, 1) , kRWI_W , 0 , 2 ), // #35 + INST(Bics , BaseLogical , (0b1101010001, 0b11100100, 1) , kRWI_W , 0 , 3 ), // #36 + INST(Bl , BaseBranchRel , (0b10010100000000000000000000000000) , 0 , 0 , 2 ), // #37 + INST(Blr , BaseBranchReg , (0b11010110001111110000000000000000) , kRWI_R , 0 , 0 ), // #38 + INST(Br , BaseBranchReg , (0b11010110000111110000000000000000) , kRWI_R , 0 , 1 ), // #39 + INST(Brk , BaseOpImm , (0b11010100001000000000000000000000, 16, 5) , 0 , 0 , 0 ), // #40 + INST(Bti , BaseOpImm , (0b11010101000000110010010000011111, 2, 6) , 0 , 0 , 1 ), // #41 + INST(Cas , BaseAtomicOp , (0b1000100010100000011111, kWX, 30, 0) , kRWI_XRX , 0 , 0 ), // #42 + INST(Casa , BaseAtomicOp , (0b1000100011100000011111, kWX, 30, 1) , kRWI_XRX , 0 , 1 ), // #43 + INST(Casab , BaseAtomicOp , (0b0000100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 2 ), // #44 + INST(Casah , BaseAtomicOp , (0b0100100011100000011111, kW , 0 , 1) , kRWI_XRX , 0 , 3 ), // #45 + INST(Casal , BaseAtomicOp , (0b1000100011100000111111, kWX, 30, 1) , kRWI_XRX , 0 , 4 ), // #46 + INST(Casalb , BaseAtomicOp , (0b0000100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 5 ), // #47 + INST(Casalh , BaseAtomicOp , (0b0100100011100000111111, kW , 0 , 1) , kRWI_XRX , 0 , 6 ), // #48 + INST(Casb , BaseAtomicOp , (0b0000100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 7 ), // #49 + INST(Cash , BaseAtomicOp , (0b0100100010100000011111, kW , 0 , 0) , kRWI_XRX , 0 , 8 ), // #50 + INST(Casl , BaseAtomicOp , (0b1000100010100000111111, kWX, 30, 0) , kRWI_XRX , 0 , 9 ), // #51 + INST(Caslb , BaseAtomicOp , (0b0000100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 10 ), // #52 + INST(Caslh , BaseAtomicOp , (0b0100100010100000111111, kW , 0 , 0) , kRWI_XRX , 0 , 11 ), // #53 + INST(Casp , BaseAtomicCasp , (0b0000100000100000011111, kWX, 30) , kRWI_XXRRX, 0 , 0 ), // #54 + INST(Caspa , BaseAtomicCasp , (0b0000100001100000011111, kWX, 30) , kRWI_XXRRX, 0 , 1 ), // #55 + INST(Caspal , BaseAtomicCasp , (0b0000100001100000111111, kWX, 30) , kRWI_XXRRX, 0 , 2 ), // #56 + INST(Caspl , BaseAtomicCasp , (0b0000100000100000111111, kWX, 30) , kRWI_XXRRX, 0 , 3 ), // #57 + INST(Cbnz , BaseBranchCmp , (0b00110101000000000000000000000000) , kRWI_R , 0 , 0 ), // #58 + INST(Cbz , BaseBranchCmp , (0b00110100000000000000000000000000) , kRWI_R , 0 , 1 ), // #59 + INST(Ccmn , BaseCCmp , (0b00111010010000000000000000000000) , kRWI_R , 0 , 0 ), // #60 + INST(Ccmp , BaseCCmp , (0b01111010010000000000000000000000) , kRWI_R , 0 , 1 ), // #61 + INST(Cfinv , BaseOp , (0b11010101000000000100000000011111) , 0 , 0 , 7 ), // #62 + INST(Chkfeat , BaseOpX16 , (0b11010101000000110010010100011111) , 0 , 0 , 0 ), // #63 + INST(Cinc , BaseCInc , (0b00011010100000000000010000000000) , kRWI_W , 0 , 0 ), // #64 + INST(Cinv , BaseCInc , (0b01011010100000000000000000000000) , kRWI_W , 0 , 1 ), // #65 + INST(Clrbhb , BaseOp , (0b11010101000000110010001011011111) , 0 , 0 , 8 ), // #66 + INST(Clrex , BaseOpImm , (0b11010101000000110011000001011111, 4, 8) , 0 , 0 , 2 ), // #67 + INST(Cls , BaseRR , (0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 5 ), // #68 + INST(Clz , BaseRR , (0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 6 ), // #69 + INST(Cmn , BaseCmpCmn , (0b0101011000, 0b0101011001, 0b0110001) , kRWI_R , 0 , 0 ), // #70 + INST(Cmp , BaseCmpCmn , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_R , 0 , 1 ), // #71 + INST(Cmpp , BaseRR , (0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true) , kRWI_R , 0 , 7 ), // #72 + INST(Cneg , BaseCInc , (0b01011010100000000000010000000000) , kRWI_W , 0 , 2 ), // #73 + INST(Cnt , BaseRR , (0b01011010110000000001110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 8 ), // #74 + INST(Crc32b , BaseRRR , (0b0001101011000000010000, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 2 ), // #75 + INST(Crc32cb , BaseRRR , (0b0001101011000000010100, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 3 ), // #76 + INST(Crc32ch , BaseRRR , (0b0001101011000000010101, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 4 ), // #77 + INST(Crc32cw , BaseRRR , (0b0001101011000000010110, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 5 ), // #78 + INST(Crc32cx , BaseRRR , (0b1001101011000000010111, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 6 ), // #79 + INST(Crc32h , BaseRRR , (0b0001101011000000010001, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 7 ), // #80 + INST(Crc32w , BaseRRR , (0b0001101011000000010010, kW, kZR, kW, kZR, kW, kZR, false) , kRWI_W , 0 , 8 ), // #81 + INST(Crc32x , BaseRRR , (0b1001101011000000010011, kW, kZR, kW, kZR, kX, kZR, false) , kRWI_W , 0 , 9 ), // #82 + INST(Csdb , BaseOp , (0b11010101000000110010001010011111) , 0 , 0 , 9 ), // #83 + INST(Csel , BaseCSel , (0b00011010100000000000000000000000) , kRWI_W , 0 , 0 ), // #84 + INST(Cset , BaseCSet , (0b00011010100111110000011111100000) , kRWI_W , 0 , 0 ), // #85 + INST(Csetm , BaseCSet , (0b01011010100111110000001111100000) , kRWI_W , 0 , 1 ), // #86 + INST(Csinc , BaseCSel , (0b00011010100000000000010000000000) , kRWI_W , 0 , 1 ), // #87 + INST(Csinv , BaseCSel , (0b01011010100000000000000000000000) , kRWI_W , 0 , 2 ), // #88 + INST(Csneg , BaseCSel , (0b01011010100000000000010000000000) , kRWI_W , 0 , 3 ), // #89 + INST(Ctz , BaseRR , (0b01011010110000000001100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 9 ), // #90 + INST(Dc , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, true) , kRWI_RX , 0 , 1 ), // #91 + INST(Dcps1 , BaseOpImm , (0b11010100101000000000000000000001, 16, 5) , 0 , 0 , 3 ), // #92 + INST(Dcps2 , BaseOpImm , (0b11010100101000000000000000000010, 16, 5) , 0 , 0 , 4 ), // #93 + INST(Dcps3 , BaseOpImm , (0b11010100101000000000000000000011, 16, 5) , 0 , 0 , 5 ), // #94 + INST(Dgh , BaseOp , (0b11010101000000110010000011011111) , 0 , 0 , 10 ), // #95 + INST(Dmb , BaseOpImm , (0b11010101000000110011000010111111, 4, 8) , 0 , 0 , 6 ), // #96 + INST(Drps , BaseOp , (0b11010110101111110000001111100000) , 0 , 0 , 11 ), // #97 + INST(Dsb , BaseOpImm , (0b11010101000000110011000010011111, 4, 8) , 0 , 0 , 7 ), // #98 + INST(Eon , BaseLogical , (0b1001010001, 0b10100100, 1) , kRWI_W , 0 , 4 ), // #99 + INST(Eor , BaseLogical , (0b1001010000, 0b10100100, 0) , kRWI_W , 0 , 5 ), // #100 + INST(Esb , BaseOp , (0b11010101000000110010001000011111) , 0 , 0 , 12 ), // #101 + INST(Extr , BaseExtract , (0b00010011100000000000000000000000) , kRWI_W , 0 , 0 ), // #102 + INST(Eret , BaseOp , (0b11010110100111110000001111100000) , 0 , 0 , 13 ), // #103 + INST(Gmi , BaseRRR , (0b1001101011000000000101, kX , kZR, kX , kSP, kX , kZR, true) , kRWI_W , 0 , 10 ), // #104 + INST(Hint , BaseOpImm , (0b11010101000000110010000000011111, 7, 5) , 0 , 0 , 8 ), // #105 + INST(Hlt , BaseOpImm , (0b11010100010000000000000000000000, 16, 5) , 0 , 0 , 9 ), // #106 + INST(Hvc , BaseOpImm , (0b11010100000000000000000000000010, 16, 5) , 0 , 0 , 10 ), // #107 + INST(Ic , BaseAtDcIcTlbi , (0b00011110000000, 0b00001110000000, false) , kRWI_RX , 0 , 2 ), // #108 + INST(Isb , BaseOpImm , (0b11010101000000110011000011011111, 4, 8) , 0 , 0 , 11 ), // #109 + INST(Ldadd , BaseAtomicOp , (0b1011100000100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 12 ), // #110 + INST(Ldadda , BaseAtomicOp , (0b1011100010100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 13 ), // #111 + INST(Ldaddab , BaseAtomicOp , (0b0011100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 14 ), // #112 + INST(Ldaddah , BaseAtomicOp , (0b0111100010100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 15 ), // #113 + INST(Ldaddal , BaseAtomicOp , (0b1011100011100000000000, kWX, 30, 1) , kRWI_WRX , 0 , 16 ), // #114 + INST(Ldaddalb , BaseAtomicOp , (0b0011100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 17 ), // #115 + INST(Ldaddalh , BaseAtomicOp , (0b0111100011100000000000, kW , 0 , 1) , kRWI_WRX , 0 , 18 ), // #116 + INST(Ldaddb , BaseAtomicOp , (0b0011100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 19 ), // #117 + INST(Ldaddh , BaseAtomicOp , (0b0111100000100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 20 ), // #118 + INST(Ldaddl , BaseAtomicOp , (0b1011100001100000000000, kWX, 30, 0) , kRWI_WRX , 0 , 21 ), // #119 + INST(Ldaddlb , BaseAtomicOp , (0b0011100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 22 ), // #120 + INST(Ldaddlh , BaseAtomicOp , (0b0111100001100000000000, kW , 0 , 0) , kRWI_WRX , 0 , 23 ), // #121 + INST(Ldar , BaseRM_NoImm , (0b1000100011011111111111, kWX, kZR, 30) , kRWI_W , 0 , 0 ), // #122 + INST(Ldarb , BaseRM_NoImm , (0b0000100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 1 ), // #123 + INST(Ldarh , BaseRM_NoImm , (0b0100100011011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 2 ), // #124 + INST(Ldaxp , BaseLdxp , (0b1000100001111111100000, kWX, 30) , kRWI_WW , 0 , 0 ), // #125 + INST(Ldaxr , BaseRM_NoImm , (0b1000100001011111111111, kWX, kZR, 30) , kRWI_W , 0 , 3 ), // #126 + INST(Ldaxrb , BaseRM_NoImm , (0b0000100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 4 ), // #127 + INST(Ldaxrh , BaseRM_NoImm , (0b0100100001011111111111, kW , kZR, 0 ) , kRWI_W , 0 , 5 ), // #128 + INST(Ldclr , BaseAtomicOp , (0b1011100000100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 24 ), // #129 + INST(Ldclra , BaseAtomicOp , (0b1011100010100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 25 ), // #130 + INST(Ldclrab , BaseAtomicOp , (0b0011100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 26 ), // #131 + INST(Ldclrah , BaseAtomicOp , (0b0111100010100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 27 ), // #132 + INST(Ldclral , BaseAtomicOp , (0b1011100011100000000100, kWX, 30, 1) , kRWI_WRX , 0 , 28 ), // #133 + INST(Ldclralb , BaseAtomicOp , (0b0011100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 29 ), // #134 + INST(Ldclralh , BaseAtomicOp , (0b0111100011100000000100, kW , 0 , 1) , kRWI_WRX , 0 , 30 ), // #135 + INST(Ldclrb , BaseAtomicOp , (0b0011100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 31 ), // #136 + INST(Ldclrh , BaseAtomicOp , (0b0111100000100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 32 ), // #137 + INST(Ldclrl , BaseAtomicOp , (0b1011100001100000000100, kWX, 30, 0) , kRWI_WRX , 0 , 33 ), // #138 + INST(Ldclrlb , BaseAtomicOp , (0b0011100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 34 ), // #139 + INST(Ldclrlh , BaseAtomicOp , (0b0111100001100000000100, kW , 0 , 0) , kRWI_WRX , 0 , 35 ), // #140 + INST(Ldeor , BaseAtomicOp , (0b1011100000100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 36 ), // #141 + INST(Ldeora , BaseAtomicOp , (0b1011100010100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 37 ), // #142 + INST(Ldeorab , BaseAtomicOp , (0b0011100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 38 ), // #143 + INST(Ldeorah , BaseAtomicOp , (0b0111100010100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 39 ), // #144 + INST(Ldeoral , BaseAtomicOp , (0b1011100011100000001000, kWX, 30, 1) , kRWI_WRX , 0 , 40 ), // #145 + INST(Ldeoralb , BaseAtomicOp , (0b0011100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 41 ), // #146 + INST(Ldeoralh , BaseAtomicOp , (0b0111100011100000001000, kW , 0 , 1) , kRWI_WRX , 0 , 42 ), // #147 + INST(Ldeorb , BaseAtomicOp , (0b0011100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 43 ), // #148 + INST(Ldeorh , BaseAtomicOp , (0b0111100000100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 44 ), // #149 + INST(Ldeorl , BaseAtomicOp , (0b1011100001100000001000, kWX, 30, 0) , kRWI_WRX , 0 , 45 ), // #150 + INST(Ldeorlb , BaseAtomicOp , (0b0011100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 46 ), // #151 + INST(Ldeorlh , BaseAtomicOp , (0b0111100001100000001000, kW , 0 , 0) , kRWI_WRX , 0 , 47 ), // #152 + INST(Ldg , BaseRM_SImm9 , (0b1101100101100000000000, 0b0000000000000000000000, kX , kZR, 0, 4) , kRWI_W , 0 , 0 ), // #153 + INST(Ldgm , BaseRM_NoImm , (0b1101100111100000000000, kX , kZR, 0 ) , kRWI_W , 0 , 6 ), // #154 + INST(Ldlar , BaseRM_NoImm , (0b1000100011011111011111, kWX, kZR, 30) , kRWI_W , 0 , 7 ), // #155 + INST(Ldlarb , BaseRM_NoImm , (0b0000100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 8 ), // #156 + INST(Ldlarh , BaseRM_NoImm , (0b0100100011011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 9 ), // #157 + INST(Ldnp , BaseLdpStp , (0b0010100001, 0 , kWX, 31, 2) , kRWI_WW , 0 , 0 ), // #158 + INST(Ldp , BaseLdpStp , (0b0010100101, 0b0010100011, kWX, 31, 2) , kRWI_WW , 0 , 1 ), // #159 + INST(Ldpsw , BaseLdpStp , (0b0110100101, 0b0110100011, kX , 0 , 2) , kRWI_WW , 0 , 2 ), // #160 + INST(Ldr , BaseLdSt , (0b1011100101, 0b10111000010, 0b10111000011, 0b00011000, kWX, 30, 2, Inst::kIdLdur) , kRWI_W , 0 , 0 ), // #161 + INST(Ldraa , BaseRM_SImm10 , (0b1111100000100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 0 ), // #162 + INST(Ldrab , BaseRM_SImm10 , (0b1111100010100000000001, kX , kZR, 0, 3) , kRWI_W , 0 , 1 ), // #163 + INST(Ldrb , BaseLdSt , (0b0011100101, 0b00111000010, 0b00111000011, 0 , kW , 0 , 0, Inst::kIdLdurb) , kRWI_W , 0 , 1 ), // #164 + INST(Ldrh , BaseLdSt , (0b0111100101, 0b01111000010, 0b01111000011, 0 , kW , 0 , 1, Inst::kIdLdurh) , kRWI_W , 0 , 2 ), // #165 + INST(Ldrsb , BaseLdSt , (0b0011100111, 0b00111000100, 0b00111000111, 0 , kWX, 22, 0, Inst::kIdLdursb) , kRWI_W , 0 , 3 ), // #166 + INST(Ldrsh , BaseLdSt , (0b0111100111, 0b01111000100, 0b01111000111, 0 , kWX, 22, 1, Inst::kIdLdursh) , kRWI_W , 0 , 4 ), // #167 + INST(Ldrsw , BaseLdSt , (0b1011100110, 0b10111000100, 0b10111000101, 0b10011000, kX , 0 , 2, Inst::kIdLdursw) , kRWI_W , 0 , 5 ), // #168 + INST(Ldset , BaseAtomicOp , (0b1011100000100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 48 ), // #169 + INST(Ldseta , BaseAtomicOp , (0b1011100010100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 49 ), // #170 + INST(Ldsetab , BaseAtomicOp , (0b0011100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 50 ), // #171 + INST(Ldsetah , BaseAtomicOp , (0b0111100010100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 51 ), // #172 + INST(Ldsetal , BaseAtomicOp , (0b1011100011100000001100, kWX, 30, 1) , kRWI_WRX , 0 , 52 ), // #173 + INST(Ldsetalb , BaseAtomicOp , (0b0011100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 53 ), // #174 + INST(Ldsetalh , BaseAtomicOp , (0b0111100011100000001100, kW , 0 , 1) , kRWI_WRX , 0 , 54 ), // #175 + INST(Ldsetb , BaseAtomicOp , (0b0011100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 55 ), // #176 + INST(Ldseth , BaseAtomicOp , (0b0111100000100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 56 ), // #177 + INST(Ldsetl , BaseAtomicOp , (0b1011100001100000001100, kWX, 30, 0) , kRWI_WRX , 0 , 57 ), // #178 + INST(Ldsetlb , BaseAtomicOp , (0b0011100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 58 ), // #179 + INST(Ldsetlh , BaseAtomicOp , (0b0111100001100000001100, kW , 0 , 0) , kRWI_WRX , 0 , 59 ), // #180 + INST(Ldsmax , BaseAtomicOp , (0b1011100000100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 60 ), // #181 + INST(Ldsmaxa , BaseAtomicOp , (0b1011100010100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 61 ), // #182 + INST(Ldsmaxab , BaseAtomicOp , (0b0011100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 62 ), // #183 + INST(Ldsmaxah , BaseAtomicOp , (0b0111100010100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 63 ), // #184 + INST(Ldsmaxal , BaseAtomicOp , (0b1011100011100000010000, kWX, 30, 1) , kRWI_WRX , 0 , 64 ), // #185 + INST(Ldsmaxalb , BaseAtomicOp , (0b0011100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 65 ), // #186 + INST(Ldsmaxalh , BaseAtomicOp , (0b0111100011100000010000, kW , 0 , 1) , kRWI_WRX , 0 , 66 ), // #187 + INST(Ldsmaxb , BaseAtomicOp , (0b0011100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 67 ), // #188 + INST(Ldsmaxh , BaseAtomicOp , (0b0111100000100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 68 ), // #189 + INST(Ldsmaxl , BaseAtomicOp , (0b1011100001100000010000, kWX, 30, 0) , kRWI_WRX , 0 , 69 ), // #190 + INST(Ldsmaxlb , BaseAtomicOp , (0b0011100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 70 ), // #191 + INST(Ldsmaxlh , BaseAtomicOp , (0b0111100001100000010000, kW , 0 , 0) , kRWI_WRX , 0 , 71 ), // #192 + INST(Ldsmin , BaseAtomicOp , (0b1011100000100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 72 ), // #193 + INST(Ldsmina , BaseAtomicOp , (0b1011100010100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 73 ), // #194 + INST(Ldsminab , BaseAtomicOp , (0b0011100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 74 ), // #195 + INST(Ldsminah , BaseAtomicOp , (0b0111100010100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 75 ), // #196 + INST(Ldsminal , BaseAtomicOp , (0b1011100011100000010100, kWX, 30, 1) , kRWI_WRX , 0 , 76 ), // #197 + INST(Ldsminalb , BaseAtomicOp , (0b0011100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 77 ), // #198 + INST(Ldsminalh , BaseAtomicOp , (0b0111100011100000010100, kW , 0 , 1) , kRWI_WRX , 0 , 78 ), // #199 + INST(Ldsminb , BaseAtomicOp , (0b0011100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 79 ), // #200 + INST(Ldsminh , BaseAtomicOp , (0b0111100000100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 80 ), // #201 + INST(Ldsminl , BaseAtomicOp , (0b1011100001100000010100, kWX, 30, 0) , kRWI_WRX , 0 , 81 ), // #202 + INST(Ldsminlb , BaseAtomicOp , (0b0011100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 82 ), // #203 + INST(Ldsminlh , BaseAtomicOp , (0b0111100001100000010100, kW , 0 , 0) , kRWI_WRX , 0 , 83 ), // #204 + INST(Ldtr , BaseRM_SImm9 , (0b1011100001000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 1 ), // #205 + INST(Ldtrb , BaseRM_SImm9 , (0b0011100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 2 ), // #206 + INST(Ldtrh , BaseRM_SImm9 , (0b0111100001000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 3 ), // #207 + INST(Ldtrsb , BaseRM_SImm9 , (0b0011100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 4 ), // #208 + INST(Ldtrsh , BaseRM_SImm9 , (0b0111100011000000000010, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 5 ), // #209 + INST(Ldtrsw , BaseRM_SImm9 , (0b1011100010000000000010, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 6 ), // #210 + INST(Ldumax , BaseAtomicOp , (0b1011100000100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 84 ), // #211 + INST(Ldumaxa , BaseAtomicOp , (0b1011100010100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 85 ), // #212 + INST(Ldumaxab , BaseAtomicOp , (0b0011100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 86 ), // #213 + INST(Ldumaxah , BaseAtomicOp , (0b0111100010100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 87 ), // #214 + INST(Ldumaxal , BaseAtomicOp , (0b1011100011100000011000, kWX, 30, 1) , kRWI_WRX , 0 , 88 ), // #215 + INST(Ldumaxalb , BaseAtomicOp , (0b0011100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 89 ), // #216 + INST(Ldumaxalh , BaseAtomicOp , (0b0111100011100000011000, kW , 0 , 1) , kRWI_WRX , 0 , 90 ), // #217 + INST(Ldumaxb , BaseAtomicOp , (0b0011100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 91 ), // #218 + INST(Ldumaxh , BaseAtomicOp , (0b0111100000100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 92 ), // #219 + INST(Ldumaxl , BaseAtomicOp , (0b1011100001100000011000, kWX, 30, 0) , kRWI_WRX , 0 , 93 ), // #220 + INST(Ldumaxlb , BaseAtomicOp , (0b0011100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 94 ), // #221 + INST(Ldumaxlh , BaseAtomicOp , (0b0111100001100000011000, kW , 0 , 0) , kRWI_WRX , 0 , 95 ), // #222 + INST(Ldumin , BaseAtomicOp , (0b1011100000100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 96 ), // #223 + INST(Ldumina , BaseAtomicOp , (0b1011100010100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 97 ), // #224 + INST(Lduminab , BaseAtomicOp , (0b0011100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 98 ), // #225 + INST(Lduminah , BaseAtomicOp , (0b0111100010100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 99 ), // #226 + INST(Lduminal , BaseAtomicOp , (0b1011100011100000011100, kWX, 30, 1) , kRWI_WRX , 0 , 100), // #227 + INST(Lduminalb , BaseAtomicOp , (0b0011100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 101), // #228 + INST(Lduminalh , BaseAtomicOp , (0b0111100011100000011100, kW , 0 , 1) , kRWI_WRX , 0 , 102), // #229 + INST(Lduminb , BaseAtomicOp , (0b0011100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 103), // #230 + INST(Lduminh , BaseAtomicOp , (0b0111100000100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 104), // #231 + INST(Lduminl , BaseAtomicOp , (0b1011100001100000011100, kWX, 30, 0) , kRWI_WRX , 0 , 105), // #232 + INST(Lduminlb , BaseAtomicOp , (0b0011100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 106), // #233 + INST(Lduminlh , BaseAtomicOp , (0b0111100001100000011100, kW , 0 , 0) , kRWI_WRX , 0 , 107), // #234 + INST(Ldur , BaseRM_SImm9 , (0b1011100001000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_W , 0 , 7 ), // #235 + INST(Ldurb , BaseRM_SImm9 , (0b0011100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 8 ), // #236 + INST(Ldurh , BaseRM_SImm9 , (0b0111100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_W , 0 , 9 ), // #237 + INST(Ldursb , BaseRM_SImm9 , (0b0011100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 10 ), // #238 + INST(Ldursh , BaseRM_SImm9 , (0b0111100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0) , kRWI_W , 0 , 11 ), // #239 + INST(Ldursw , BaseRM_SImm9 , (0b1011100010000000000000, 0b0000000000000000000000, kX , kZR, 0 , 0) , kRWI_W , 0 , 12 ), // #240 + INST(Ldxp , BaseLdxp , (0b1000100001111111000000, kWX, 30) , kRWI_WW , 0 , 1 ), // #241 + INST(Ldxr , BaseRM_NoImm , (0b1000100001011111011111, kWX, kZR, 30) , kRWI_W , 0 , 10 ), // #242 + INST(Ldxrb , BaseRM_NoImm , (0b0000100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 11 ), // #243 + INST(Ldxrh , BaseRM_NoImm , (0b0100100001011111011111, kW , kZR, 0 ) , kRWI_W , 0 , 12 ), // #244 + INST(Lsl , BaseShift , (0b0001101011000000001000, 0b0101001100000000000000, 0) , kRWI_W , 0 , 2 ), // #245 + INST(Lslv , BaseShift , (0b0001101011000000001000, 0b0000000000000000000000, 0) , kRWI_W , 0 , 3 ), // #246 + INST(Lsr , BaseShift , (0b0001101011000000001001, 0b0101001100000000011111, 0) , kRWI_W , 0 , 4 ), // #247 + INST(Lsrv , BaseShift , (0b0001101011000000001001, 0b0000000000000000000000, 0) , kRWI_W , 0 , 5 ), // #248 + INST(Madd , BaseRRRR , (0b0001101100000000000000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 0 ), // #249 + INST(Mneg , BaseRRR , (0b0001101100000000111111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 11 ), // #250 + INST(Mov , BaseMov , (_) , kRWI_W , 0 , 0 ), // #251 + INST(Movk , BaseMovKNZ , (0b01110010100000000000000000000000) , kRWI_X , 0 , 0 ), // #252 + INST(Movn , BaseMovKNZ , (0b00010010100000000000000000000000) , kRWI_W , 0 , 1 ), // #253 + INST(Movz , BaseMovKNZ , (0b01010010100000000000000000000000) , kRWI_W , 0 , 2 ), // #254 + INST(Mrs , BaseMrs , (_) , kRWI_W , 0 , 0 ), // #255 + INST(Msr , BaseMsr , (_) , kRWI_W , 0 , 0 ), // #256 + INST(Msub , BaseRRRR , (0b0001101100000000100000, kWX, kZR, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 1 ), // #257 + INST(Mul , BaseRRR , (0b0001101100000000011111, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 12 ), // #258 + INST(Mvn , BaseMvnNeg , (0b00101010001000000000001111100000) , kRWI_W , 0 , 0 ), // #259 + INST(Neg , BaseMvnNeg , (0b01001011000000000000001111100000) , kRWI_W , 0 , 1 ), // #260 + INST(Negs , BaseMvnNeg , (0b01101011000000000000001111100000) , kRWI_W , 0 , 2 ), // #261 + INST(Ngc , BaseRR , (0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 10 ), // #262 + INST(Ngcs , BaseRR , (0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true) , kRWI_W , 0 , 11 ), // #263 + INST(Nop , BaseOp , (0b11010101000000110010000000011111) , 0 , 0 , 14 ), // #264 + INST(Orn , BaseLogical , (0b0101010001, 0b01100100, 1) , kRWI_W , 0 , 6 ), // #265 + INST(Orr , BaseLogical , (0b0101010000, 0b01100100, 0) , kRWI_W , 0 , 7 ), // #266 + INST(Pacda , BaseRR , (0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 12 ), // #267 + INST(Pacdb , BaseRR , (0b11011010110000010000110000000000, kX, kZR, 0, kX, kSP, 5, true) , kRWI_X , 0 , 13 ), // #268 + INST(Pacdza , BaseR , (0b11011010110000010010101111100000, kX, kZR, 0) , kRWI_X , 0 , 4 ), // #269 + INST(Pacdzb , BaseR , (0b11011010110000010010111111100000, kX, kZR, 0) , kRWI_X , 0 , 5 ), // #270 + INST(Pacga , BaseRRR , (0b1001101011000000001100, kX, kZR, kX, kZR, kX, kSP, false) , kRWI_W , 0 , 13 ), // #271 + INST(Prfm , BasePrfm , (0b11111000101, 0b1111100110, 0b11111000100, 0b11011000) , kRWI_R , 0 , 0 ), // #272 + INST(Pssbb , BaseOp , (0b11010101000000110011010010011111) , 0 , 0 , 15 ), // #273 + INST(Rbit , BaseRR , (0b01011010110000000000000000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 14 ), // #274 + INST(Ret , BaseBranchReg , (0b11010110010111110000000000000000) , kRWI_R , 0 , 2 ), // #275 + INST(Rev , BaseRev , (_) , kRWI_W , 0 , 0 ), // #276 + INST(Rev16 , BaseRR , (0b01011010110000000000010000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 15 ), // #277 + INST(Rev32 , BaseRR , (0b11011010110000000000100000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 16 ), // #278 + INST(Rev64 , BaseRR , (0b11011010110000000000110000000000, kWX, kZR, 0, kWX, kZR, 5, true) , kRWI_W , 0 , 17 ), // #279 + INST(Ror , BaseShift , (0b0001101011000000001011, 0b0001001110000000000000, 1) , kRWI_W , 0 , 6 ), // #280 + INST(Rorv , BaseShift , (0b0001101011000000001011, 0b0000000000000000000000, 1) , kRWI_W , 0 , 7 ), // #281 + INST(Sbc , BaseRRR , (0b0101101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 14 ), // #282 + INST(Sbcs , BaseRRR , (0b0111101000000000000000, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 15 ), // #283 + INST(Sbfiz , BaseBfi , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #284 + INST(Sbfm , BaseBfm , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #285 + INST(Sbfx , BaseBfx , (0b00010011000000000000000000000000) , kRWI_W , 0 , 1 ), // #286 + INST(Sdiv , BaseRRR , (0b0001101011000000000011, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 16 ), // #287 + INST(Setf8 , BaseR , (0b00111010000000000000100000001101, kW, kZR, 5) , 0 , 0 , 6 ), // #288 + INST(Setf16 , BaseR , (0b00111010000000000100100000001101, kW, kZR, 5) , 0 , 0 , 7 ), // #289 + INST(Sev , BaseOp , (0b11010101000000110010000010011111) , 0 , 0 , 16 ), // #290 + INST(Sevl , BaseOp , (0b11010101000000110010000010111111) , 0 , 0 , 17 ), // #291 + INST(Smaddl , BaseRRRR , (0b1001101100100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 2 ), // #292 + INST(Smax , BaseMinMax , (0b00011010110000000110000000000000, 0b00010001110000000000000000000000) , kRWI_W , 0 , 0 ), // #293 + INST(Smc , BaseOpImm , (0b11010100000000000000000000000011, 16, 5) , 0 , 0 , 12 ), // #294 + INST(Smin , BaseMinMax , (0b00011010110000000110100000000000, 0b00010001110010000000000000000000) , kRWI_W , 0 , 1 ), // #295 + INST(Smnegl , BaseRRR , (0b1001101100100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 17 ), // #296 + INST(Smsubl , BaseRRRR , (0b1001101100100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 3 ), // #297 + INST(Smulh , BaseRRR , (0b1001101101000000011111, kX , kZR, kX , kZR, kX , kZR, true) , kRWI_W , 0 , 18 ), // #298 + INST(Smull , BaseRRR , (0b1001101100100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 19 ), // #299 + INST(Ssbb , BaseOp , (0b11010101000000110011000010011111) , 0 , 0 , 18 ), // #300 + INST(St2g , BaseRM_SImm9 , (0b1101100110100000000010, 0b1101100110100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 13 ), // #301 + INST(Stadd , BaseAtomicSt , (0b1011100000100000000000, kWX, 30) , kRWI_RX , 0 , 0 ), // #302 + INST(Staddl , BaseAtomicSt , (0b1011100001100000000000, kWX, 30) , kRWI_RX , 0 , 1 ), // #303 + INST(Staddb , BaseAtomicSt , (0b0011100000100000000000, kW , 0 ) , kRWI_RX , 0 , 2 ), // #304 + INST(Staddlb , BaseAtomicSt , (0b0011100001100000000000, kW , 0 ) , kRWI_RX , 0 , 3 ), // #305 + INST(Staddh , BaseAtomicSt , (0b0111100000100000000000, kW , 0 ) , kRWI_RX , 0 , 4 ), // #306 + INST(Staddlh , BaseAtomicSt , (0b0111100001100000000000, kW , 0 ) , kRWI_RX , 0 , 5 ), // #307 + INST(Stclr , BaseAtomicSt , (0b1011100000100000000100, kWX, 30) , kRWI_RX , 0 , 6 ), // #308 + INST(Stclrl , BaseAtomicSt , (0b1011100001100000000100, kWX, 30) , kRWI_RX , 0 , 7 ), // #309 + INST(Stclrb , BaseAtomicSt , (0b0011100000100000000100, kW , 0 ) , kRWI_RX , 0 , 8 ), // #310 + INST(Stclrlb , BaseAtomicSt , (0b0011100001100000000100, kW , 0 ) , kRWI_RX , 0 , 9 ), // #311 + INST(Stclrh , BaseAtomicSt , (0b0111100000100000000100, kW , 0 ) , kRWI_RX , 0 , 10 ), // #312 + INST(Stclrlh , BaseAtomicSt , (0b0111100001100000000100, kW , 0 ) , kRWI_RX , 0 , 11 ), // #313 + INST(Steor , BaseAtomicSt , (0b1011100000100000001000, kWX, 30) , kRWI_RX , 0 , 12 ), // #314 + INST(Steorl , BaseAtomicSt , (0b1011100001100000001000, kWX, 30) , kRWI_RX , 0 , 13 ), // #315 + INST(Steorb , BaseAtomicSt , (0b0011100000100000001000, kW , 0 ) , kRWI_RX , 0 , 14 ), // #316 + INST(Steorlb , BaseAtomicSt , (0b0011100001100000001000, kW , 0 ) , kRWI_RX , 0 , 15 ), // #317 + INST(Steorh , BaseAtomicSt , (0b0111100000100000001000, kW , 0 ) , kRWI_RX , 0 , 16 ), // #318 + INST(Steorlh , BaseAtomicSt , (0b0111100001100000001000, kW , 0 ) , kRWI_RX , 0 , 17 ), // #319 + INST(Stg , BaseRM_SImm9 , (0b1101100100100000000010, 0b1101100100100000000001, kX, kSP, 0, 4) , kRWI_RW , 0 , 14 ), // #320 + INST(Stgm , BaseRM_NoImm , (0b1101100110100000000000, kX , kZR, 0 ) , kRWI_RW , 0 , 13 ), // #321 + INST(Stgp , BaseLdpStp , (0b0110100100, 0b0110100010, kX, 0, 4) , kRWI_RRW , 0 , 3 ), // #322 + INST(Stllr , BaseRM_NoImm , (0b1000100010011111011111, kWX, kZR, 30) , kRWI_RW , 0 , 14 ), // #323 + INST(Stllrb , BaseRM_NoImm , (0b0000100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 15 ), // #324 + INST(Stllrh , BaseRM_NoImm , (0b0100100010011111011111, kW , kZR, 0 ) , kRWI_RW , 0 , 16 ), // #325 + INST(Stlr , BaseRM_NoImm , (0b1000100010011111111111, kWX, kZR, 30) , kRWI_RW , 0 , 17 ), // #326 + INST(Stlrb , BaseRM_NoImm , (0b0000100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 18 ), // #327 + INST(Stlrh , BaseRM_NoImm , (0b0100100010011111111111, kW , kZR, 0 ) , kRWI_RW , 0 , 19 ), // #328 + INST(Stlxp , BaseStxp , (0b1000100000100000100000, kWX, 30) , kRWI_WRRX , 0 , 0 ), // #329 + INST(Stlxr , BaseAtomicOp , (0b1000100000000000111111, kWX, 30, 1) , kRWI_WRX , 0 , 108), // #330 + INST(Stlxrb , BaseAtomicOp , (0b0000100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 109), // #331 + INST(Stlxrh , BaseAtomicOp , (0b0100100000000000111111, kW , 0 , 1) , kRWI_WRX , 0 , 110), // #332 + INST(Stnp , BaseLdpStp , (0b0010100000, 0 , kWX, 31, 2) , kRWI_RRW , 0 , 4 ), // #333 + INST(Stp , BaseLdpStp , (0b0010100100, 0b0010100010, kWX, 31, 2) , kRWI_RRW , 0 , 5 ), // #334 + INST(Str , BaseLdSt , (0b1011100100, 0b10111000000, 0b10111000001, 0 , kWX, 30, 2, Inst::kIdStur) , kRWI_RW , 0 , 6 ), // #335 + INST(Strb , BaseLdSt , (0b0011100100, 0b00111000000, 0b00111000001, 0 , kW , 30, 0, Inst::kIdSturb) , kRWI_RW , 0 , 7 ), // #336 + INST(Strh , BaseLdSt , (0b0111100100, 0b01111000000, 0b01111000001, 0 , kWX, 30, 1, Inst::kIdSturh) , kRWI_RW , 0 , 8 ), // #337 + INST(Stset , BaseAtomicSt , (0b1011100000100000001100, kWX, 30) , kRWI_RX , 0 , 18 ), // #338 + INST(Stsetl , BaseAtomicSt , (0b1011100001100000001100, kWX, 30) , kRWI_RX , 0 , 19 ), // #339 + INST(Stsetb , BaseAtomicSt , (0b0011100000100000001100, kW , 0 ) , kRWI_RX , 0 , 20 ), // #340 + INST(Stsetlb , BaseAtomicSt , (0b0011100001100000001100, kW , 0 ) , kRWI_RX , 0 , 21 ), // #341 + INST(Stseth , BaseAtomicSt , (0b0111100000100000001100, kW , 0 ) , kRWI_RX , 0 , 22 ), // #342 + INST(Stsetlh , BaseAtomicSt , (0b0111100001100000001100, kW , 0 ) , kRWI_RX , 0 , 23 ), // #343 + INST(Stsmax , BaseAtomicSt , (0b1011100000100000010000, kWX, 30) , kRWI_RX , 0 , 24 ), // #344 + INST(Stsmaxl , BaseAtomicSt , (0b1011100001100000010000, kWX, 30) , kRWI_RX , 0 , 25 ), // #345 + INST(Stsmaxb , BaseAtomicSt , (0b0011100000100000010000, kW , 0 ) , kRWI_RX , 0 , 26 ), // #346 + INST(Stsmaxlb , BaseAtomicSt , (0b0011100001100000010000, kW , 0 ) , kRWI_RX , 0 , 27 ), // #347 + INST(Stsmaxh , BaseAtomicSt , (0b0111100000100000010000, kW , 0 ) , kRWI_RX , 0 , 28 ), // #348 + INST(Stsmaxlh , BaseAtomicSt , (0b0111100001100000010000, kW , 0 ) , kRWI_RX , 0 , 29 ), // #349 + INST(Stsmin , BaseAtomicSt , (0b1011100000100000010100, kWX, 30) , kRWI_RX , 0 , 30 ), // #350 + INST(Stsminl , BaseAtomicSt , (0b1011100001100000010100, kWX, 30) , kRWI_RX , 0 , 31 ), // #351 + INST(Stsminb , BaseAtomicSt , (0b0011100000100000010100, kW , 0 ) , kRWI_RX , 0 , 32 ), // #352 + INST(Stsminlb , BaseAtomicSt , (0b0011100001100000010100, kW , 0 ) , kRWI_RX , 0 , 33 ), // #353 + INST(Stsminh , BaseAtomicSt , (0b0111100000100000010100, kW , 0 ) , kRWI_RX , 0 , 34 ), // #354 + INST(Stsminlh , BaseAtomicSt , (0b0111100001100000010100, kW , 0 ) , kRWI_RX , 0 , 35 ), // #355 + INST(Sttr , BaseRM_SImm9 , (0b1011100000000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 15 ), // #356 + INST(Sttrb , BaseRM_SImm9 , (0b0011100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 16 ), // #357 + INST(Sttrh , BaseRM_SImm9 , (0b0111100000000000000010, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 17 ), // #358 + INST(Stumax , BaseAtomicSt , (0b1011100000100000011000, kWX, 30) , kRWI_RX , 0 , 36 ), // #359 + INST(Stumaxl , BaseAtomicSt , (0b1011100001100000011000, kWX, 30) , kRWI_RX , 0 , 37 ), // #360 + INST(Stumaxb , BaseAtomicSt , (0b0011100000100000011000, kW , 0 ) , kRWI_RX , 0 , 38 ), // #361 + INST(Stumaxlb , BaseAtomicSt , (0b0011100001100000011000, kW , 0 ) , kRWI_RX , 0 , 39 ), // #362 + INST(Stumaxh , BaseAtomicSt , (0b0111100000100000011000, kW , 0 ) , kRWI_RX , 0 , 40 ), // #363 + INST(Stumaxlh , BaseAtomicSt , (0b0111100001100000011000, kW , 0 ) , kRWI_RX , 0 , 41 ), // #364 + INST(Stumin , BaseAtomicSt , (0b1011100000100000011100, kWX, 30) , kRWI_RX , 0 , 42 ), // #365 + INST(Stuminl , BaseAtomicSt , (0b1011100001100000011100, kWX, 30) , kRWI_RX , 0 , 43 ), // #366 + INST(Stuminb , BaseAtomicSt , (0b0011100000100000011100, kW , 0 ) , kRWI_RX , 0 , 44 ), // #367 + INST(Stuminlb , BaseAtomicSt , (0b0011100001100000011100, kW , 0 ) , kRWI_RX , 0 , 45 ), // #368 + INST(Stuminh , BaseAtomicSt , (0b0111100000100000011100, kW , 0 ) , kRWI_RX , 0 , 46 ), // #369 + INST(Stuminlh , BaseAtomicSt , (0b0111100001100000011100, kW , 0 ) , kRWI_RX , 0 , 47 ), // #370 + INST(Stur , BaseRM_SImm9 , (0b1011100000000000000000, 0b0000000000000000000000, kWX, kZR, 30, 0) , kRWI_RW , 0 , 18 ), // #371 + INST(Sturb , BaseRM_SImm9 , (0b0011100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 19 ), // #372 + INST(Sturh , BaseRM_SImm9 , (0b0111100000000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0) , kRWI_RW , 0 , 20 ), // #373 + INST(Stxp , BaseStxp , (0b1000100000100000000000, kWX, 30) , kRWI_WRRW , 0 , 1 ), // #374 + INST(Stxr , BaseStx , (0b1000100000000000011111, kWX, 30) , kRWI_WRW , 0 , 0 ), // #375 + INST(Stxrb , BaseStx , (0b0000100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 1 ), // #376 + INST(Stxrh , BaseStx , (0b0100100000000000011111, kW , 0 ) , kRWI_WRW , 0 , 2 ), // #377 + INST(Stz2g , BaseRM_SImm9 , (0b1101100111100000000010, 0b1101100111100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 21 ), // #378 + INST(Stzg , BaseRM_SImm9 , (0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4) , kRWI_RW , 0 , 22 ), // #379 + INST(Stzgm , BaseRM_NoImm , (0b1101100100100000000000, kX , kZR, 0) , kRWI_RW , 0 , 20 ), // #380 + INST(Sub , BaseAddSub , (0b1001011000, 0b1001011001, 0b1010001) , kRWI_W , 0 , 2 ), // #381 + INST(Subg , BaseRRII , (0b1101000110000000000000, kX, kSP, kX, kSP, 6, 4, 16, 4, 0, 10) , kRWI_W , 0 , 1 ), // #382 + INST(Subp , BaseRRR , (0b1001101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 20 ), // #383 + INST(Subps , BaseRRR , (0b1011101011000000000000, kX, kZR, kX, kSP, kX, kSP, false) , kRWI_W , 0 , 21 ), // #384 + INST(Subs , BaseAddSub , (0b1101011000, 0b1101011001, 0b1110001) , kRWI_W , 0 , 3 ), // #385 + INST(Svc , BaseOpImm , (0b11010100000000000000000000000001, 16, 5) , 0 , 0 , 13 ), // #386 + INST(Swp , BaseAtomicOp , (0b1011100000100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 111), // #387 + INST(Swpa , BaseAtomicOp , (0b1011100010100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 112), // #388 + INST(Swpab , BaseAtomicOp , (0b0011100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 113), // #389 + INST(Swpah , BaseAtomicOp , (0b0111100010100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 114), // #390 + INST(Swpal , BaseAtomicOp , (0b1011100011100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 115), // #391 + INST(Swpalb , BaseAtomicOp , (0b0011100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 116), // #392 + INST(Swpalh , BaseAtomicOp , (0b0111100011100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 117), // #393 + INST(Swpb , BaseAtomicOp , (0b0011100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 118), // #394 + INST(Swph , BaseAtomicOp , (0b0111100000100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 119), // #395 + INST(Swpl , BaseAtomicOp , (0b1011100001100000100000, kWX, 30, 1) , kRWI_RWX , 0 , 120), // #396 + INST(Swplb , BaseAtomicOp , (0b0011100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 121), // #397 + INST(Swplh , BaseAtomicOp , (0b0111100001100000100000, kW , 0 , 1) , kRWI_RWX , 0 , 122), // #398 + INST(Sxtb , BaseExtend , (0b0001001100000000000111, kWX, 0) , kRWI_W , 0 , 0 ), // #399 + INST(Sxth , BaseExtend , (0b0001001100000000001111, kWX, 0) , kRWI_W , 0 , 1 ), // #400 + INST(Sxtw , BaseExtend , (0b1001001101000000011111, kX , 0) , kRWI_W , 0 , 2 ), // #401 + INST(Sys , BaseSys , (_) , kRWI_W , 0 , 0 ), // #402 + INST(Tlbi , BaseAtDcIcTlbi , (0b00011110000000, 0b00010000000000, false) , kRWI_RX , 0 , 3 ), // #403 + INST(Tst , BaseTst , (0b1101010000, 0b111001000) , kRWI_R , 0 , 0 ), // #404 + INST(Tbnz , BaseBranchTst , (0b00110111000000000000000000000000) , kRWI_R , 0 , 0 ), // #405 + INST(Tbz , BaseBranchTst , (0b00110110000000000000000000000000) , kRWI_R , 0 , 1 ), // #406 + INST(Ubfiz , BaseBfi , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #407 + INST(Ubfm , BaseBfm , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #408 + INST(Ubfx , BaseBfx , (0b01010011000000000000000000000000) , kRWI_W , 0 , 2 ), // #409 + INST(Udf , BaseOpImm , (0b00000000000000000000000000000000, 16, 0) , 0 , 0 , 14 ), // #410 + INST(Udiv , BaseRRR , (0b0001101011000000000010, kWX, kZR, kWX, kZR, kWX, kZR, true) , kRWI_W , 0 , 22 ), // #411 + INST(Umaddl , BaseRRRR , (0b1001101110100000000000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 4 ), // #412 + INST(Umax , BaseMinMax , (0b00011010110000000110010000000000, 0b00010001110001000000000000000000) , kRWI_W , 0 , 2 ), // #413 + INST(Umin , BaseMinMax , (0b00011010110000000110110000000000, 0b00010001110011000000000000000000) , kRWI_W , 0 , 3 ), // #414 + INST(Umnegl , BaseRRR , (0b1001101110100000111111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 23 ), // #415 + INST(Umull , BaseRRR , (0b1001101110100000011111, kX , kZR, kW , kZR, kW , kZR, false) , kRWI_W , 0 , 24 ), // #416 + INST(Umulh , BaseRRR , (0b1001101111000000011111, kX , kZR, kX , kZR, kX , kZR, false) , kRWI_W , 0 , 25 ), // #417 + INST(Umsubl , BaseRRRR , (0b1001101110100000100000, kX , kZR, kW , kZR, kW , kZR, kX , kZR, false) , kRWI_W , 0 , 5 ), // #418 + INST(Uxtb , BaseExtend , (0b0101001100000000000111, kW, 1) , kRWI_W , 0 , 3 ), // #419 + INST(Uxth , BaseExtend , (0b0101001100000000001111, kW, 1) , kRWI_W , 0 , 4 ), // #420 + INST(Wfe , BaseOp , (0b11010101000000110010000001011111) , 0 , 0 , 19 ), // #421 + INST(Wfi , BaseOp , (0b11010101000000110010000001111111) , 0 , 0 , 20 ), // #422 + INST(Xaflag , BaseOp , (0b11010101000000000100000000111111) , 0 , 0 , 21 ), // #423 + INST(Xpacd , BaseR , (0b11011010110000010100011111100000, kX, kZR, 0) , kRWI_X , 0 , 8 ), // #424 + INST(Xpaci , BaseR , (0b11011010110000010100001111100000, kX, kZR, 0) , kRWI_X , 0 , 9 ), // #425 + INST(Xpaclri , BaseOp , (0b11010101000000110010000011111111) , kRWI_X , 0 , 22 ), // #426 + INST(Yield , BaseOp , (0b11010101000000110010000000111111) , 0 , 0 , 23 ), // #427 + INST(Abs_v , ISimdVV , (0b0000111000100000101110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #428 + INST(Add_v , ISimdVVV , (0b0000111000100000100001, kVO_V_Any) , kRWI_W , 0 , 0 ), // #429 + INST(Addhn_v , ISimdVVV , (0b0000111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 1 ), // #430 + INST(Addhn2_v , ISimdVVV , (0b0100111000100000010000, kVO_V_B16H8S4) , kRWI_W , F(Narrow) , 2 ), // #431 + INST(Addp_v , ISimdPair , (0b0101111000110001101110, 0b0000111000100000101111, kVO_V_Any) , kRWI_W , F(Pair) , 0 ), // #432 + INST(Addv_v , ISimdSV , (0b0000111000110001101110, kVO_V_BH_4S) , kRWI_W , 0 , 0 ), // #433 + INST(Aesd_v , ISimdVVx , (0b0100111000101000010110, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 0 ), // #434 + INST(Aese_v , ISimdVVx , (0b0100111000101000010010, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 1 ), // #435 + INST(Aesimc_v , ISimdVVx , (0b0100111000101000011110, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 2 ), // #436 + INST(Aesmc_v , ISimdVVx , (0b0100111000101000011010, kOp_V16B, kOp_V16B) , kRWI_W , 0 , 3 ), // #437 + INST(And_v , ISimdVVV , (0b0000111000100000000111, kVO_V_B) , kRWI_W , 0 , 3 ), // #438 + INST(Bcax_v , ISimdVVVV , (0b1100111000100000000000, kVO_V_B16) , kRWI_W , 0 , 0 ), // #439 + INST(Bfcvt_v , ISimdVVx , (0b0001111001100011010000, kOp_H, kOp_S) , kRWI_W , 0 , 4 ), // #440 + INST(Bfcvtn_v , ISimdVVx , (0b0000111010100001011010, kOp_V4H, kOp_V4S) , kRWI_W , F(Narrow) , 5 ), // #441 + INST(Bfcvtn2_v , ISimdVVx , (0b0100111010100001011010, kOp_V8H, kOp_V4S) , kRWI_W , F(Narrow) , 6 ), // #442 + INST(Bfdot_v , SimdDot , (0b0010111001000000111111, 0b0000111101000000111100, kET_S, kET_H, kET_2H) , kRWI_X , 0 , 0 ), // #443 + INST(Bfmlalb_v , SimdFmlal , (0b0010111011000000111111, 0b0000111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 0 ), // #444 + INST(Bfmlalt_v , SimdFmlal , (0b0110111011000000111111, 0b0100111111000000111100, 0, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 1 ), // #445 + INST(Bfmmla_v , ISimdVVVx , (0b0110111001000000111011, kOp_V4S, kOp_V8H, kOp_V8H) , kRWI_X , F(Long) , 0 ), // #446 + INST(Bic_v , SimdBicOrr , (0b0000111001100000000111, 0b0010111100000000000001) , kRWI_W , 0 , 0 ), // #447 + INST(Bif_v , ISimdVVV , (0b0010111011100000000111, kVO_V_B) , kRWI_X , 0 , 4 ), // #448 + INST(Bit_v , ISimdVVV , (0b0010111010100000000111, kVO_V_B) , kRWI_X , 0 , 5 ), // #449 + INST(Bsl_v , ISimdVVV , (0b0010111001100000000111, kVO_V_B) , kRWI_X , 0 , 6 ), // #450 + INST(Cls_v , ISimdVV , (0b0000111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 1 ), // #451 + INST(Clz_v , ISimdVV , (0b0010111000100000010010, kVO_V_BHS) , kRWI_W , 0 , 2 ), // #452 + INST(Cmeq_v , SimdCmp , (0b0010111000100000100011, 0b0000111000100000100110, kVO_V_Any) , kRWI_W , 0 , 0 ), // #453 + INST(Cmge_v , SimdCmp , (0b0000111000100000001111, 0b0010111000100000100010, kVO_V_Any) , kRWI_W , 0 , 1 ), // #454 + INST(Cmgt_v , SimdCmp , (0b0000111000100000001101, 0b0000111000100000100010, kVO_V_Any) , kRWI_W , 0 , 2 ), // #455 + INST(Cmhi_v , SimdCmp , (0b0010111000100000001101, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 3 ), // #456 + INST(Cmhs_v , SimdCmp , (0b0010111000100000001111, 0b0000000000000000000000, kVO_V_Any) , kRWI_W , 0 , 4 ), // #457 + INST(Cmle_v , SimdCmp , (0b0000000000000000000000, 0b0010111000100000100110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #458 + INST(Cmlt_v , SimdCmp , (0b0000000000000000000000, 0b0000111000100000101010, kVO_V_Any) , kRWI_W , 0 , 6 ), // #459 + INST(Cmtst_v , ISimdVVV , (0b0000111000100000100011, kVO_V_Any) , kRWI_W , 0 , 7 ), // #460 + INST(Cnt_v , ISimdVV , (0b0000111000100000010110, kVO_V_B) , kRWI_W , 0 , 3 ), // #461 + INST(Dup_v , SimdDup , (_) , kRWI_W , 0 , 0 ), // #462 + INST(Eor_v , ISimdVVV , (0b0010111000100000000111, kVO_V_B) , kRWI_W , 0 , 8 ), // #463 + INST(Eor3_v , ISimdVVVV , (0b1100111000000000000000, kVO_V_B16) , kRWI_W , 0 , 1 ), // #464 + INST(Ext_v , ISimdVVVI , (0b0010111000000000000000, kVO_V_B, 4, 11, 1) , kRWI_W , 0 , 0 ), // #465 + INST(Fabd_v , FSimdVVV , (0b0111111010100000110101, kHF_C, 0b0010111010100000110101, kHF_C) , kRWI_W , 0 , 0 ), // #466 + INST(Fabs_v , FSimdVV , (0b0001111000100000110000, kHF_A, 0b0000111010100000111110, kHF_B) , kRWI_W , 0 , 0 ), // #467 + INST(Facge_v , FSimdVVV , (0b0111111000100000111011, kHF_C, 0b0010111000100000111011, kHF_C) , kRWI_W , 0 , 1 ), // #468 + INST(Facgt_v , FSimdVVV , (0b0111111010100000111011, kHF_C, 0b0010111010100000111011, kHF_C) , kRWI_W , 0 , 2 ), // #469 + INST(Fadd_v , FSimdVVV , (0b0001111000100000001010, kHF_A, 0b0000111000100000110101, kHF_C) , kRWI_W , 0 , 3 ), // #470 + INST(Faddp_v , FSimdPair , (0b0111111000110000110110, 0b0010111000100000110101) , kRWI_W , 0 , 0 ), // #471 + INST(Fcadd_v , SimdFcadd , (0b0010111000000000111001) , kRWI_W , 0 , 0 ), // #472 + INST(Fccmp_v , SimdFccmpFccmpe , (0b00011110001000000000010000000000) , kRWI_R , 0 , 0 ), // #473 + INST(Fccmpe_v , SimdFccmpFccmpe , (0b00011110001000000000010000010000) , kRWI_R , 0 , 1 ), // #474 + INST(Fcmeq_v , SimdFcm , (0b0000111000100000111001, kHF_C, 0b0000111010100000110110) , kRWI_W , 0 , 0 ), // #475 + INST(Fcmge_v , SimdFcm , (0b0010111000100000111001, kHF_C, 0b0010111010100000110010) , kRWI_W , 0 , 1 ), // #476 + INST(Fcmgt_v , SimdFcm , (0b0010111010100000111001, kHF_C, 0b0000111010100000110010) , kRWI_W , 0 , 2 ), // #477 + INST(Fcmla_v , SimdFcmla , (0b0010111000000000110001, 0b0010111100000000000100) , kRWI_X , 0 , 0 ), // #478 + INST(Fcmle_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0010111010100000110110) , kRWI_W , 0 , 3 ), // #479 + INST(Fcmlt_v , SimdFcm , (0b0000000000000000000000, kHF_C, 0b0000111010100000111010) , kRWI_W , 0 , 4 ), // #480 + INST(Fcmp_v , SimdFcmpFcmpe , (0b00011110001000000010000000000000) , kRWI_R , 0 , 0 ), // #481 + INST(Fcmpe_v , SimdFcmpFcmpe , (0b00011110001000000010000000010000) , kRWI_R , 0 , 1 ), // #482 + INST(Fcsel_v , SimdFcsel , (_) , kRWI_W , 0 , 0 ), // #483 + INST(Fcvt_v , SimdFcvt , (_) , kRWI_W , 0 , 0 ), // #484 + INST(Fcvtas_v , SimdFcvtSV , (0b0000111000100001110010, 0b0000000000000000000000, 0b0001111000100100000000, 1) , kRWI_W , 0 , 0 ), // #485 + INST(Fcvtau_v , SimdFcvtSV , (0b0010111000100001110010, 0b0000000000000000000000, 0b0001111000100101000000, 1) , kRWI_W , 0 , 1 ), // #486 + INST(Fcvtl_v , SimdFcvtLN , (0b0000111000100001011110, 0, 0) , kRWI_W , F(Long) , 0 ), // #487 + INST(Fcvtl2_v , SimdFcvtLN , (0b0100111000100001011110, 0, 0) , kRWI_W , F(Long) , 1 ), // #488 + INST(Fcvtms_v , SimdFcvtSV , (0b0000111000100001101110, 0b0000000000000000000000, 0b0001111000110000000000, 1) , kRWI_W , 0 , 2 ), // #489 + INST(Fcvtmu_v , SimdFcvtSV , (0b0010111000100001101110, 0b0000000000000000000000, 0b0001111000110001000000, 1) , kRWI_W , 0 , 3 ), // #490 + INST(Fcvtn_v , SimdFcvtLN , (0b0000111000100001011010, 0, 0) , kRWI_W , F(Narrow) , 2 ), // #491 + INST(Fcvtn2_v , SimdFcvtLN , (0b0100111000100001011010, 0, 0) , kRWI_X , F(Narrow) , 3 ), // #492 + INST(Fcvtns_v , SimdFcvtSV , (0b0000111000100001101010, 0b0000000000000000000000, 0b0001111000100000000000, 1) , kRWI_W , 0 , 4 ), // #493 + INST(Fcvtnu_v , SimdFcvtSV , (0b0010111000100001101010, 0b0000000000000000000000, 0b0001111000100001000000, 1) , kRWI_W , 0 , 5 ), // #494 + INST(Fcvtps_v , SimdFcvtSV , (0b0000111010100001101010, 0b0000000000000000000000, 0b0001111000101000000000, 1) , kRWI_W , 0 , 6 ), // #495 + INST(Fcvtpu_v , SimdFcvtSV , (0b0010111010100001101010, 0b0000000000000000000000, 0b0001111000101001000000, 1) , kRWI_W , 0 , 7 ), // #496 + INST(Fcvtxn_v , SimdFcvtLN , (0b0010111000100001011010, 1, 1) , kRWI_W , F(Narrow) , 4 ), // #497 + INST(Fcvtxn2_v , SimdFcvtLN , (0b0110111000100001011010, 1, 0) , kRWI_X , F(Narrow) , 5 ), // #498 + INST(Fcvtzs_v , SimdFcvtSV , (0b0000111010100001101110, 0b0000111100000000111111, 0b0001111000111000000000, 1) , kRWI_W , 0 , 8 ), // #499 + INST(Fcvtzu_v , SimdFcvtSV , (0b0010111010100001101110, 0b0010111100000000111111, 0b0001111000111001000000, 1) , kRWI_W , 0 , 9 ), // #500 + INST(Fdiv_v , FSimdVVV , (0b0001111000100000000110, kHF_A, 0b0010111000100000111111, kHF_C) , kRWI_W , 0 , 4 ), // #501 + INST(Fjcvtzs_v , ISimdVVx , (0b0001111001111110000000, kOp_GpW, kOp_D) , kRWI_W , 0 , 7 ), // #502 + INST(Fmadd_v , FSimdVVVV , (0b0001111100000000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 0 ), // #503 + INST(Fmax_v , FSimdVVV , (0b0001111000100000010010, kHF_A, 0b0000111000100000111101, kHF_C) , kRWI_W , 0 , 5 ), // #504 + INST(Fmaxnm_v , FSimdVVV , (0b0001111000100000011010, kHF_A, 0b0000111000100000110001, kHF_C) , kRWI_W , 0 , 6 ), // #505 + INST(Fmaxnmp_v , FSimdPair , (0b0111111000110000110010, 0b0010111000100000110001) , kRWI_W , 0 , 1 ), // #506 + INST(Fmaxnmv_v , FSimdSV , (0b0010111000110000110010) , kRWI_W , 0 , 0 ), // #507 + INST(Fmaxp_v , FSimdPair , (0b0111111000110000111110, 0b0010111000100000111101) , kRWI_W , 0 , 2 ), // #508 + INST(Fmaxv_v , FSimdSV , (0b0010111000110000111110) , kRWI_W , 0 , 1 ), // #509 + INST(Fmin_v , FSimdVVV , (0b0001111000100000010110, kHF_A, 0b0000111010100000111101, kHF_C) , kRWI_W , 0 , 7 ), // #510 + INST(Fminnm_v , FSimdVVV , (0b0001111000100000011110, kHF_A, 0b0000111010100000110001, kHF_C) , kRWI_W , 0 , 8 ), // #511 + INST(Fminnmp_v , FSimdPair , (0b0111111010110000110010, 0b0010111010100000110001) , kRWI_W , 0 , 3 ), // #512 + INST(Fminnmv_v , FSimdSV , (0b0010111010110000110010) , kRWI_W , 0 , 2 ), // #513 + INST(Fminp_v , FSimdPair , (0b0111111010110000111110, 0b0010111010100000111101) , kRWI_W , 0 , 4 ), // #514 + INST(Fminv_v , FSimdSV , (0b0010111010110000111110) , kRWI_W , 0 , 3 ), // #515 + INST(Fmla_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111000100000110011, 0b0000111110000000000100) , kRWI_X , F(VH0_15) , 0 ), // #516 + INST(Fmlal_v , SimdFmlal , (0b0000111000100000111011, 0b0000111110000000000000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 2 ), // #517 + INST(Fmlal2_v , SimdFmlal , (0b0010111000100000110011, 0b0010111110000000100000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 3 ), // #518 + INST(Fmls_v , FSimdVVVe , (0b0000000000000000000000, kHF_N, 0b0000111010100000110011, 0b0000111110000000010100) , kRWI_X , F(VH0_15) , 1 ), // #519 + INST(Fmlsl_v , SimdFmlal , (0b0000111010100000111011, 0b0000111110000000010000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 4 ), // #520 + INST(Fmlsl2_v , SimdFmlal , (0b0010111010100000110011, 0b0010111110000000110000, 1, kET_S, kET_H, kET_H) , kRWI_X , F(VH0_15) , 5 ), // #521 + INST(Fmov_v , SimdFmov , (_) , kRWI_W , 0 , 0 ), // #522 + INST(Fmsub_v , FSimdVVVV , (0b0001111100000000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 1 ), // #523 + INST(Fmul_v , FSimdVVVe , (0b0001111000100000000010, kHF_A, 0b0010111000100000110111, 0b0000111110000000100100) , kRWI_W , F(VH0_15) , 2 ), // #524 + INST(Fmulx_v , FSimdVVVe , (0b0101111000100000110111, kHF_C, 0b0000111000100000110111, 0b0010111110000000100100) , kRWI_W , F(VH0_15) , 3 ), // #525 + INST(Fneg_v , FSimdVV , (0b0001111000100001010000, kHF_A, 0b0010111010100000111110, kHF_B) , kRWI_W , 0 , 1 ), // #526 + INST(Fnmadd_v , FSimdVVVV , (0b0001111100100000000000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 2 ), // #527 + INST(Fnmsub_v , FSimdVVVV , (0b0001111100100000100000, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #528 + INST(Fnmul_v , FSimdVVV , (0b0001111000100000100010, kHF_A, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 9 ), // #529 + INST(Frecpe_v , FSimdVV , (0b0101111010100001110110, kHF_B, 0b0000111010100001110110, kHF_B) , kRWI_W , 0 , 2 ), // #530 + INST(Frecps_v , FSimdVVV , (0b0101111000100000111111, kHF_C, 0b0000111000100000111111, kHF_C) , kRWI_W , 0 , 10 ), // #531 + INST(Frecpx_v , FSimdVV , (0b0101111010100001111110, kHF_B, 0b0000000000000000000000, kHF_N) , kRWI_W , 0 , 3 ), // #532 + INST(Frint32x_v , FSimdVV , (0b0001111000101000110000, kHF_N, 0b0010111000100001111010, kHF_N) , kRWI_W , 0 , 4 ), // #533 + INST(Frint32z_v , FSimdVV , (0b0001111000101000010000, kHF_N, 0b0000111000100001111010, kHF_N) , kRWI_W , 0 , 5 ), // #534 + INST(Frint64x_v , FSimdVV , (0b0001111000101001110000, kHF_N, 0b0010111000100001111110, kHF_N) , kRWI_W , 0 , 6 ), // #535 + INST(Frint64z_v , FSimdVV , (0b0001111000101001010000, kHF_N, 0b0000111000100001111110, kHF_N) , kRWI_W , 0 , 7 ), // #536 + INST(Frinta_v , FSimdVV , (0b0001111000100110010000, kHF_A, 0b0010111000100001100010, kHF_B) , kRWI_W , 0 , 8 ), // #537 + INST(Frinti_v , FSimdVV , (0b0001111000100111110000, kHF_A, 0b0010111010100001100110, kHF_B) , kRWI_W , 0 , 9 ), // #538 + INST(Frintm_v , FSimdVV , (0b0001111000100101010000, kHF_A, 0b0000111000100001100110, kHF_B) , kRWI_W , 0 , 10 ), // #539 + INST(Frintn_v , FSimdVV , (0b0001111000100100010000, kHF_A, 0b0000111000100001100010, kHF_B) , kRWI_W , 0 , 11 ), // #540 + INST(Frintp_v , FSimdVV , (0b0001111000100100110000, kHF_A, 0b0000111010100001100010, kHF_B) , kRWI_W , 0 , 12 ), // #541 + INST(Frintx_v , FSimdVV , (0b0001111000100111010000, kHF_A, 0b0010111000100001100110, kHF_B) , kRWI_W , 0 , 13 ), // #542 + INST(Frintz_v , FSimdVV , (0b0001111000100101110000, kHF_A, 0b0000111010100001100110, kHF_B) , kRWI_W , 0 , 14 ), // #543 + INST(Frsqrte_v , FSimdVV , (0b0111111010100001110110, kHF_B, 0b0010111010100001110110, kHF_B) , kRWI_W , 0 , 15 ), // #544 + INST(Frsqrts_v , FSimdVVV , (0b0101111010100000111111, kHF_C, 0b0000111010100000111111, kHF_C) , kRWI_W , 0 , 11 ), // #545 + INST(Fsqrt_v , FSimdVV , (0b0001111000100001110000, kHF_A, 0b0010111010100001111110, kHF_B) , kRWI_W , 0 , 16 ), // #546 + INST(Fsub_v , FSimdVVV , (0b0001111000100000001110, kHF_A, 0b0000111010100000110101, kHF_C) , kRWI_W , 0 , 12 ), // #547 + INST(Ins_v , SimdIns , (_) , kRWI_X , 0 , 0 ), // #548 + INST(Ld1_v , SimdLdNStN , (0b0000110101000000000000, 0b0000110001000000001000, 1, 0) , kRWI_LDn , F(Consecutive) , 0 ), // #549 + INST(Ld1r_v , SimdLdNStN , (0b0000110101000000110000, 0b0000000000000000000000, 1, 1) , kRWI_LDn , F(Consecutive) , 1 ), // #550 + INST(Ld2_v , SimdLdNStN , (0b0000110101100000000000, 0b0000110001000000100000, 2, 0) , kRWI_LDn , F(Consecutive) , 2 ), // #551 + INST(Ld2r_v , SimdLdNStN , (0b0000110101100000110000, 0b0000000000000000000000, 2, 1) , kRWI_LDn , F(Consecutive) , 3 ), // #552 + INST(Ld3_v , SimdLdNStN , (0b0000110101000000001000, 0b0000110001000000010000, 3, 0) , kRWI_LDn , F(Consecutive) , 4 ), // #553 + INST(Ld3r_v , SimdLdNStN , (0b0000110101000000111000, 0b0000000000000000000000, 3, 1) , kRWI_LDn , F(Consecutive) , 5 ), // #554 + INST(Ld4_v , SimdLdNStN , (0b0000110101100000001000, 0b0000110001000000000000, 4, 0) , kRWI_LDn , F(Consecutive) , 6 ), // #555 + INST(Ld4r_v , SimdLdNStN , (0b0000110101100000111000, 0b0000000000000000000000, 4, 1) , kRWI_LDn , F(Consecutive) , 7 ), // #556 + INST(Ldnp_v , SimdLdpStp , (0b0010110001, 0b0000000000) , kRWI_WW , 0 , 0 ), // #557 + INST(Ldp_v , SimdLdpStp , (0b0010110101, 0b0010110011) , kRWI_WW , 0 , 1 ), // #558 + INST(Ldr_v , SimdLdSt , (0b0011110101, 0b00111100010, 0b00111100011, 0b00011100, Inst::kIdLdur_v) , kRWI_W , 0 , 0 ), // #559 + INST(Ldur_v , SimdLdurStur , (0b0011110001000000000000) , kRWI_W , 0 , 0 ), // #560 + INST(Mla_v , ISimdVVVe , (0b0000111000100000100101, kVO_V_BHS, 0b0010111100000000000000, kVO_V_HS) , kRWI_X , F(VH0_15) , 0 ), // #561 + INST(Mls_v , ISimdVVVe , (0b0010111000100000100101, kVO_V_BHS, 0b0010111100000000010000, kVO_V_HS) , kRWI_X , F(VH0_15) , 1 ), // #562 + INST(Mov_v , SimdMov , (_) , kRWI_W , 0 , 0 ), // #563 + INST(Movi_v , SimdMoviMvni , (0b0000111100000000000001, 0) , kRWI_W , 0 , 0 ), // #564 + INST(Mul_v , ISimdVVVe , (0b0000111000100000100111, kVO_V_BHS, 0b0000111100000000100000, kVO_V_HS) , kRWI_W , F(VH0_15) , 2 ), // #565 + INST(Mvn_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 4 ), // #566 + INST(Mvni_v , SimdMoviMvni , (0b0000111100000000000001, 1) , kRWI_W , 0 , 1 ), // #567 + INST(Neg_v , ISimdVV , (0b0010111000100000101110, kVO_V_Any) , kRWI_W , 0 , 5 ), // #568 + INST(Not_v , ISimdVV , (0b0010111000100000010110, kVO_V_B) , kRWI_W , 0 , 6 ), // #569 + INST(Orn_v , ISimdVVV , (0b0000111011100000000111, kVO_V_B) , kRWI_W , 0 , 9 ), // #570 + INST(Orr_v , SimdBicOrr , (0b0000111010100000000111, 0b0000111100000000000001) , kRWI_W , 0 , 1 ), // #571 + INST(Pmul_v , ISimdVVV , (0b0010111000100000100111, kVO_V_B) , kRWI_W , 0 , 10 ), // #572 + INST(Pmull_v , ISimdVVV , (0b0000111000100000111000, kVO_V_B8D1) , kRWI_W , F(Long) , 11 ), // #573 + INST(Pmull2_v , ISimdVVV , (0b0100111000100000111000, kVO_V_B16D2) , kRWI_W , F(Long) , 12 ), // #574 + INST(Raddhn_v , ISimdVVV , (0b0010111000100000010000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #575 + INST(Raddhn2_v , ISimdVVV , (0b0110111000100000010000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #576 + INST(Rax1_v , ISimdVVV , (0b1100111001100000100011, kVO_V_D2) , kRWI_W , 0 , 15 ), // #577 + INST(Rbit_v , ISimdVV , (0b0010111001100000010110, kVO_V_B) , kRWI_W , 0 , 7 ), // #578 + INST(Rev16_v , ISimdVV , (0b0000111000100000000110, kVO_V_B) , kRWI_W , 0 , 8 ), // #579 + INST(Rev32_v , ISimdVV , (0b0010111000100000000010, kVO_V_BH) , kRWI_W , 0 , 9 ), // #580 + INST(Rev64_v , ISimdVV , (0b0000111000100000000010, kVO_V_BHS) , kRWI_W , 0 , 10 ), // #581 + INST(Rshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100011, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 0 ), // #582 + INST(Rshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 1 ), // #583 + INST(Rsubhn_v , ISimdVVV , (0b0010111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 16 ), // #584 + INST(Rsubhn2_v , ISimdVVV , (0b0110111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 17 ), // #585 + INST(Saba_v , ISimdVVV , (0b0000111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 18 ), // #586 + INST(Sabal_v , ISimdVVV , (0b0000111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 19 ), // #587 + INST(Sabal2_v , ISimdVVV , (0b0100111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 20 ), // #588 + INST(Sabd_v , ISimdVVV , (0b0000111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 21 ), // #589 + INST(Sabdl_v , ISimdVVV , (0b0000111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #590 + INST(Sabdl2_v , ISimdVVV , (0b0100111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #591 + INST(Sadalp_v , ISimdVV , (0b0000111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 11 ), // #592 + INST(Saddl_v , ISimdVVV , (0b0000111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 24 ), // #593 + INST(Saddl2_v , ISimdVVV , (0b0100111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 25 ), // #594 + INST(Saddlp_v , ISimdVV , (0b0000111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 12 ), // #595 + INST(Saddlv_v , ISimdSV , (0b0000111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 1 ), // #596 + INST(Saddw_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 0 ), // #597 + INST(Saddw2_v , ISimdWWV , (0b0000111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 1 ), // #598 + INST(Scvtf_v , SimdFcvtSV , (0b0000111000100001110110, 0b0000111100000000111001, 0b0001111000100010000000, 0) , kRWI_W , 0 , 10 ), // #599 + INST(Sdot_v , SimdDot , (0b0000111010000000100101, 0b0000111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 1 ), // #600 + INST(Sha1c_v , ISimdVVVx , (0b0101111000000000000000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 1 ), // #601 + INST(Sha1h_v , ISimdVVx , (0b0101111000101000000010, kOp_S, kOp_S) , kRWI_W , 0 , 8 ), // #602 + INST(Sha1m_v , ISimdVVVx , (0b0101111000000000001000, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 2 ), // #603 + INST(Sha1p_v , ISimdVVVx , (0b0101111000000000000100, kOp_Q, kOp_S, kOp_V4S) , kRWI_X , 0 , 3 ), // #604 + INST(Sha1su0_v , ISimdVVVx , (0b0101111000000000001100, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 4 ), // #605 + INST(Sha1su1_v , ISimdVVx , (0b0101111000101000000110, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 9 ), // #606 + INST(Sha256h_v , ISimdVVVx , (0b0101111000000000010000, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 5 ), // #607 + INST(Sha256h2_v , ISimdVVVx , (0b0101111000000000010100, kOp_Q, kOp_Q, kOp_V4S) , kRWI_X , 0 , 6 ), // #608 + INST(Sha256su0_v , ISimdVVx , (0b0101111000101000001010, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 10 ), // #609 + INST(Sha256su1_v , ISimdVVVx , (0b0101111000000000011000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 7 ), // #610 + INST(Sha512h_v , ISimdVVVx , (0b1100111001100000100000, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 8 ), // #611 + INST(Sha512h2_v , ISimdVVVx , (0b1100111001100000100001, kOp_Q, kOp_Q, kOp_V2D) , kRWI_X , 0 , 9 ), // #612 + INST(Sha512su0_v , ISimdVVx , (0b1100111011000000100000, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 11 ), // #613 + INST(Sha512su1_v , ISimdVVVx , (0b1100111001100000100010, kOp_V2D, kOp_V2D, kOp_V2D) , kRWI_X , 0 , 10 ), // #614 + INST(Shadd_v , ISimdVVV , (0b0000111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 26 ), // #615 + INST(Shl_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000010101, 0, kVO_V_Any) , kRWI_W , 0 , 2 ), // #616 + INST(Shll_v , SimdShiftES , (0b0010111000100001001110, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #617 + INST(Shll2_v , SimdShiftES , (0b0110111000100001001110, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #618 + INST(Shrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100001, 1, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 3 ), // #619 + INST(Shrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 4 ), // #620 + INST(Shsub_v , ISimdVVV , (0b0000111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 27 ), // #621 + INST(Sli_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010101, 0, kVO_V_Any) , kRWI_X , 0 , 5 ), // #622 + INST(Sm3partw1_v , ISimdVVVx , (0b1100111001100000110000, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 11 ), // #623 + INST(Sm3partw2_v , ISimdVVVx , (0b1100111001100000110001, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #624 + INST(Sm3ss1_v , ISimdVVVVx , (0b1100111001000000000000, kOp_V4S, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_W , 0 , 0 ), // #625 + INST(Sm3tt1a_v , SimdSm3tt , (0b1100111001000000100000) , kRWI_X , 0 , 0 ), // #626 + INST(Sm3tt1b_v , SimdSm3tt , (0b1100111001000000100001) , kRWI_X , 0 , 1 ), // #627 + INST(Sm3tt2a_v , SimdSm3tt , (0b1100111001000000100010) , kRWI_X , 0 , 2 ), // #628 + INST(Sm3tt2b_v , SimdSm3tt , (0b1100111001000000100011) , kRWI_X , 0 , 3 ), // #629 + INST(Sm4e_v , ISimdVVx , (0b1100111011000000100001, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 12 ), // #630 + INST(Sm4ekey_v , ISimdVVVx , (0b1100111001100000110010, kOp_V4S, kOp_V4S, kOp_V4S) , kRWI_X , 0 , 13 ), // #631 + INST(Smax_v , ISimdVVV , (0b0000111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 28 ), // #632 + INST(Smaxp_v , ISimdVVV , (0b0000111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 29 ), // #633 + INST(Smaxv_v , ISimdSV , (0b0000111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 2 ), // #634 + INST(Smin_v , ISimdVVV , (0b0000111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 30 ), // #635 + INST(Sminp_v , ISimdVVV , (0b0000111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 31 ), // #636 + INST(Sminv_v , ISimdSV , (0b0000111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 3 ), // #637 + INST(Smlal_v , ISimdVVVe , (0b0000111000100000100000, kVO_V_B8H4S2, 0b0000111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 3 ), // #638 + INST(Smlal2_v , ISimdVVVe , (0b0100111000100000100000, kVO_V_B16H8S4, 0b0100111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 4 ), // #639 + INST(Smlsl_v , ISimdVVVe , (0b0000111000100000101000, kVO_V_B8H4S2, 0b0000111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 5 ), // #640 + INST(Smlsl2_v , ISimdVVVe , (0b0100111000100000101000, kVO_V_B16H8S4, 0b0100111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 6 ), // #641 + INST(Smmla_v , ISimdVVVx , (0b0100111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 14 ), // #642 + INST(Smov_v , SimdSmovUmov , (0b0000111000000000001011, kVO_V_BHS, 1) , kRWI_W , 0 , 0 ), // #643 + INST(Smull_v , ISimdVVVe , (0b0000111000100000110000, kVO_V_B8H4S2, 0b0000111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 7 ), // #644 + INST(Smull2_v , ISimdVVVe , (0b0100111000100000110000, kVO_V_B16H8S4, 0b0100111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 8 ), // #645 + INST(Sqabs_v , ISimdVV , (0b0000111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 13 ), // #646 + INST(Sqadd_v , ISimdVVV , (0b0000111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 32 ), // #647 + INST(Sqdmlal_v , ISimdVVVe , (0b0000111000100000100100, kVO_SV_BHS, 0b0000111100000000001100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 9 ), // #648 + INST(Sqdmlal2_v , ISimdVVVe , (0b0100111000100000100100, kVO_V_B16H8S4, 0b0100111100000000001100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 10 ), // #649 + INST(Sqdmlsl_v , ISimdVVVe , (0b0000111000100000101100, kVO_SV_BHS, 0b0000111100000000011100, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 11 ), // #650 + INST(Sqdmlsl2_v , ISimdVVVe , (0b0100111000100000101100, kVO_V_B16H8S4, 0b0100111100000000011100, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 12 ), // #651 + INST(Sqdmulh_v , ISimdVVVe , (0b0000111000100000101101, kVO_SV_HS, 0b0000111100000000110000, kVO_SV_HS) , kRWI_W , F(VH0_15) , 13 ), // #652 + INST(Sqdmull_v , ISimdVVVe , (0b0000111000100000110100, kVO_SV_BHS, 0b0000111100000000101100, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 14 ), // #653 + INST(Sqdmull2_v , ISimdVVVe , (0b0100111000100000110100, kVO_V_B16H8S4, 0b0100111100000000101100, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 15 ), // #654 + INST(Sqneg_v , ISimdVV , (0b0010111000100000011110, kVO_SV_Any) , kRWI_W , 0 , 14 ), // #655 + INST(Sqrdmlah_v , ISimdVVVe , (0b0010111000000000100001, kVO_SV_HS, 0b0010111100000000110100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 16 ), // #656 + INST(Sqrdmlsh_v , ISimdVVVe , (0b0010111000000000100011, kVO_SV_HS, 0b0010111100000000111100, kVO_SV_HS) , kRWI_X , F(VH0_15) , 17 ), // #657 + INST(Sqrdmulh_v , ISimdVVVe , (0b0010111000100000101101, kVO_SV_HS, 0b0000111100000000110100, kVO_SV_HS) , kRWI_W , F(VH0_15) , 18 ), // #658 + INST(Sqrshl_v , SimdShift , (0b0000111000100000010111, 0b0000000000000000000000, 1, kVO_SV_Any) , kRWI_W , 0 , 6 ), // #659 + INST(Sqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 7 ), // #660 + INST(Sqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 8 ), // #661 + INST(Sqrshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100011, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 9 ), // #662 + INST(Sqrshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100011, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 10 ), // #663 + INST(Sqshl_v , SimdShift , (0b0000111000100000010011, 0b0000111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 11 ), // #664 + INST(Sqshlu_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000011001, 0, kVO_SV_Any) , kRWI_W , 0 , 12 ), // #665 + INST(Sqshrn_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 13 ), // #666 + INST(Sqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 14 ), // #667 + INST(Sqshrun_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100001, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #668 + INST(Sqshrun2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100001, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #669 + INST(Sqsub_v , ISimdVVV , (0b0000111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 33 ), // #670 + INST(Sqxtn_v , ISimdVV , (0b0000111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 15 ), // #671 + INST(Sqxtn2_v , ISimdVV , (0b0100111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 16 ), // #672 + INST(Sqxtun_v , ISimdVV , (0b0010111000100001001010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 17 ), // #673 + INST(Sqxtun2_v , ISimdVV , (0b0110111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 18 ), // #674 + INST(Srhadd_v , ISimdVVV , (0b0000111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 34 ), // #675 + INST(Sri_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000010001, 1, kVO_V_Any) , kRWI_W , 0 , 17 ), // #676 + INST(Srshl_v , SimdShift , (0b0000111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 18 ), // #677 + INST(Srshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 19 ), // #678 + INST(Srsra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 20 ), // #679 + INST(Sshl_v , SimdShift , (0b0000111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 21 ), // #680 + INST(Sshll_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 22 ), // #681 + INST(Sshll2_v , SimdShift , (0b0000000000000000000000, 0b0100111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 23 ), // #682 + INST(Sshr_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 24 ), // #683 + INST(Ssra_v , SimdShift , (0b0000000000000000000000, 0b0000111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 25 ), // #684 + INST(Ssubl_v , ISimdVVV , (0b0000111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 35 ), // #685 + INST(Ssubl2_v , ISimdVVV , (0b0100111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 36 ), // #686 + INST(Ssubw_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 2 ), // #687 + INST(Ssubw2_v , ISimdWWV , (0b0000111000100000001100, kVO_V_B16H8S4) , kRWI_X , 0 , 3 ), // #688 + INST(St1_v , SimdLdNStN , (0b0000110100000000000000, 0b0000110000000000001000, 1, 0) , kRWI_STn , F(Consecutive) , 8 ), // #689 + INST(St2_v , SimdLdNStN , (0b0000110100100000000000, 0b0000110000000000100000, 2, 0) , kRWI_STn , F(Consecutive) , 9 ), // #690 + INST(St3_v , SimdLdNStN , (0b0000110100000000001000, 0b0000110000000000010000, 3, 0) , kRWI_STn , F(Consecutive) , 10 ), // #691 + INST(St4_v , SimdLdNStN , (0b0000110100100000001000, 0b0000110000000000000000, 4, 0) , kRWI_STn , F(Consecutive) , 11 ), // #692 + INST(Stnp_v , SimdLdpStp , (0b0010110000, 0b0000000000) , kRWI_RRW , 0 , 2 ), // #693 + INST(Stp_v , SimdLdpStp , (0b0010110100, 0b0010110010) , kRWI_RRW , 0 , 3 ), // #694 + INST(Str_v , SimdLdSt , (0b0011110100, 0b00111100000, 0b00111100001, 0b00000000, Inst::kIdStur_v) , kRWI_RW , 0 , 1 ), // #695 + INST(Stur_v , SimdLdurStur , (0b0011110000000000000000) , kRWI_RW , 0 , 1 ), // #696 + INST(Sub_v , ISimdVVV , (0b0010111000100000100001, kVO_V_Any) , kRWI_W , 0 , 37 ), // #697 + INST(Subhn_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 38 ), // #698 + INST(Subhn2_v , ISimdVVV , (0b0000111000100000011000, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 39 ), // #699 + INST(Sudot_v , SimdDot , (0b0000000000000000000000, 0b0000111100000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 2 ), // #700 + INST(Suqadd_v , ISimdVV , (0b0000111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 19 ), // #701 + INST(Sxtl_v , SimdSxtlUxtl , (0b0000111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 0 ), // #702 + INST(Sxtl2_v , SimdSxtlUxtl , (0b0100111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 1 ), // #703 + INST(Tbl_v , SimdTblTbx , (0b0000111000000000000000) , kRWI_W , 0 , 0 ), // #704 + INST(Tbx_v , SimdTblTbx , (0b0000111000000000000100) , kRWI_W , 0 , 1 ), // #705 + INST(Trn1_v , ISimdVVV , (0b0000111000000000001010, kVO_V_BHS_D2) , kRWI_W , 0 , 40 ), // #706 + INST(Trn2_v , ISimdVVV , (0b0000111000000000011010, kVO_V_BHS_D2) , kRWI_W , 0 , 41 ), // #707 + INST(Uaba_v , ISimdVVV , (0b0010111000100000011111, kVO_V_BHS) , kRWI_X , 0 , 42 ), // #708 + INST(Uabal_v , ISimdVVV , (0b0010111000100000010100, kVO_V_B8H4S2) , kRWI_X , F(Long) , 43 ), // #709 + INST(Uabal2_v , ISimdVVV , (0b0110111000100000010100, kVO_V_B16H8S4) , kRWI_X , F(Long) , 44 ), // #710 + INST(Uabd_v , ISimdVVV , (0b0010111000100000011101, kVO_V_BHS) , kRWI_W , 0 , 45 ), // #711 + INST(Uabdl_v , ISimdVVV , (0b0010111000100000011100, kVO_V_B8H4S2) , kRWI_W , F(Long) , 46 ), // #712 + INST(Uabdl2_v , ISimdVVV , (0b0110111000100000011100, kVO_V_B16H8S4) , kRWI_W , F(Long) , 47 ), // #713 + INST(Uadalp_v , ISimdVV , (0b0010111000100000011010, kVO_V_BHS) , kRWI_X , F(Long) | F(Pair) , 20 ), // #714 + INST(Uaddl_v , ISimdVVV , (0b0010111000100000000000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 48 ), // #715 + INST(Uaddl2_v , ISimdVVV , (0b0110111000100000000000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 49 ), // #716 + INST(Uaddlp_v , ISimdVV , (0b0010111000100000001010, kVO_V_BHS) , kRWI_W , F(Long) | F(Pair) , 21 ), // #717 + INST(Uaddlv_v , ISimdSV , (0b0010111000110000001110, kVO_V_BH_4S) , kRWI_W , F(Long) , 4 ), // #718 + INST(Uaddw_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B8H4S2) , kRWI_W , 0 , 4 ), // #719 + INST(Uaddw2_v , ISimdWWV , (0b0010111000100000000100, kVO_V_B16H8S4) , kRWI_W , 0 , 5 ), // #720 + INST(Ucvtf_v , SimdFcvtSV , (0b0010111000100001110110, 0b0010111100000000111001, 0b0001111000100011000000, 0) , kRWI_W , 0 , 11 ), // #721 + INST(Udot_v , SimdDot , (0b0010111010000000100101, 0b0010111110000000111000, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 3 ), // #722 + INST(Uhadd_v , ISimdVVV , (0b0010111000100000000001, kVO_V_BHS) , kRWI_W , 0 , 50 ), // #723 + INST(Uhsub_v , ISimdVVV , (0b0010111000100000001001, kVO_V_BHS) , kRWI_W , 0 , 51 ), // #724 + INST(Umax_v , ISimdVVV , (0b0010111000100000011001, kVO_V_BHS) , kRWI_W , 0 , 52 ), // #725 + INST(Umaxp_v , ISimdVVV , (0b0010111000100000101001, kVO_V_BHS) , kRWI_W , 0 , 53 ), // #726 + INST(Umaxv_v , ISimdSV , (0b0010111000110000101010, kVO_V_BH_4S) , kRWI_W , 0 , 5 ), // #727 + INST(Umin_v , ISimdVVV , (0b0010111000100000011011, kVO_V_BHS) , kRWI_W , 0 , 54 ), // #728 + INST(Uminp_v , ISimdVVV , (0b0010111000100000101011, kVO_V_BHS) , kRWI_W , 0 , 55 ), // #729 + INST(Uminv_v , ISimdSV , (0b0010111000110001101010, kVO_V_BH_4S) , kRWI_W , 0 , 6 ), // #730 + INST(Umlal_v , ISimdVVVe , (0b0010111000100000100000, kVO_V_B8H4S2, 0b0010111100000000001000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 19 ), // #731 + INST(Umlal2_v , ISimdVVVe , (0b0110111000100000100000, kVO_V_B16H8S4, 0b0010111100000000001000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 20 ), // #732 + INST(Umlsl_v , ISimdVVVe , (0b0010111000100000101000, kVO_V_B8H4S2, 0b0010111100000000011000, kVO_V_H4S2) , kRWI_X , F(Long) | F(VH0_15) , 21 ), // #733 + INST(Umlsl2_v , ISimdVVVe , (0b0110111000100000101000, kVO_V_B16H8S4, 0b0110111100000000011000, kVO_V_H8S4) , kRWI_X , F(Long) | F(VH0_15) , 22 ), // #734 + INST(Ummla_v , ISimdVVVx , (0b0110111010000000101001, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 15 ), // #735 + INST(Umov_v , SimdSmovUmov , (0b0000111000000000001111, kVO_V_Any, 0) , kRWI_W , 0 , 1 ), // #736 + INST(Umull_v , ISimdVVVe , (0b0010111000100000110000, kVO_V_B8H4S2, 0b0010111100000000101000, kVO_V_H4S2) , kRWI_W , F(Long) | F(VH0_15) , 23 ), // #737 + INST(Umull2_v , ISimdVVVe , (0b0110111000100000110000, kVO_V_B16H8S4, 0b0110111100000000101000, kVO_V_H8S4) , kRWI_W , F(Long) | F(VH0_15) , 24 ), // #738 + INST(Uqadd_v , ISimdVVV , (0b0010111000100000000011, kVO_SV_Any) , kRWI_W , 0 , 56 ), // #739 + INST(Uqrshl_v , SimdShift , (0b0010111000100000010111, 0b0000000000000000000000, 0, kVO_SV_Any) , kRWI_W , 0 , 26 ), // #740 + INST(Uqrshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100111, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #741 + INST(Uqrshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100111, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #742 + INST(Uqshl_v , SimdShift , (0b0010111000100000010011, 0b0010111100000000011101, 0, kVO_SV_Any) , kRWI_W , 0 , 29 ), // #743 + INST(Uqshrn_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000100101, 1, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 30 ), // #744 + INST(Uqshrn2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000100101, 1, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 31 ), // #745 + INST(Uqsub_v , ISimdVVV , (0b0010111000100000001011, kVO_SV_Any) , kRWI_W , 0 , 57 ), // #746 + INST(Uqxtn_v , ISimdVV , (0b0010111000100001010010, kVO_SV_B8H4S2) , kRWI_W , F(Narrow) , 22 ), // #747 + INST(Uqxtn2_v , ISimdVV , (0b0110111000100001010010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 23 ), // #748 + INST(Urecpe_v , ISimdVV , (0b0000111010100001110010, kVO_V_S) , kRWI_W , 0 , 24 ), // #749 + INST(Urhadd_v , ISimdVVV , (0b0010111000100000000101, kVO_V_BHS) , kRWI_W , 0 , 58 ), // #750 + INST(Urshl_v , SimdShift , (0b0010111000100000010101, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 32 ), // #751 + INST(Urshr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001001, 1, kVO_V_Any) , kRWI_W , 0 , 33 ), // #752 + INST(Ursqrte_v , ISimdVV , (0b0010111010100001110010, kVO_V_S) , kRWI_W , 0 , 25 ), // #753 + INST(Ursra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000001101, 1, kVO_V_Any) , kRWI_X , 0 , 34 ), // #754 + INST(Usdot_v , SimdDot , (0b0000111010000000100111, 0b0000111110000000111100, kET_S, kET_B, kET_4B) , kRWI_X , 0 , 4 ), // #755 + INST(Ushl_v , SimdShift , (0b0010111000100000010001, 0b0000000000000000000000, 0, kVO_V_Any) , kRWI_W , 0 , 35 ), // #756 + INST(Ushll_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000101001, 0, kVO_V_B8H4S2) , kRWI_W , F(Long) , 36 ), // #757 + INST(Ushll2_v , SimdShift , (0b0000000000000000000000, 0b0110111100000000101001, 0, kVO_V_B16H8S4) , kRWI_W , F(Long) , 37 ), // #758 + INST(Ushr_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000001, 1, kVO_V_Any) , kRWI_W , 0 , 38 ), // #759 + INST(Usmmla_v , ISimdVVVx , (0b0100111010000000101011, kOp_V4S, kOp_V16B, kOp_V16B) , kRWI_X , 0 , 16 ), // #760 + INST(Usqadd_v , ISimdVV , (0b0010111000100000001110, kVO_SV_Any) , kRWI_X , 0 , 26 ), // #761 + INST(Usra_v , SimdShift , (0b0000000000000000000000, 0b0010111100000000000101, 1, kVO_V_Any) , kRWI_X , 0 , 39 ), // #762 + INST(Usubl_v , ISimdVVV , (0b0010111000100000001000, kVO_V_B8H4S2) , kRWI_W , F(Long) , 59 ), // #763 + INST(Usubl2_v , ISimdVVV , (0b0110111000100000001000, kVO_V_B16H8S4) , kRWI_W , F(Long) , 60 ), // #764 + INST(Usubw_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B8H4S2) , kRWI_W , 0 , 6 ), // #765 + INST(Usubw2_v , ISimdWWV , (0b0010111000100000001100, kVO_V_B16H8S4) , kRWI_W , 0 , 7 ), // #766 + INST(Uxtl_v , SimdSxtlUxtl , (0b0010111100000000101001, kVO_V_B8H4S2) , kRWI_W , F(Long) , 2 ), // #767 + INST(Uxtl2_v , SimdSxtlUxtl , (0b0110111100000000101001, kVO_V_B16H8S4) , kRWI_W , F(Long) , 3 ), // #768 + INST(Uzp1_v , ISimdVVV , (0b0000111000000000000110, kVO_V_BHS_D2) , kRWI_W , 0 , 61 ), // #769 + INST(Uzp2_v , ISimdVVV , (0b0000111000000000010110, kVO_V_BHS_D2) , kRWI_W , 0 , 62 ), // #770 + INST(Xar_v , ISimdVVVI , (0b1100111001100000100011, kVO_V_D2, 6, 10, 0) , kRWI_W , 0 , 1 ), // #771 + INST(Xtn_v , ISimdVV , (0b0000111000100001001010, kVO_V_B8H4S2) , kRWI_W , F(Narrow) , 27 ), // #772 + INST(Xtn2_v , ISimdVV , (0b0100111000100001001010, kVO_V_B16H8S4) , kRWI_X , F(Narrow) , 28 ), // #773 + INST(Zip1_v , ISimdVVV , (0b0000111000000000001110, kVO_V_BHS_D2) , kRWI_W , 0 , 63 ), // #774 + INST(Zip2_v , ISimdVVV , (0b0000111000000000011110, kVO_V_BHS_D2) , kRWI_W , 0 , 64 ) // #775 // ${InstInfo:End} }; @@ -1069,8 +1078,9 @@ const BaseBranchReg baseBranchReg[3] = { { 0b11010110010111110000000000000000 } // ret }; -const BaseBranchRel baseBranchRel[2] = { +const BaseBranchRel baseBranchRel[3] = { { 0b00010100000000000000000000000000 }, // b + { 0b00010100000000000000000000010000 }, // bc { 0b10010100000000000000000000000000 } // bl }; @@ -1156,6 +1166,13 @@ const BaseLogical baseLogical[8] = { { 0b0101010000, 0b01100100, 0 } // orr }; +const BaseMinMax baseMinMax[4] = { + { 0b00011010110000000110000000000000, 0b00010001110000000000000000000000 }, // smax + { 0b00011010110000000110100000000000, 0b00010001110010000000000000000000 }, // smin + { 0b00011010110000000110010000000000, 0b00010001110001000000000000000000 }, // umax + { 0b00011010110000000110110000000000, 0b00010001110011000000000000000000 } // umin +}; + const BaseMovKNZ baseMovKNZ[3] = { { 0b01110010100000000000000000000000 }, // movk { 0b00010010100000000000000000000000 }, // movn @@ -1168,7 +1185,7 @@ const BaseMvnNeg baseMvnNeg[3] = { { 0b01101011000000000000001111100000 } // negs }; -const BaseOp baseOp[23] = { +const BaseOp baseOp[24] = { { 0b11010101000000110010000110011111 }, // autia1716 { 0b11010101000000110010001110111111 }, // autiasp { 0b11010101000000110010001110011111 }, // autiaz @@ -1177,6 +1194,7 @@ const BaseOp baseOp[23] = { { 0b11010101000000110010001111011111 }, // autibz { 0b11010101000000000100000001011111 }, // axflag { 0b11010101000000000100000000011111 }, // cfinv + { 0b11010101000000110010001011011111 }, // clrbhb { 0b11010101000000110010001010011111 }, // csdb { 0b11010101000000110010000011011111 }, // dgh { 0b11010110101111110000001111100000 }, // drps @@ -1194,8 +1212,9 @@ const BaseOp baseOp[23] = { { 0b11010101000000110010000000111111 } // yield }; -const BaseOpImm baseOpImm[14] = { +const BaseOpImm baseOpImm[15] = { { 0b11010100001000000000000000000000, 16, 5 }, // brk + { 0b11010101000000110010010000011111, 2, 6 }, // bti { 0b11010101000000110011000001011111, 4, 8 }, // clrex { 0b11010100101000000000000000000001, 16, 5 }, // dcps1 { 0b11010100101000000000000000000010, 16, 5 }, // dcps2 @@ -1211,6 +1230,14 @@ const BaseOpImm baseOpImm[14] = { { 0b00000000000000000000000000000000, 16, 0 } // udf }; +const BaseOpX16 baseOpX16[1] = { + { 0b11010101000000110010010100011111 } // chkfeat +}; + +const BasePrfm basePrfm[1] = { + { 0b11111000101, 0b1111100110, 0b11111000100, 0b11011000 } // prfm +}; + const BaseR baseR[10] = { { 0b11011010110000010011101111100000, kX, kZR, 0 }, // autdza { 0b11011010110000010011111111100000, kX, kZR, 0 }, // autdzb @@ -1266,7 +1293,7 @@ const BaseRM_SImm9 baseRM_SImm9[23] = { { 0b0111100001000000000000, 0b0000000000000000000000, kW , kZR, 0 , 0 }, // ldurh { 0b0011100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0 }, // ldursb { 0b0111100011000000000000, 0b0000000000000000000000, kWX, kZR, 22, 0 }, // ldursh - { 0b1011100010000000000000, 0b0000000000000000000000, kX, kZR, 0 , 0 }, // ldursw + { 0b1011100010000000000000, 0b0000000000000000000000, kX , kZR, 0 , 0 }, // ldursw { 0b1101100110100000000010, 0b1101100110100000000001, kX, kSP, 0, 4 }, // st2g { 0b1101100100100000000010, 0b1101100100100000000001, kX, kSP, 0, 4 }, // stg { 0b1011100000000000000010, 0b0000000000000000000000, kWX, kZR, 30, 0 }, // sttr @@ -1279,7 +1306,8 @@ const BaseRM_SImm9 baseRM_SImm9[23] = { { 0b1101100101100000000010, 0b1101100101100000000001, kX , kSP, 0, 4 } // stzg }; -const BaseRR baseRR[15] = { +const BaseRR baseRR[18] = { + { 0b01011010110000000010000000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // abs { 0b11011010110000010001100000000000, kX, kZR, 0, kX, kSP, 5, true }, // autda { 0b11011010110000010001110000000000, kX, kZR, 0, kX, kSP, 5, true }, // autdb { 0b11011010110000010001000000000000, kX, kZR, 0, kX, kSP, 5, true }, // autia @@ -1287,6 +1315,8 @@ const BaseRR baseRR[15] = { { 0b01011010110000000001010000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // cls { 0b01011010110000000001000000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // clz { 0b10111010110000000000000000011111, kX, kSP, 5, kX, kSP, 16, true }, // cmpp + { 0b01011010110000000001110000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // cnt + { 0b01011010110000000001100000000000, kWX, kZR, 0, kWX, kZR, 5, true }, // ctz { 0b01011010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true }, // ngc { 0b01111010000000000000001111100000, kWX, kZR, 0, kWX, kZR, 16, true }, // ngcs { 0b11011010110000010000100000000000, kX, kZR, 0, kX, kSP, 5, true }, // pacda @@ -1846,8 +1876,53 @@ const InstDB::CommonInfo InstDB::commonData[] = { #ifndef ASMJIT_NO_TEXT // ${NameData:Begin} // ------------------- Automatically generated, do not edit ------------------- +const InstNameIndex InstDB::instNameIndex = {{ + { Inst::kIdAbs , Inst::kIdAnd_v + 1 }, + { Inst::kIdB , Inst::kIdBsl_v + 1 }, + { Inst::kIdCas , Inst::kIdCnt_v + 1 }, + { Inst::kIdDc , Inst::kIdDup_v + 1 }, + { Inst::kIdEon , Inst::kIdExt_v + 1 }, + { Inst::kIdFabd_v , Inst::kIdFsub_v + 1 }, + { Inst::kIdGmi , Inst::kIdGmi + 1 }, + { Inst::kIdHint , Inst::kIdHvc + 1 }, + { Inst::kIdIc , Inst::kIdIns_v + 1 }, + { Inst::kIdNone , Inst::kIdNone + 1 }, + { Inst::kIdNone , Inst::kIdNone + 1 }, + { Inst::kIdLdadd , Inst::kIdLdur_v + 1 }, + { Inst::kIdMadd , Inst::kIdMvni_v + 1 }, + { Inst::kIdNeg , Inst::kIdNot_v + 1 }, + { Inst::kIdOrn , Inst::kIdOrr_v + 1 }, + { Inst::kIdPacda , Inst::kIdPmull2_v + 1 }, + { Inst::kIdNone , Inst::kIdNone + 1 }, + { Inst::kIdRbit , Inst::kIdRsubhn2_v + 1 }, + { Inst::kIdSbc , Inst::kIdSxtl2_v + 1 }, + { Inst::kIdTlbi , Inst::kIdTrn2_v + 1 }, + { Inst::kIdUbfiz , Inst::kIdUzp2_v + 1 }, + { Inst::kIdNone , Inst::kIdNone + 1 }, + { Inst::kIdWfe , Inst::kIdWfi + 1 }, + { Inst::kIdXaflag , Inst::kIdXtn2_v + 1 }, + { Inst::kIdYield , Inst::kIdYield + 1 }, + { Inst::kIdZip1_v , Inst::kIdZip2_v + 1 } +}, uint16_t(9)}; + +const char InstDB::_instNameStringTable[] = + "\x61\x75\x74\x69\x61\x31\x37\x31\x36\x61\x75\x74\x69\x62\x6C\x64\x73\x6D\x61\x78\x61\x6C\x68\x6C\x64\x73\x6D\x69\x6E" + "\x61\x6C\x6C\x64\x75\x6D\x61\x78\x61\x6C\x6C\x64\x75\x6D\x69\x6E\x61\x6C\x73\x68\x61\x32\x35\x36\x73\x75\x30\x73\x68" + "\x61\x35\x31\x32\x73\x75\x31\x73\x6D\x33\x70\x61\x72\x74\x77\x73\x71\x72\x73\x68\x72\x75\x6E\x6C\x64\x61\x64\x64\x61" + "\x6C\x6C\x64\x63\x6C\x72\x61\x6C\x6C\x64\x65\x6F\x72\x61\x6C\x6C\x64\x73\x65\x74\x61\x6C\x6C\x62\x73\x74\x73\x6D\x61" + "\x78\x73\x74\x73\x6D\x69\x6E\x73\x74\x75\x6D\x61\x78\x73\x74\x75\x6D\x69\x6E\x66\x72\x69\x6E\x74\x33\x32\x7A\x36\x34" + "\x78\x36\x34\x7A\x68\x32\x73\x71\x64\x6D\x6C\x61\x6C\x73\x6C\x32\x73\x71\x64\x6D\x75\x6C\x73\x71\x72\x64\x6D\x6C\x61" + "\x75\x6C\x68\x6E\x32\x73\x71\x73\x68\x72\x75\x75\x71\x72\x73\x68\x72\x73\x70\x63\x68\x6B\x66\x65\x61\x63\x72\x63\x33" + "\x32\x63\x73\x74\x61\x64\x64\x73\x74\x63\x6C\x72\x73\x74\x65\x6F\x72\x73\x74\x73\x65\x74\x78\x70\x61\x63\x6C\x62\x66" + "\x63\x76\x74\x62\x66\x6D\x6C\x61\x6C\x74\x66\x63\x76\x74\x78\x66\x6A\x63\x76\x74\x7A\x66\x6D\x61\x78\x6E\x6D\x66\x6D" + "\x69\x6E\x6E\x6D\x66\x72\x73\x71\x72\x72\x61\x64\x64\x72\x73\x75\x62\x73\x68\x61\x31\x73\x6D\x33\x74\x74\x31\x32\x61" + "\x32\x62\x73\x6D\x34\x65\x6B\x65\x79\x73\x71\x78\x74\x75\x75\x71\x73\x68\x72\x75\x72\x73\x71\x72\x73\x65\x74\x66\x72" + "\x65\x76\x38"; + + const uint32_t InstDB::_instNameIndexTable[] = { 0x80000000, // Small ''. + 0x80004C41, // Small 'abs'. 0x80000C81, // Small 'adc'. 0x80098C81, // Small 'adcs'. 0x80001081, // Small 'add'. @@ -1876,6 +1951,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x85A4D2A1, // Small 'autizb'. 0x8E161B01, // Small 'axflag'. 0x80000002, // Small 'b'. + 0x80000062, // Small 'bc'. 0x80000CC2, // Small 'bfc'. 0x800024C2, // Small 'bfi'. 0x800034C2, // Small 'bfm'. @@ -1886,6 +1962,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80004982, // Small 'blr'. 0x80000242, // Small 'br'. 0x80002E42, // Small 'brk'. + 0x80002682, // Small 'bti'. 0x80004C23, // Small 'cas'. 0x8000CC23, // Small 'casa'. 0x8020CC23, // Small 'casab'. @@ -1907,8 +1984,10 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80073463, // Small 'ccmn'. 0x80083463, // Small 'ccmp'. 0x816724C3, // Small 'cfinv'. + 0x100260C1, // Large 'chkfea|t'. 0x8001B923, // Small 'cinc'. 0x800B3923, // Small 'cinv'. + 0x84814983, // Small 'clrbhb'. 0x8182C983, // Small 'clrex'. 0x80004D83, // Small 'cls'. 0x80006983, // Small 'clz'. @@ -1916,11 +1995,12 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800041A3, // Small 'cmp'. 0x800841A3, // Small 'cmpp'. 0x800395C3, // Small 'cneg'. + 0x800051C3, // Small 'cnt'. 0x85DF0E43, // Small 'crc32b'. - 0x100D60C1, // Large 'crc32c|b'. - 0x101660C1, // Large 'crc32c|h'. - 0x104860C1, // Large 'crc32c|w'. - 0x101360C1, // Large 'crc32c|x'. + 0x100D60C7, // Large 'crc32c|b'. + 0x101660C7, // Large 'crc32c|h'. + 0x104860C7, // Large 'crc32c|w'. + 0x101360C7, // Large 'crc32c|x'. 0x91DF0E43, // Small 'crc32h'. 0xAFDF0E43, // Small 'crc32w'. 0xB1DF0E43, // Small 'crc32x'. @@ -1931,6 +2011,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80372663, // Small 'csinc'. 0x81672663, // Small 'csinv'. 0x8072BA63, // Small 'csneg'. + 0x80006A83, // Small 'ctz'. 0x80000064, // Small 'dc'. 0x81C9C064, // Small 'dcps1'. 0x81D9C064, // Small 'dcps2'. @@ -2112,13 +2193,14 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x83A20C30, // Small 'pacdza'. 0x85A20C30, // Small 'pacdzb'. 0x80138C30, // Small 'pacga'. + 0x80069A50, // Small 'prfm'. 0x80214E70, // Small 'pssbb'. 0x800A2452, // Small 'rbit'. 0x800050B2, // Small 'ret'. 0x800058B2, // Small 'rev'. - 0x20073138, // Large 'rev|16'. + 0x2007313E, // Large 'rev|16'. 0x81DF58B2, // Small 'rev32'. - 0x208F3138, // Large 'rev|64'. + 0x208F313E, // Large 'rev|64'. 0x800049F2, // Small 'ror'. 0x800B49F2, // Small 'rorv'. 0x80000C53, // Small 'sbc'. @@ -2127,12 +2209,14 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80069853, // Small 'sbfm'. 0x800C1853, // Small 'sbfx'. 0x800B2493, // Small 'sdiv'. - 0x113B4134, // Large 'setf|8'. - 0x20074134, // Large 'setf|16'. + 0x1141413A, // Large 'setf|8'. + 0x2007413A, // Large 'setf|16'. 0x800058B3, // Small 'sev'. 0x800658B3, // Small 'sevl'. 0x984205B3, // Small 'smaddl'. + 0x800C05B3, // Small 'smax'. 0x80000DB3, // Small 'smc'. + 0x800725B3, // Small 'smin'. 0x9872B9B3, // Small 'smnegl'. 0x982ACDB3, // Small 'smsubl'. 0x808655B3, // Small 'smulh'. @@ -2142,21 +2226,21 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80420693, // Small 'stadd'. 0x98420693, // Small 'staddl'. 0x84420693, // Small 'staddb'. - 0x206D50C7, // Large 'stadd|lb'. + 0x206D50CD, // Large 'stadd|lb'. 0x90420693, // Small 'staddh'. - 0x201550C7, // Large 'stadd|lh'. + 0x201550CD, // Large 'stadd|lh'. 0x81260E93, // Small 'stclr'. 0x99260E93, // Small 'stclrl'. 0x85260E93, // Small 'stclrb'. - 0x206D50CC, // Large 'stclr|lb'. + 0x206D50D2, // Large 'stclr|lb'. 0x91260E93, // Small 'stclrh'. - 0x201550CC, // Large 'stclr|lh'. + 0x201550D2, // Large 'stclr|lh'. 0x81279693, // Small 'steor'. 0x99279693, // Small 'steorl'. 0x85279693, // Small 'steorb'. - 0x206D50D1, // Large 'steor|lb'. + 0x206D50D7, // Large 'steor|lb'. 0x91279693, // Small 'steorh'. - 0x201550D1, // Large 'steor|lh'. + 0x201550D7, // Large 'steor|lh'. 0x80001E93, // Small 'stg'. 0x80069E93, // Small 'stgm'. 0x80081E93, // Small 'stgp'. @@ -2178,9 +2262,9 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8142CE93, // Small 'stset'. 0x9942CE93, // Small 'stsetl'. 0x8542CE93, // Small 'stsetb'. - 0x206D50D6, // Large 'stset|lb'. + 0x206D50DC, // Large 'stset|lb'. 0x9142CE93, // Small 'stseth'. - 0x201550D6, // Large 'stset|lh'. + 0x201550DC, // Large 'stset|lh'. 0xB016CE93, // Small 'stsmax'. 0x100E606F, // Large 'stsmax|l'. 0x100D606F, // Large 'stsmax|b'. @@ -2250,6 +2334,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80001895, // Small 'udf'. 0x800B2495, // Small 'udiv'. 0x984205B5, // Small 'umaddl'. + 0x800C05B5, // Small 'umax'. + 0x800725B5, // Small 'umin'. 0x9872B9B5, // Small 'umnegl'. 0x80C655B5, // Small 'umull'. 0x808655B5, // Small 'umulh'. @@ -2261,7 +2347,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8E161838, // Small 'xaflag'. 0x80418618, // Small 'xpacd'. 0x80918618, // Small 'xpaci'. - 0x208850DB, // Large 'xpacl|ri'. + 0x208850E1, // Large 'xpacl|ri'. 0x80461539, // Small 'yield'. 0x80004C41, // Small 'abs'. 0x80001081, // Small 'add'. @@ -2277,10 +2363,10 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800C0462, // Small 'bcax'. 0x814B0CC2, // Small 'bfcvt'. 0x9D4B0CC2, // Small 'bfcvtn'. - 0x20B150E0, // Large 'bfcvt|n2'. + 0x20B150E6, // Large 'bfcvt|n2'. 0x814790C2, // Small 'bfdot'. - 0x206D50E5, // Large 'bfmla|lb'. - 0x20EA50E5, // Large 'bfmla|lt'. + 0x206D50EB, // Large 'bfmla|lb'. + 0x20F050EB, // Large 'bfmla|lt'. 0x82C6B4C2, // Small 'bfmmla'. 0x80000D22, // Small 'bic'. 0x80001922, // Small 'bif'. @@ -2333,22 +2419,22 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0xA70A5866, // Small 'fcvtps'. 0xAB0A5866, // Small 'fcvtpu'. 0x9D8A5866, // Small 'fcvtxn'. - 0x20B150EC, // Large 'fcvtx|n2'. + 0x20B150F2, // Large 'fcvtx|n2'. 0xA7AA5866, // Small 'fcvtzs'. 0xABAA5866, // Small 'fcvtzu'. 0x800B2486, // Small 'fdiv'. - 0x101060F1, // Large 'fjcvtz|s'. + 0x101060F7, // Large 'fjcvtz|s'. 0x804205A6, // Small 'fmadd'. 0x800C05A6, // Small 'fmax'. 0x9AEC05A6, // Small 'fmaxnm'. - 0x104460F7, // Large 'fmaxnm|p'. - 0x10E360F7, // Large 'fmaxnm|v'. + 0x104460FD, // Large 'fmaxnm|p'. + 0x10E960FD, // Large 'fmaxnm|v'. 0x810C05A6, // Small 'fmaxp'. 0x816C05A6, // Small 'fmaxv'. 0x800725A6, // Small 'fmin'. 0x9AE725A6, // Small 'fminnm'. - 0x104460FD, // Large 'fminnm|p'. - 0x10E360FD, // Large 'fminnm|v'. + 0x10446103, // Large 'fminnm|p'. + 0x10E96103, // Large 'fminnm|v'. 0x810725A6, // Small 'fminp'. 0x816725A6, // Small 'fminv'. 0x8000B1A6, // Small 'fmla'. @@ -2379,8 +2465,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0xA1472646, // Small 'frintp'. 0xB1472646, // Small 'frintx'. 0xB5472646, // Small 'frintz'. - 0x20D25103, // Large 'frsqr|te'. - 0x20705103, // Large 'frsqr|ts'. + 0x20D85109, // Large 'frsqr|te'. + 0x20705109, // Large 'frsqr|ts'. 0x81494666, // Small 'fsqrt'. 0x80015666, // Small 'fsub'. 0x80004DC9, // Small 'ins'. @@ -2411,16 +2497,16 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80C655B0, // Small 'pmull'. 0xBAC655B0, // Small 'pmull2'. 0x9C821032, // Small 'raddhn'. - 0x30B04108, // Large 'radd|hn2'. + 0x30B0410E, // Large 'radd|hn2'. 0x800E6032, // Small 'rax1'. 0x800A2452, // Small 'rbit'. - 0x20073138, // Large 'rev|16'. + 0x2007313E, // Large 'rev|16'. 0x81DF58B2, // Small 'rev32'. - 0x208F3138, // Large 'rev|64'. + 0x208F313E, // Large 'rev|64'. 0x80E92272, // Small 'rshrn'. 0xBAE92272, // Small 'rshrn2'. 0x9C815672, // Small 'rsubhn'. - 0x30B0410C, // Large 'rsub|hn2'. + 0x30B04112, // Large 'rsub|hn2'. 0x80008833, // Small 'saba'. 0x80C08833, // Small 'sabal'. 0xBAC08833, // Small 'sabal2'. @@ -2440,8 +2526,8 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x808E0513, // Small 'sha1h'. 0x80DE0513, // Small 'sha1m'. 0x810E0513, // Small 'sha1p'. - 0x30354110, // Large 'sha1|su0'. - 0x303E4110, // Large 'sha1|su1'. + 0x30354116, // Large 'sha1|su0'. + 0x303E4116, // Large 'sha1|su1'. 0x1016602F, // Large 'sha256|h'. 0x2095602F, // Large 'sha256|h2'. 0x0000902F, // Large 'sha256su0'. @@ -2461,12 +2547,12 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x10058041, // Large 'sm3partw|1'. 0x10328041, // Large 'sm3partw|2'. 0xB939F9B3, // Small 'sm3ss1'. - 0x10006114, // Large 'sm3tt1|a'. - 0x100D6114, // Large 'sm3tt1|b'. - 0x211A5114, // Large 'sm3tt|2a'. - 0x211C5114, // Large 'sm3tt|2b'. + 0x1000611A, // Large 'sm3tt1|a'. + 0x100D611A, // Large 'sm3tt1|b'. + 0x2120511A, // Large 'sm3tt|2a'. + 0x2122511A, // Large 'sm3tt|2b'. 0x8002FDB3, // Small 'sm4e'. - 0x0000711E, // Large 'sm4ekey'. + 0x00007124, // Large 'sm4ekey'. 0x800C05B3, // Small 'smax'. 0x810C05B3, // Small 'smaxp'. 0x816C05B3, // Small 'smaxv'. @@ -2509,7 +2595,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x80EA6233, // Small 'sqxtn'. 0xBAEA6233, // Small 'sqxtn2'. 0x9D5A6233, // Small 'sqxtun'. - 0x20B15125, // Large 'sqxtu|n2'. + 0x20B1512B, // Large 'sqxtu|n2'. 0x8840A253, // Small 'srhadd'. 0x80002653, // Small 'sri'. 0x80C44E53, // Small 'srshl'. @@ -2580,7 +2666,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x20B160B9, // Large 'uqrshr|n2'. 0x80C44E35, // Small 'uqshl'. 0x9D244E35, // Small 'uqshrn'. - 0x20B1512A, // Large 'uqshr|n2'. + 0x20B15130, // Large 'uqshr|n2'. 0x802ACE35, // Small 'uqsub'. 0x80EA6235, // Small 'uqxtn'. 0xBAEA6235, // Small 'uqxtn2'. @@ -2588,7 +2674,7 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x8840A255, // Small 'urhadd'. 0x80C44E55, // Small 'urshl'. 0x81244E55, // Small 'urshr'. - 0x20D2512F, // Large 'ursqr|te'. + 0x20D85135, // Large 'ursqr|te'. 0x80194E55, // Small 'ursra'. 0x81479275, // Small 'usdot'. 0x80062275, // Small 'ushl'. @@ -2612,42 +2698,6 @@ const uint32_t InstDB::_instNameIndexTable[] = { 0x800E413A, // Small 'zip1'. 0x800EC13A // Small 'zip2'. }; - -const char InstDB::_instNameStringTable[] = - "autia1716autibldsmaxalhldsminalldumaxallduminalsha256su0sha512su1sm3partwsqrshru" - "nldaddalldclralldeoralldsetallbstsmaxstsminstumaxstuminfrint32z64x64zh2sqdmlalsl" - "2sqdmulsqrdmlaulhn2sqshruuqrshrspcrc32cstaddstclrsteorstsetxpaclbfcvtbfmlaltfcvt" - "xfjcvtzfmaxnmfminnmfrsqrraddrsubsha1sm3tt12a2bsm4ekeysqxtuuqshrursqrsetfrev8"; - - -const InstDB::InstNameIndex InstDB::instNameIndex[26] = { - { Inst::kIdAdc , Inst::kIdAnd_v + 1 }, - { Inst::kIdB , Inst::kIdBsl_v + 1 }, - { Inst::kIdCas , Inst::kIdCnt_v + 1 }, - { Inst::kIdDc , Inst::kIdDup_v + 1 }, - { Inst::kIdEon , Inst::kIdExt_v + 1 }, - { Inst::kIdFabd_v , Inst::kIdFsub_v + 1 }, - { Inst::kIdGmi , Inst::kIdGmi + 1 }, - { Inst::kIdHint , Inst::kIdHvc + 1 }, - { Inst::kIdIc , Inst::kIdIns_v + 1 }, - { Inst::kIdNone , Inst::kIdNone + 1 }, - { Inst::kIdNone , Inst::kIdNone + 1 }, - { Inst::kIdLdadd , Inst::kIdLdur_v + 1 }, - { Inst::kIdMadd , Inst::kIdMvni_v + 1 }, - { Inst::kIdNeg , Inst::kIdNot_v + 1 }, - { Inst::kIdOrn , Inst::kIdOrr_v + 1 }, - { Inst::kIdPacda , Inst::kIdPmull2_v + 1 }, - { Inst::kIdNone , Inst::kIdNone + 1 }, - { Inst::kIdRbit , Inst::kIdRsubhn2_v + 1 }, - { Inst::kIdSbc , Inst::kIdSxtl2_v + 1 }, - { Inst::kIdTlbi , Inst::kIdTrn2_v + 1 }, - { Inst::kIdUbfiz , Inst::kIdUzp2_v + 1 }, - { Inst::kIdNone , Inst::kIdNone + 1 }, - { Inst::kIdWfe , Inst::kIdWfi + 1 }, - { Inst::kIdXaflag , Inst::kIdXtn2_v + 1 }, - { Inst::kIdYield , Inst::kIdYield + 1 }, - { Inst::kIdZip1_v , Inst::kIdZip2_v + 1 } -}; // ---------------------------------------------------------------------------- // ${NameData:End} #endif // !ASMJIT_NO_TEXT diff --git a/pe-packer/asmjit/arm/a64instdb.h b/pe-packer/asmjit/arm/a64instdb.h index d0be3ca..50e4178 100644 --- a/pe-packer/asmjit/arm/a64instdb.h +++ b/pe-packer/asmjit/arm/a64instdb.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTDB_H_INCLUDED @@ -29,7 +29,7 @@ enum InstFlags : uint32_t { //! SIMD element access of half-words can only be used with v0..15. kInstFlagVH0_15 = 0x00000010u, - //! Instruction may consecutive registers if the number of operands is greater than 2. + //! Instruction uses consecutive registers if the number of operands is greater than 2. kInstFlagConsecutive = 0x00000080u }; @@ -47,9 +47,13 @@ struct InstInfo { //! \name Accessors //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t rwInfoIndex() const noexcept { return _rwInfoIndex; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t flags() const noexcept { return _flags; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(uint32_t flag) const { return (_flags & flag) != 0; } //! \} @@ -57,6 +61,7 @@ struct InstInfo { ASMJIT_VARAPI const InstInfo _instInfoTable[]; +[[nodiscard]] static inline const InstInfo& infoById(InstId instId) noexcept { instId &= uint32_t(InstIdParts::kRealId); ASMJIT_ASSERT(Inst::isDefinedId(instId)); diff --git a/pe-packer/asmjit/arm/a64instdb_p.h b/pe-packer/asmjit/arm/a64instdb_p.h index 9a6edf1..2c971e1 100644 --- a/pe-packer/asmjit/arm/a64instdb_p.h +++ b/pe-packer/asmjit/arm/a64instdb_p.h @@ -1,12 +1,13 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64INSTDB_H_P_INCLUDED #define ASMJIT_ARM_A64INSTDB_H_P_INCLUDED #include "../core/codeholder.h" +#include "../core/instdb_p.h" #include "../arm/a64instdb.h" #include "../arm/a64operand.h" @@ -58,14 +59,14 @@ enum RWInfoType : uint32_t { // a64::InstDB - ElementType // ========================= -enum ElementType : uint8_t { - kET_None = Vec::kElementTypeNone, - kET_B = Vec::kElementTypeB, - kET_H = Vec::kElementTypeH, - kET_S = Vec::kElementTypeS, - kET_D = Vec::kElementTypeD, - kET_2H = Vec::kElementTypeH2, - kET_4B = Vec::kElementTypeB4 +enum InstElementType : uint8_t { + kET_None = uint8_t(VecElementType::kNone), + kET_B = uint8_t(VecElementType::kB), + kET_H = uint8_t(VecElementType::kH), + kET_S = uint8_t(VecElementType::kS), + kET_D = uint8_t(VecElementType::kD), + kET_2H = uint8_t(VecElementType::kH2), + kET_4B = uint8_t(VecElementType::kB4) }; // a64::InstDB - GpType @@ -81,23 +82,22 @@ enum GpType : uint8_t { // =================== enum kOpSignature : uint32_t { - kOp_GpW = GpW::kSignature, - kOp_GpX = GpX::kSignature, + kOp_GpW = RegTraits::kSignature, + kOp_GpX = RegTraits::kSignature, + kOp_B = RegTraits::kSignature, + kOp_H = RegTraits::kSignature, + kOp_S = RegTraits::kSignature, + kOp_D = RegTraits::kSignature, + kOp_Q = RegTraits::kSignature, - kOp_B = VecB::kSignature, - kOp_H = VecH::kSignature, - kOp_S = VecS::kSignature, - kOp_D = VecD::kSignature, - kOp_Q = VecV::kSignature, + kOp_V8B = kOp_D | Vec::kSignatureElementB, + kOp_V4H = kOp_D | Vec::kSignatureElementH, + kOp_V2S = kOp_D | Vec::kSignatureElementS, - kOp_V8B = VecD::kSignature | Vec::kSignatureElementB, - kOp_V4H = VecD::kSignature | Vec::kSignatureElementH, - kOp_V2S = VecD::kSignature | Vec::kSignatureElementS, - - kOp_V16B = VecV::kSignature | Vec::kSignatureElementB, - kOp_V8H = VecV::kSignature | Vec::kSignatureElementH, - kOp_V4S = VecV::kSignature | Vec::kSignatureElementS, - kOp_V2D = VecV::kSignature | Vec::kSignatureElementD + kOp_V16B = kOp_Q | Vec::kSignatureElementB, + kOp_V8H = kOp_Q | Vec::kSignatureElementH, + kOp_V4S = kOp_Q | Vec::kSignatureElementS, + kOp_V2D = kOp_Q | Vec::kSignatureElementD }; // a64::InstDB - HFConv @@ -185,6 +185,7 @@ enum EncodingId : uint32_t { kEncodingBaseLdpStp, kEncodingBaseLdxp, kEncodingBaseLogical, + kEncodingBaseMinMax, kEncodingBaseMov, kEncodingBaseMovKNZ, kEncodingBaseMrs, @@ -192,6 +193,8 @@ enum EncodingId : uint32_t { kEncodingBaseMvnNeg, kEncodingBaseOp, kEncodingBaseOpImm, + kEncodingBaseOpX16, + kEncodingBasePrfm, kEncodingBaseR, kEncodingBaseRM_NoImm, kEncodingBaseRM_SImm10, @@ -262,12 +265,16 @@ namespace EncodingData { #define M_OPCODE(field, bits) \ uint32_t _##field : bits; \ - ASMJIT_INLINE_NODEBUG constexpr uint32_t field() const noexcept { return uint32_t(_##field) << (32 - bits); } + ASMJIT_INLINE_CONSTEXPR uint32_t field() const noexcept { return uint32_t(_##field) << (32 - bits); } struct BaseOp { uint32_t opcode; }; +struct BaseOpX16 { + uint32_t opcode; +}; + struct BaseOpImm { uint32_t opcode; uint16_t immBits; @@ -340,6 +347,11 @@ struct BaseAdcSbc { uint32_t opcode; }; +struct BaseMinMax { + uint32_t regOp; + uint32_t immOp; +}; + struct BaseAddSub { uint32_t shiftedOp : 10; // sf|.......|Sh|.|Rm| Imm:6 |Rn|Rd| uint32_t extendedOp : 10; // sf|.......|..|.|Rm|Opt|Imm3|Rn|Rd| @@ -412,6 +424,13 @@ struct BaseRM_SImm10 { uint32_t immShift : 4; }; +struct BasePrfm { + uint32_t registerOp : 11; + uint32_t sOffsetOp : 10; + uint32_t uOffsetOp : 11; + uint32_t literalOp; +}; + struct BaseLdSt { uint32_t uOffsetOp : 10; uint32_t prePostOp : 11; @@ -468,20 +487,20 @@ struct BaseAtomicCasp { uint32_t xOffset : 5; }; -typedef BaseOp BaseBranchReg; -typedef BaseOp BaseBranchRel; -typedef BaseOp BaseBranchCmp; -typedef BaseOp BaseBranchTst; -typedef BaseOp BaseExtract; -typedef BaseOp BaseBfc; -typedef BaseOp BaseBfi; -typedef BaseOp BaseBfx; -typedef BaseOp BaseCCmp; -typedef BaseOp BaseCInc; -typedef BaseOp BaseCSet; -typedef BaseOp BaseCSel; -typedef BaseOp BaseMovKNZ; -typedef BaseOp BaseMull; +using BaseBranchReg = BaseOp; +using BaseBranchRel = BaseOp; +using BaseBranchCmp = BaseOp; +using BaseBranchTst = BaseOp; +using BaseExtract = BaseOp; +using BaseBfc = BaseOp; +using BaseBfi = BaseOp; +using BaseBfx = BaseOp; +using BaseCCmp = BaseOp; +using BaseCInc = BaseOp; +using BaseCSet = BaseOp; +using BaseCSel = BaseOp; +using BaseMovKNZ = BaseOp; +using BaseMull = BaseOp; struct FSimdGeneric { uint32_t _scalarOp : 28; @@ -495,9 +514,9 @@ struct FSimdGeneric { constexpr uint32_t vectorHf() const noexcept { return uint32_t(_vectorHf); } }; -typedef FSimdGeneric FSimdVV; -typedef FSimdGeneric FSimdVVV; -typedef FSimdGeneric FSimdVVVV; +using FSimdVV = FSimdGeneric; +using FSimdVVV = FSimdGeneric; +using FSimdVVVV = FSimdGeneric; struct FSimdSV { uint32_t opcode; @@ -770,7 +789,7 @@ extern const BaseBfm baseBfm[3]; extern const BaseBfx baseBfx[3]; extern const BaseBranchCmp baseBranchCmp[2]; extern const BaseBranchReg baseBranchReg[3]; -extern const BaseBranchRel baseBranchRel[2]; +extern const BaseBranchRel baseBranchRel[3]; extern const BaseBranchTst baseBranchTst[2]; extern const BaseCCmp baseCCmp[2]; extern const BaseCInc baseCInc[3]; @@ -783,15 +802,18 @@ extern const BaseLdSt baseLdSt[9]; extern const BaseLdpStp baseLdpStp[6]; extern const BaseLdxp baseLdxp[2]; extern const BaseLogical baseLogical[8]; +extern const BaseMinMax baseMinMax[4]; extern const BaseMovKNZ baseMovKNZ[3]; extern const BaseMvnNeg baseMvnNeg[3]; -extern const BaseOp baseOp[23]; -extern const BaseOpImm baseOpImm[14]; +extern const BaseOp baseOp[24]; +extern const BaseOpImm baseOpImm[15]; +extern const BaseOpX16 baseOpX16[1]; +extern const BasePrfm basePrfm[1]; extern const BaseR baseR[10]; extern const BaseRM_NoImm baseRM_NoImm[21]; extern const BaseRM_SImm10 baseRM_SImm10[2]; extern const BaseRM_SImm9 baseRM_SImm9[23]; -extern const BaseRR baseRR[15]; +extern const BaseRR baseRR[18]; extern const BaseRRII baseRRII[2]; extern const BaseRRR baseRRR[26]; extern const BaseRRRR baseRRRR[6]; @@ -843,27 +865,13 @@ extern const SimdTblTbx simdTblTbx[2]; } // {EncodingData} -// a64::InstDB - InstNameIndex -// =========================== - -// ${NameLimits:Begin} -// ------------------- Automatically generated, do not edit ------------------- -enum : uint32_t { kMaxNameSize = 9 }; -// ---------------------------------------------------------------------------- -// ${NameLimits:End} - -struct InstNameIndex { - uint16_t start; - uint16_t end; -}; - // a64::InstDB - Tables // ==================== #ifndef ASMJIT_NO_TEXT -extern const uint32_t _instNameIndexTable[]; +extern const InstNameIndex instNameIndex; extern const char _instNameStringTable[]; -extern const InstNameIndex instNameIndex[26]; +extern const uint32_t _instNameIndexTable[]; #endif // !ASMJIT_NO_TEXT } // {InstDB} diff --git a/pe-packer/asmjit/arm/a64operand.cpp b/pe-packer/asmjit/arm/a64operand.cpp index 3dba96d..b2c2949 100644 --- a/pe-packer/asmjit/arm/a64operand.cpp +++ b/pe-packer/asmjit/arm/a64operand.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -32,49 +32,49 @@ UNIT(a64_operand) { EXPECT_EQ(sp.id(), Gp::kIdSp); EXPECT_EQ(w0.size(), 4u); EXPECT_EQ(x0.size(), 8u); - EXPECT_EQ(w0.type(), RegType::kARM_GpW); - EXPECT_EQ(x0.type(), RegType::kARM_GpX); - EXPECT_EQ(w0.group(), RegGroup::kGp); - EXPECT_EQ(x0.group(), RegGroup::kGp); + EXPECT_EQ(w0.regType(), RegType::kGp32); + EXPECT_EQ(x0.regType(), RegType::kGp64); + EXPECT_EQ(w0.regGroup(), RegGroup::kGp); + EXPECT_EQ(x0.regGroup(), RegGroup::kGp); INFO("Checking Vec register properties"); - EXPECT_EQ(v0.type(), RegType::kARM_VecV); - EXPECT_EQ(d0.type(), RegType::kARM_VecD); - EXPECT_EQ(s0.type(), RegType::kARM_VecS); - EXPECT_EQ(h0.type(), RegType::kARM_VecH); - EXPECT_EQ(b0.type(), RegType::kARM_VecB); + EXPECT_EQ(v0.regType(), RegType::kVec128); + EXPECT_EQ(d0.regType(), RegType::kVec64); + EXPECT_EQ(s0.regType(), RegType::kVec32); + EXPECT_EQ(h0.regType(), RegType::kVec16); + EXPECT_EQ(b0.regType(), RegType::kVec8); - EXPECT_EQ(v0.group(), RegGroup::kVec); - EXPECT_EQ(d0.group(), RegGroup::kVec); - EXPECT_EQ(s0.group(), RegGroup::kVec); - EXPECT_EQ(h0.group(), RegGroup::kVec); - EXPECT_EQ(b0.group(), RegGroup::kVec); + EXPECT_EQ(v0.regGroup(), RegGroup::kVec); + EXPECT_EQ(d0.regGroup(), RegGroup::kVec); + EXPECT_EQ(s0.regGroup(), RegGroup::kVec); + EXPECT_EQ(h0.regGroup(), RegGroup::kVec); + EXPECT_EQ(b0.regGroup(), RegGroup::kVec); INFO("Checking Vec register element[] access"); Vec vd_1 = v15.d(1); - EXPECT_EQ(vd_1.type(), RegType::kARM_VecV); - EXPECT_EQ(vd_1.group(), RegGroup::kVec); + EXPECT_EQ(vd_1.regType(), RegType::kVec128); + EXPECT_EQ(vd_1.regGroup(), RegGroup::kVec); EXPECT_EQ(vd_1.id(), 15u); EXPECT_TRUE(vd_1.isVecD2()); - EXPECT_EQ(vd_1.elementType(), Vec::kElementTypeD); + EXPECT_EQ(vd_1.elementType(), VecElementType::kD); EXPECT_TRUE(vd_1.hasElementIndex()); EXPECT_EQ(vd_1.elementIndex(), 1u); Vec vs_3 = v15.s(3); - EXPECT_EQ(vs_3.type(), RegType::kARM_VecV); - EXPECT_EQ(vs_3.group(), RegGroup::kVec); + EXPECT_EQ(vs_3.regType(), RegType::kVec128); + EXPECT_EQ(vs_3.regGroup(), RegGroup::kVec); EXPECT_EQ(vs_3.id(), 15u); EXPECT_TRUE(vs_3.isVecS4()); - EXPECT_EQ(vs_3.elementType(), Vec::kElementTypeS); + EXPECT_EQ(vs_3.elementType(), VecElementType::kS); EXPECT_TRUE(vs_3.hasElementIndex()); EXPECT_EQ(vs_3.elementIndex(), 3u); Vec vb_4 = v15.b4(3); - EXPECT_EQ(vb_4.type(), RegType::kARM_VecV); - EXPECT_EQ(vb_4.group(), RegGroup::kVec); + EXPECT_EQ(vb_4.regType(), RegType::kVec128); + EXPECT_EQ(vb_4.regGroup(), RegGroup::kVec); EXPECT_EQ(vb_4.id(), 15u); EXPECT_TRUE(vb_4.isVecB4x4()); - EXPECT_EQ(vb_4.elementType(), Vec::kElementTypeB4); + EXPECT_EQ(vb_4.elementType(), VecElementType::kB4); EXPECT_TRUE(vb_4.hasElementIndex()); EXPECT_EQ(vb_4.elementIndex(), 3u); } diff --git a/pe-packer/asmjit/arm/a64operand.h b/pe-packer/asmjit/arm/a64operand.h index c2d3c17..2f5573b 100644 --- a/pe-packer/asmjit/arm/a64operand.h +++ b/pe-packer/asmjit/arm/a64operand.h @@ -1,302 +1,983 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64OPERAND_H_INCLUDED #define ASMJIT_ARM_A64OPERAND_H_INCLUDED -#include "../arm/armoperand.h" +#include "../core/operand.h" +#include "../arm/armglobals.h" ASMJIT_BEGIN_SUB_NAMESPACE(a64) //! \addtogroup asmjit_a64 //! \{ -using arm::Reg; -using arm::Mem; -using arm::Gp; -using arm::GpW; -using arm::GpX; +//! General purpose register (AArch64). +class Gp : public UniGp { +public: + ASMJIT_DEFINE_ABSTRACT_REG(Gp, UniGp) -using arm::Vec; -using arm::VecB; -using arm::VecH; -using arm::VecS; -using arm::VecD; -using arm::VecV; + //! \name Constants + //! \{ + + //! Special register id. + enum Id : uint32_t { + //! Register that depends on OS, could be used as TLS offset. + kIdOs = 18, + //! Frame pointer register id. + kIdFp = 29, + //! Link register id. + kIdLr = 30, + //! Stack register id. + kIdSp = 31, + //! Zero register id. + //! + //! Although zero register has the same id as stack register it has a special treatment, because we need to be + //! able to distinguish between these two at API level. Some instructions were designed to be used with SP and + //! some other with ZR - so we need a way to distinguish these two to make sure we emit the right thing. + //! + //! The number 63 is not random, when you perform `id & 31` you would always get 31 for both SP and ZR inputs, + //! which is the identifier used by AArch64 ISA to encode either SP or ZR depending on the instruction. + kIdZr = 63 + }; + + //! \} + + //! \name Static Constructors + //! \{ + + //! Creates a new 32-bit low general purpose register (W) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_r32(uint32_t regId) noexcept { return Gp(_signatureOf(), regId); } + + //! Creates a new 64-bit low general purpose register (X) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_r64(uint32_t regId) noexcept { return Gp(_signatureOf(), regId); } + + //! Creates a new 32-bit low general purpose register (W) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_w(uint32_t regId) noexcept { return make_r32(regId); } + + //! Creates a new 64-bit low general purpose register (X) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Gp make_x(uint32_t regId) noexcept { return make_r64(regId); } + + //! \} + + //! \name Gp Register Accessors + //! \{ + + //! Test whether this register is ZR register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isZR() const noexcept { return id() == kIdZr; } + + //! Test whether this register is SP register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSP() const noexcept { return id() == kIdSp; } + + //! Clones and casts this register to a 32-bit (W) register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Gp r32() const noexcept { return make_r32(id()); } + + //! Clones and casts this register to a 64-bit (X) register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Gp r64() const noexcept { return make_r64(id()); } + + //! Clones and casts this register to a 32-bit (W) register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Gp w() const noexcept { return r32(); } + + //! Clones and casts this register to a 64-bit (X) register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Gp x() const noexcept { return r64(); } + + //! \} +}; + +//! Vector element type (AArch64). +enum class VecElementType : uint32_t { + //! No element type specified. + kNone = 0, + //! Byte elements (B8 or B16). + kB, + //! Halfword elements (H4 or H8). + kH, + //! Singleword elements (S2 or S4). + kS, + //! Doubleword elements (D2). + kD, + //! Byte elements grouped by 4 bytes (B4). + //! + //! \note This element-type is only used by few instructions. + kB4, + //! Halfword elements grouped by 2 halfwords (H2). + //! + //! \note This element-type is only used by few instructions. + kH2, + + //! Maximum value of \ref VecElementType + kMaxValue = kH2 +}; + +//! Vector register (AArch64). +class Vec : public UniVec { +public: + ASMJIT_DEFINE_ABSTRACT_REG(Vec, UniVec) + + //! \cond + + // Register element type (3 bits). + // |........|........|.XXX....|........| + static inline constexpr uint32_t kSignatureRegElementTypeShift = 12; + static inline constexpr uint32_t kSignatureRegElementTypeMask = 0x07 << kSignatureRegElementTypeShift; + + // Register has element index (1 bit). + // |........|........|X.......|........| + static inline constexpr uint32_t kSignatureRegElementFlagShift = 15; + static inline constexpr uint32_t kSignatureRegElementFlagMask = 0x01 << kSignatureRegElementFlagShift; + + // Register element index (4 bits). + // |........|....XXXX|........|........| + static inline constexpr uint32_t kSignatureRegElementIndexShift = 16; + static inline constexpr uint32_t kSignatureRegElementIndexMask = 0x0F << kSignatureRegElementIndexShift; + + static inline constexpr uint32_t kSignatureElementB = uint32_t(VecElementType::kB) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementH = uint32_t(VecElementType::kH) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementS = uint32_t(VecElementType::kS) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementD = uint32_t(VecElementType::kD) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementB4 = uint32_t(VecElementType::kB4) << kSignatureRegElementTypeShift; + static inline constexpr uint32_t kSignatureElementH2 = uint32_t(VecElementType::kH2) << kSignatureRegElementTypeShift; + + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature _makeElementAccessSignature(VecElementType elementType, uint32_t elementIndex) noexcept { + return OperandSignature{ + uint32_t(RegTraits::kSignature) | + uint32_t(kSignatureRegElementFlagMask) | + (uint32_t(elementType) << kSignatureRegElementTypeShift) | + (uint32_t(elementIndex << kSignatureRegElementIndexShift)) + }; + } + + //! \endcond + + //! \name Static Constructors + //! \{ + + //! Creates a new 8-bit vector register (B) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v8(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 16-bit vector register (H) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v16(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 32-bit vector register (S) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v32(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 64-bit vector register (D) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v64(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 128-bit vector register (Q) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128(uint32_t regId) noexcept { return Vec(_signatureOf(), regId); } + + //! Creates a new 8-bit vector register (B) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_b(uint32_t regId) noexcept { return make_v8(regId); } + + //! Creates a new 16-bit vector register (H) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_h(uint32_t regId) noexcept { return make_v16(regId); } + + //! Creates a new 32-bit vector register (S) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_s(uint32_t regId) noexcept { return make_v32(regId); } + + //! Creates a new 64-bit vector register (D) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_d(uint32_t regId) noexcept { return make_v64(regId); } + + //! Creates a new 128-bit vector register (Q) having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_q(uint32_t regId) noexcept { return make_v128(regId); } + + //! Creates a new 32-bit vector register (S) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v32_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 64-bit vector register (D) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v64_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 128-bit vector register (Q) having the given vector `elementType` and register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128_with_element_type(VecElementType elementType, uint32_t regId) noexcept { + uint32_t signature = RegTraits::kSignature | uint32_t(elementType) << kSignatureRegElementTypeShift; + return Vec(OperandSignature{signature}, regId); + } + + //! Creates a new 128-bit vector of type specified by `elementType` and `elementIndex`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR Vec make_v128_with_element_index(VecElementType elementType, uint32_t elementIndex, uint32_t regId) noexcept { + return Vec(_makeElementAccessSignature(elementType, elementIndex), regId); + } + + //! \} + + //! \name Vector Register Accessors + //! \{ + + //! Returns whether the register has element type or element index (or both). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasElementTypeOrIndex() const noexcept { + return _signature.hasField(); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB8() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecS2() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecD1() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB16() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH8() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecS4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecD2() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementD); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecB4x4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB4); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVecH2x4() const noexcept { + return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH2); + } + + //! Clones and casts the register to an 8-bit B register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v8() const noexcept { return make_v8(id()); } + + //! Clones and casts the register to a 16-bit H register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v16() const noexcept { return make_v16(id()); } + + //! Clones and casts the register to a 32-bit S register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v32() const noexcept { return make_v32(id()); } + + //! Clones and casts the register to a 64-bit D register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v64() const noexcept { return make_v64(id()); } + + //! Clones and casts the register to a 128-bit Q register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec v128() const noexcept { return make_v128(id()); } + + //! Clones and casts the register to an 8-bit B register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b() const noexcept { return make_v8(id()); } + + //! Clones and casts the register to a 16-bit H register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h() const noexcept { return make_v16(id()); } + + //! Clones and casts the register to a 32-bit S register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s() const noexcept { return make_v32(id()); } + + //! Clones and casts the register to a 64-bit D register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d() const noexcept { return make_v64(id()); } + + //! Clones and casts the register to a 128-bit Q register (element type & index is not cloned). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec q() const noexcept { return make_v128(id()); } + + //! Clones and casts the register to a 128-bit V.B[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kB, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.H[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kH, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.S[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kS, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.D[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kD, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.H2[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h2(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kH2, elementIndex, id()); } + + //! Clones and casts the register to a 128-bit V.B4[elementIndex] register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b4(uint32_t elementIndex) const noexcept { return make_v128_with_element_index(VecElementType::kB4, elementIndex, id()); } + + //! Clones and casts the register to V.8B. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b8() const noexcept { return make_v64_with_element_type(VecElementType::kB, id()); } + + //! Clones and casts the register to V.16B. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec b16() const noexcept { return make_v128_with_element_type(VecElementType::kB, id()); } + + //! Clones and casts the register to V.2H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h2() const noexcept { return make_v32_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.4H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h4() const noexcept { return make_v64_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.8H. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec h8() const noexcept { return make_v128_with_element_type(VecElementType::kH, id()); } + + //! Clones and casts the register to V.2S. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s2() const noexcept { return make_v64_with_element_type(VecElementType::kS, id()); } + + //! Clones and casts the register to V.4S. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec s4() const noexcept { return make_v128_with_element_type(VecElementType::kS, id()); } + + //! Clones and casts the register to V.2D. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec d2() const noexcept { return make_v128_with_element_type(VecElementType::kD, id()); } + + //! \} + + //! \name Element Type Accessors + //! \{ + + //! Returns whether the vector register has associated a vector element type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasElementType() const noexcept { + return _signature.hasField(); + } + + //! Returns vector element type of the register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR VecElementType elementType() const noexcept { + return VecElementType(_signature.getField()); + } + + //! Sets vector element type of the register to `elementType`. + ASMJIT_INLINE_CONSTEXPR void setElementType(VecElementType elementType) noexcept { + _signature.setField(uint32_t(elementType)); + } + + //! Resets vector element type to none. + ASMJIT_INLINE_CONSTEXPR void resetElementType() noexcept { + _signature.setField(0); + } + + //! \} + + //! \name Element Index Accessors + //! \{ + + //! Returns whether the register has element index (it's an element index access). + ASMJIT_INLINE_CONSTEXPR bool hasElementIndex() const noexcept { + return _signature.hasField(); + } + + //! Returns element index of the register. + ASMJIT_INLINE_CONSTEXPR uint32_t elementIndex() const noexcept { + return _signature.getField(); + } + + //! Sets element index of the register to `elementType`. + ASMJIT_INLINE_CONSTEXPR void setElementIndex(uint32_t elementIndex) noexcept { + _signature |= kSignatureRegElementFlagMask; + _signature.setField(elementIndex); + } + + //! Resets element index of the register. + ASMJIT_INLINE_CONSTEXPR void resetElementIndex() noexcept { + _signature &= ~(kSignatureRegElementFlagMask | kSignatureRegElementIndexMask); + } + + //! Clones a vector register with element access enabled at the given `elementIndex`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Vec at(uint32_t elementIndex) const noexcept { + return Vec((signature() & ~kSignatureRegElementIndexMask) | (elementIndex << kSignatureRegElementIndexShift) | kSignatureRegElementFlagMask, id()); + } + + //! \} +}; + +//! Memory operand (AArch64). +class Mem : public BaseMem { +public: + //! \cond INTERNAL + + // Index shift value (5 bits). + // |........|.....XXX|XX......|........| + static inline constexpr uint32_t kSignatureMemShiftValueShift = 14; + static inline constexpr uint32_t kSignatureMemShiftValueMask = 0x1Fu << kSignatureMemShiftValueShift; + + // Index shift operation (4 bits). + // |........|XXXX....|........|........| + static inline constexpr uint32_t kSignatureMemShiftOpShift = 20; + static inline constexpr uint32_t kSignatureMemShiftOpMask = 0x0Fu << kSignatureMemShiftOpShift; + + // Offset mode type (2 bits). + // |......XX|........|........|........| + static inline constexpr uint32_t kSignatureMemOffsetModeShift = 24; + static inline constexpr uint32_t kSignatureMemOffsetModeMask = 0x03u << kSignatureMemOffsetModeShift; + + //! \endcond + + //! \name Construction & Destruction + //! \{ + + //! Construct a default `Mem` operand, that points to [0]. + ASMJIT_INLINE_CONSTEXPR Mem() noexcept + : BaseMem() {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Mem& other) noexcept + : BaseMem(other) {} + + ASMJIT_INLINE_NODEBUG explicit Mem(Globals::NoInit_) noexcept + : BaseMem(Globals::NoInit) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Signature& signature, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept + : BaseMem(signature, baseId, indexId, offset) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(const Label& base, int32_t off = 0, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(RegType::kLabelTag) | + signature, base.id(), 0, off) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(const Reg& base, int32_t off = 0, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + signature, base.id(), 0, off) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Reg& base, const Reg& index, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + Signature::fromMemIndexType(index.regType()) | + signature, base.id(), index.id(), 0) {} + + ASMJIT_INLINE_CONSTEXPR Mem(const Reg& base, const Reg& index, const Shift& shift, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + Signature::fromMemBaseType(base.regType()) | + Signature::fromMemIndexType(index.regType()) | + Signature::fromValue(uint32_t(shift.op())) | + Signature::fromValue(shift.value()) | + signature, base.id(), index.id(), 0) {} + + ASMJIT_INLINE_CONSTEXPR explicit Mem(uint64_t base, Signature signature = Signature{0}) noexcept + : BaseMem(Signature::fromOpType(OperandType::kMem) | + signature, uint32_t(base >> 32), 0, int32_t(uint32_t(base & 0xFFFFFFFFu))) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + ASMJIT_INLINE_CONSTEXPR Mem& operator=(const Mem& other) noexcept { + copyFrom(other); + return *this; + } + + //! \} + + //! \name Clone + //! \{ + + //! Clones the memory operand. + ASMJIT_INLINE_CONSTEXPR Mem clone() const noexcept { return Mem(*this); } + + //! Gets new memory operand adjusted by `off`. + ASMJIT_INLINE_CONSTEXPR Mem cloneAdjusted(int64_t off) const noexcept { + Mem result(*this); + result.addOffset(off); + return result; + } + + //! Clones the memory operand and makes it pre-index. + ASMJIT_INLINE_CONSTEXPR Mem pre() const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPreIndex); + return result; + } + + //! Clones the memory operand, applies a given offset `off` and makes it pre-index. + ASMJIT_INLINE_CONSTEXPR Mem pre(int64_t off) const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPreIndex); + result.addOffset(off); + return result; + } + + //! Clones the memory operand and makes it post-index. + ASMJIT_INLINE_CONSTEXPR Mem post() const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPostIndex); + return result; + } + + //! Clones the memory operand, applies a given offset `off` and makes it post-index. + ASMJIT_INLINE_CONSTEXPR Mem post(int64_t off) const noexcept { + Mem result(*this); + result.setOffsetMode(OffsetMode::kPostIndex); + result.addOffset(off); + return result; + } + + //! \} + + //! \name Base & Index + //! \{ + + //! Converts memory `baseType` and `baseId` to `arm::Reg` instance. + //! + //! The memory must have a valid base register otherwise the result will be wrong. + ASMJIT_INLINE_NODEBUG Reg baseReg() const noexcept { return Reg::fromTypeAndId(baseType(), baseId()); } + + //! Converts memory `indexType` and `indexId` to `arm::Reg` instance. + //! + //! The memory must have a valid index register otherwise the result will be wrong. + ASMJIT_INLINE_NODEBUG Reg indexReg() const noexcept { return Reg::fromTypeAndId(indexType(), indexId()); } + + using BaseMem::setIndex; + + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index, uint32_t shift) noexcept { + setIndex(index); + setShift(shift); + } + + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index, Shift shift) noexcept { + setIndex(index); + setShift(shift); + } + + //! \} + + //! \name ARM Specific Features + //! \{ + + //! Gets offset mode. + ASMJIT_INLINE_CONSTEXPR OffsetMode offsetMode() const noexcept { return OffsetMode(_signature.getField()); } + //! Sets offset mode to `mode`. + ASMJIT_INLINE_CONSTEXPR void setOffsetMode(OffsetMode mode) noexcept { _signature.setField(uint32_t(mode)); } + //! Resets offset mode to default (fixed offset, without write-back). + ASMJIT_INLINE_CONSTEXPR void resetOffsetMode() noexcept { _signature.setField(uint32_t(OffsetMode::kFixed)); } + + //! Tests whether the current memory offset mode is fixed (see \ref OffsetMode::kFixed). + ASMJIT_INLINE_CONSTEXPR bool isFixedOffset() const noexcept { return offsetMode() == OffsetMode::kFixed; } + //! Tests whether the current memory offset mode is either pre-index or post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPreOrPost() const noexcept { return offsetMode() != OffsetMode::kFixed; } + //! Tests whether the current memory offset mode is pre-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPreIndex() const noexcept { return offsetMode() == OffsetMode::kPreIndex; } + //! Tests whether the current memory offset mode is post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR bool isPostIndex() const noexcept { return offsetMode() == OffsetMode::kPostIndex; } + + //! Sets offset mode of this memory operand to pre-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR void makePreIndex() noexcept { setOffsetMode(OffsetMode::kPreIndex); } + //! Sets offset mode of this memory operand to post-index (write-back is used). + ASMJIT_INLINE_CONSTEXPR void makePostIndex() noexcept { setOffsetMode(OffsetMode::kPostIndex); } + + //! Gets shift operation that is used by index register. + ASMJIT_INLINE_CONSTEXPR ShiftOp shiftOp() const noexcept { return ShiftOp(_signature.getField()); } + //! Sets shift operation that is used by index register. + ASMJIT_INLINE_CONSTEXPR void setShiftOp(ShiftOp sop) noexcept { _signature.setField(uint32_t(sop)); } + //! Resets shift operation that is used by index register to LSL (default value). + ASMJIT_INLINE_CONSTEXPR void resetShiftOp() noexcept { _signature.setField(uint32_t(ShiftOp::kLSL)); } + + //! Gets whether the memory operand has shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR bool hasShift() const noexcept { return _signature.hasField(); } + //! Gets the memory operand's shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR uint32_t shift() const noexcept { return _signature.getField(); } + //! Sets the memory operand's shift (aka scale) constant. + ASMJIT_INLINE_CONSTEXPR void setShift(uint32_t shift) noexcept { _signature.setField(shift); } + + //! Sets the memory operand's shift and shift operation. + ASMJIT_INLINE_CONSTEXPR void setShift(Shift shift) noexcept { + _signature.setField(uint32_t(shift.op())); + _signature.setField(shift.value()); + } + + //! Resets the memory operand's shift (aka scale) constant to zero. + ASMJIT_INLINE_CONSTEXPR void resetShift() noexcept { _signature.setField(0); } + + //! \} +}; + +//! \name Shift Operation Construction +//! \{ + +//! Constructs a `LSL #value` shift (logical shift left). +static ASMJIT_INLINE_CONSTEXPR Shift lsl(uint32_t value) noexcept { return Shift(ShiftOp::kLSL, value); } +//! Constructs a `LSR #value` shift (logical shift right). +static ASMJIT_INLINE_CONSTEXPR Shift lsr(uint32_t value) noexcept { return Shift(ShiftOp::kLSR, value); } +//! Constructs a `ASR #value` shift (arithmetic shift right). +static ASMJIT_INLINE_CONSTEXPR Shift asr(uint32_t value) noexcept { return Shift(ShiftOp::kASR, value); } +//! Constructs a `ROR #value` shift (rotate right). +static ASMJIT_INLINE_CONSTEXPR Shift ror(uint32_t value) noexcept { return Shift(ShiftOp::kROR, value); } +//! Constructs a `RRX` shift (rotate with carry by 1). +static ASMJIT_INLINE_CONSTEXPR Shift rrx() noexcept { return Shift(ShiftOp::kRRX, 0); } +//! Constructs a `MSL #value` shift (logical shift left filling ones). +static ASMJIT_INLINE_CONSTEXPR Shift msl(uint32_t value) noexcept { return Shift(ShiftOp::kMSL, value); } + +//! \} #ifndef _DOXYGEN namespace regs { #endif -using namespace ::asmjit::arm::regs; +//! Creates a 32-bit W register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Gp w(uint32_t id) noexcept { return Gp::make_r32(id); } -static constexpr GpW w0 = GpW(0); -static constexpr GpW w1 = GpW(1); -static constexpr GpW w2 = GpW(2); -static constexpr GpW w3 = GpW(3); -static constexpr GpW w4 = GpW(4); -static constexpr GpW w5 = GpW(5); -static constexpr GpW w6 = GpW(6); -static constexpr GpW w7 = GpW(7); -static constexpr GpW w8 = GpW(8); -static constexpr GpW w9 = GpW(9); -static constexpr GpW w10 = GpW(10); -static constexpr GpW w11 = GpW(11); -static constexpr GpW w12 = GpW(12); -static constexpr GpW w13 = GpW(13); -static constexpr GpW w14 = GpW(14); -static constexpr GpW w15 = GpW(15); -static constexpr GpW w16 = GpW(16); -static constexpr GpW w17 = GpW(17); -static constexpr GpW w18 = GpW(18); -static constexpr GpW w19 = GpW(19); -static constexpr GpW w20 = GpW(20); -static constexpr GpW w21 = GpW(21); -static constexpr GpW w22 = GpW(22); -static constexpr GpW w23 = GpW(23); -static constexpr GpW w24 = GpW(24); -static constexpr GpW w25 = GpW(25); -static constexpr GpW w26 = GpW(26); -static constexpr GpW w27 = GpW(27); -static constexpr GpW w28 = GpW(28); -static constexpr GpW w29 = GpW(29); -static constexpr GpW w30 = GpW(30); -static constexpr GpW wzr = GpW(Gp::kIdZr); -static constexpr GpW wsp = GpW(Gp::kIdSp); +//! Creates a 32-bit W register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Gp gp32(uint32_t id) noexcept { return Gp::make_r32(id); } -static constexpr GpX x0 = GpX(0); -static constexpr GpX x1 = GpX(1); -static constexpr GpX x2 = GpX(2); -static constexpr GpX x3 = GpX(3); -static constexpr GpX x4 = GpX(4); -static constexpr GpX x5 = GpX(5); -static constexpr GpX x6 = GpX(6); -static constexpr GpX x7 = GpX(7); -static constexpr GpX x8 = GpX(8); -static constexpr GpX x9 = GpX(9); -static constexpr GpX x10 = GpX(10); -static constexpr GpX x11 = GpX(11); -static constexpr GpX x12 = GpX(12); -static constexpr GpX x13 = GpX(13); -static constexpr GpX x14 = GpX(14); -static constexpr GpX x15 = GpX(15); -static constexpr GpX x16 = GpX(16); -static constexpr GpX x17 = GpX(17); -static constexpr GpX x18 = GpX(18); -static constexpr GpX x19 = GpX(19); -static constexpr GpX x20 = GpX(20); -static constexpr GpX x21 = GpX(21); -static constexpr GpX x22 = GpX(22); -static constexpr GpX x23 = GpX(23); -static constexpr GpX x24 = GpX(24); -static constexpr GpX x25 = GpX(25); -static constexpr GpX x26 = GpX(26); -static constexpr GpX x27 = GpX(27); -static constexpr GpX x28 = GpX(28); -static constexpr GpX x29 = GpX(29); -static constexpr GpX x30 = GpX(30); -static constexpr GpX xzr = GpX(Gp::kIdZr); -static constexpr GpX sp = GpX(Gp::kIdSp); +//! Creates a 64-bit X register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Gp x(uint32_t id) noexcept { return Gp::make_r64(id); } -static constexpr VecB b0 = VecB(0); -static constexpr VecB b1 = VecB(1); -static constexpr VecB b2 = VecB(2); -static constexpr VecB b3 = VecB(3); -static constexpr VecB b4 = VecB(4); -static constexpr VecB b5 = VecB(5); -static constexpr VecB b6 = VecB(6); -static constexpr VecB b7 = VecB(7); -static constexpr VecB b8 = VecB(8); -static constexpr VecB b9 = VecB(9); -static constexpr VecB b10 = VecB(10); -static constexpr VecB b11 = VecB(11); -static constexpr VecB b12 = VecB(12); -static constexpr VecB b13 = VecB(13); -static constexpr VecB b14 = VecB(14); -static constexpr VecB b15 = VecB(15); -static constexpr VecB b16 = VecB(16); -static constexpr VecB b17 = VecB(17); -static constexpr VecB b18 = VecB(18); -static constexpr VecB b19 = VecB(19); -static constexpr VecB b20 = VecB(20); -static constexpr VecB b21 = VecB(21); -static constexpr VecB b22 = VecB(22); -static constexpr VecB b23 = VecB(23); -static constexpr VecB b24 = VecB(24); -static constexpr VecB b25 = VecB(25); -static constexpr VecB b26 = VecB(26); -static constexpr VecB b27 = VecB(27); -static constexpr VecB b28 = VecB(28); -static constexpr VecB b29 = VecB(29); -static constexpr VecB b30 = VecB(30); -static constexpr VecB b31 = VecB(31); +//! Creates a 64-bit X register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Gp gp64(uint32_t id) noexcept { return Gp::make_r64(id); } -static constexpr VecH h0 = VecH(0); -static constexpr VecH h1 = VecH(1); -static constexpr VecH h2 = VecH(2); -static constexpr VecH h3 = VecH(3); -static constexpr VecH h4 = VecH(4); -static constexpr VecH h5 = VecH(5); -static constexpr VecH h6 = VecH(6); -static constexpr VecH h7 = VecH(7); -static constexpr VecH h8 = VecH(8); -static constexpr VecH h9 = VecH(9); -static constexpr VecH h10 = VecH(10); -static constexpr VecH h11 = VecH(11); -static constexpr VecH h12 = VecH(12); -static constexpr VecH h13 = VecH(13); -static constexpr VecH h14 = VecH(14); -static constexpr VecH h15 = VecH(15); -static constexpr VecH h16 = VecH(16); -static constexpr VecH h17 = VecH(17); -static constexpr VecH h18 = VecH(18); -static constexpr VecH h19 = VecH(19); -static constexpr VecH h20 = VecH(20); -static constexpr VecH h21 = VecH(21); -static constexpr VecH h22 = VecH(22); -static constexpr VecH h23 = VecH(23); -static constexpr VecH h24 = VecH(24); -static constexpr VecH h25 = VecH(25); -static constexpr VecH h26 = VecH(26); -static constexpr VecH h27 = VecH(27); -static constexpr VecH h28 = VecH(28); -static constexpr VecH h29 = VecH(29); -static constexpr VecH h30 = VecH(30); -static constexpr VecH h31 = VecH(31); +//! Creates an 8-bit B register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec b(uint32_t id) noexcept { return Vec::make_v8(id); } -static constexpr VecS s0 = VecS(0); -static constexpr VecS s1 = VecS(1); -static constexpr VecS s2 = VecS(2); -static constexpr VecS s3 = VecS(3); -static constexpr VecS s4 = VecS(4); -static constexpr VecS s5 = VecS(5); -static constexpr VecS s6 = VecS(6); -static constexpr VecS s7 = VecS(7); -static constexpr VecS s8 = VecS(8); -static constexpr VecS s9 = VecS(9); -static constexpr VecS s10 = VecS(10); -static constexpr VecS s11 = VecS(11); -static constexpr VecS s12 = VecS(12); -static constexpr VecS s13 = VecS(13); -static constexpr VecS s14 = VecS(14); -static constexpr VecS s15 = VecS(15); -static constexpr VecS s16 = VecS(16); -static constexpr VecS s17 = VecS(17); -static constexpr VecS s18 = VecS(18); -static constexpr VecS s19 = VecS(19); -static constexpr VecS s20 = VecS(20); -static constexpr VecS s21 = VecS(21); -static constexpr VecS s22 = VecS(22); -static constexpr VecS s23 = VecS(23); -static constexpr VecS s24 = VecS(24); -static constexpr VecS s25 = VecS(25); -static constexpr VecS s26 = VecS(26); -static constexpr VecS s27 = VecS(27); -static constexpr VecS s28 = VecS(28); -static constexpr VecS s29 = VecS(29); -static constexpr VecS s30 = VecS(30); -static constexpr VecS s31 = VecS(31); +//! Creates a 16-bit H register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec h(uint32_t id) noexcept { return Vec::make_v16(id); } -static constexpr VecD d0 = VecD(0); -static constexpr VecD d1 = VecD(1); -static constexpr VecD d2 = VecD(2); -static constexpr VecD d3 = VecD(3); -static constexpr VecD d4 = VecD(4); -static constexpr VecD d5 = VecD(5); -static constexpr VecD d6 = VecD(6); -static constexpr VecD d7 = VecD(7); -static constexpr VecD d8 = VecD(8); -static constexpr VecD d9 = VecD(9); -static constexpr VecD d10 = VecD(10); -static constexpr VecD d11 = VecD(11); -static constexpr VecD d12 = VecD(12); -static constexpr VecD d13 = VecD(13); -static constexpr VecD d14 = VecD(14); -static constexpr VecD d15 = VecD(15); -static constexpr VecD d16 = VecD(16); -static constexpr VecD d17 = VecD(17); -static constexpr VecD d18 = VecD(18); -static constexpr VecD d19 = VecD(19); -static constexpr VecD d20 = VecD(20); -static constexpr VecD d21 = VecD(21); -static constexpr VecD d22 = VecD(22); -static constexpr VecD d23 = VecD(23); -static constexpr VecD d24 = VecD(24); -static constexpr VecD d25 = VecD(25); -static constexpr VecD d26 = VecD(26); -static constexpr VecD d27 = VecD(27); -static constexpr VecD d28 = VecD(28); -static constexpr VecD d29 = VecD(29); -static constexpr VecD d30 = VecD(30); -static constexpr VecD d31 = VecD(31); +//! Creates a 32-bit S register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec s(uint32_t id) noexcept { return Vec::make_v32(id); } -static constexpr VecV q0 = VecV(0); -static constexpr VecV q1 = VecV(1); -static constexpr VecV q2 = VecV(2); -static constexpr VecV q3 = VecV(3); -static constexpr VecV q4 = VecV(4); -static constexpr VecV q5 = VecV(5); -static constexpr VecV q6 = VecV(6); -static constexpr VecV q7 = VecV(7); -static constexpr VecV q8 = VecV(8); -static constexpr VecV q9 = VecV(9); -static constexpr VecV q10 = VecV(10); -static constexpr VecV q11 = VecV(11); -static constexpr VecV q12 = VecV(12); -static constexpr VecV q13 = VecV(13); -static constexpr VecV q14 = VecV(14); -static constexpr VecV q15 = VecV(15); -static constexpr VecV q16 = VecV(16); -static constexpr VecV q17 = VecV(17); -static constexpr VecV q18 = VecV(18); -static constexpr VecV q19 = VecV(19); -static constexpr VecV q20 = VecV(20); -static constexpr VecV q21 = VecV(21); -static constexpr VecV q22 = VecV(22); -static constexpr VecV q23 = VecV(23); -static constexpr VecV q24 = VecV(24); -static constexpr VecV q25 = VecV(25); -static constexpr VecV q26 = VecV(26); -static constexpr VecV q27 = VecV(27); -static constexpr VecV q28 = VecV(28); -static constexpr VecV q29 = VecV(29); -static constexpr VecV q30 = VecV(30); -static constexpr VecV q31 = VecV(31); +//! Creates a 64-bit D register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec d(uint32_t id) noexcept { return Vec::make_v64(id); } -static constexpr VecV v0 = VecV(0); -static constexpr VecV v1 = VecV(1); -static constexpr VecV v2 = VecV(2); -static constexpr VecV v3 = VecV(3); -static constexpr VecV v4 = VecV(4); -static constexpr VecV v5 = VecV(5); -static constexpr VecV v6 = VecV(6); -static constexpr VecV v7 = VecV(7); -static constexpr VecV v8 = VecV(8); -static constexpr VecV v9 = VecV(9); -static constexpr VecV v10 = VecV(10); -static constexpr VecV v11 = VecV(11); -static constexpr VecV v12 = VecV(12); -static constexpr VecV v13 = VecV(13); -static constexpr VecV v14 = VecV(14); -static constexpr VecV v15 = VecV(15); -static constexpr VecV v16 = VecV(16); -static constexpr VecV v17 = VecV(17); -static constexpr VecV v18 = VecV(18); -static constexpr VecV v19 = VecV(19); -static constexpr VecV v20 = VecV(20); -static constexpr VecV v21 = VecV(21); -static constexpr VecV v22 = VecV(22); -static constexpr VecV v23 = VecV(23); -static constexpr VecV v24 = VecV(24); -static constexpr VecV v25 = VecV(25); -static constexpr VecV v26 = VecV(26); -static constexpr VecV v27 = VecV(27); -static constexpr VecV v28 = VecV(28); -static constexpr VecV v29 = VecV(29); -static constexpr VecV v30 = VecV(30); -static constexpr VecV v31 = VecV(31); +//! Creates a 1282-bit V register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec q(uint32_t id) noexcept { return Vec::make_v128(id); } + +//! Creates a 1282-bit V register operand. +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Vec v(uint32_t id) noexcept { return Vec::make_v128(id); } + +static constexpr Gp w0 = Gp::make_r32(0); +static constexpr Gp w1 = Gp::make_r32(1); +static constexpr Gp w2 = Gp::make_r32(2); +static constexpr Gp w3 = Gp::make_r32(3); +static constexpr Gp w4 = Gp::make_r32(4); +static constexpr Gp w5 = Gp::make_r32(5); +static constexpr Gp w6 = Gp::make_r32(6); +static constexpr Gp w7 = Gp::make_r32(7); +static constexpr Gp w8 = Gp::make_r32(8); +static constexpr Gp w9 = Gp::make_r32(9); +static constexpr Gp w10 = Gp::make_r32(10); +static constexpr Gp w11 = Gp::make_r32(11); +static constexpr Gp w12 = Gp::make_r32(12); +static constexpr Gp w13 = Gp::make_r32(13); +static constexpr Gp w14 = Gp::make_r32(14); +static constexpr Gp w15 = Gp::make_r32(15); +static constexpr Gp w16 = Gp::make_r32(16); +static constexpr Gp w17 = Gp::make_r32(17); +static constexpr Gp w18 = Gp::make_r32(18); +static constexpr Gp w19 = Gp::make_r32(19); +static constexpr Gp w20 = Gp::make_r32(20); +static constexpr Gp w21 = Gp::make_r32(21); +static constexpr Gp w22 = Gp::make_r32(22); +static constexpr Gp w23 = Gp::make_r32(23); +static constexpr Gp w24 = Gp::make_r32(24); +static constexpr Gp w25 = Gp::make_r32(25); +static constexpr Gp w26 = Gp::make_r32(26); +static constexpr Gp w27 = Gp::make_r32(27); +static constexpr Gp w28 = Gp::make_r32(28); +static constexpr Gp w29 = Gp::make_r32(29); +static constexpr Gp w30 = Gp::make_r32(30); +static constexpr Gp wzr = Gp::make_r32(Gp::kIdZr); +static constexpr Gp wsp = Gp::make_r32(Gp::kIdSp); + +static constexpr Gp x0 = Gp::make_r64(0); +static constexpr Gp x1 = Gp::make_r64(1); +static constexpr Gp x2 = Gp::make_r64(2); +static constexpr Gp x3 = Gp::make_r64(3); +static constexpr Gp x4 = Gp::make_r64(4); +static constexpr Gp x5 = Gp::make_r64(5); +static constexpr Gp x6 = Gp::make_r64(6); +static constexpr Gp x7 = Gp::make_r64(7); +static constexpr Gp x8 = Gp::make_r64(8); +static constexpr Gp x9 = Gp::make_r64(9); +static constexpr Gp x10 = Gp::make_r64(10); +static constexpr Gp x11 = Gp::make_r64(11); +static constexpr Gp x12 = Gp::make_r64(12); +static constexpr Gp x13 = Gp::make_r64(13); +static constexpr Gp x14 = Gp::make_r64(14); +static constexpr Gp x15 = Gp::make_r64(15); +static constexpr Gp x16 = Gp::make_r64(16); +static constexpr Gp x17 = Gp::make_r64(17); +static constexpr Gp x18 = Gp::make_r64(18); +static constexpr Gp x19 = Gp::make_r64(19); +static constexpr Gp x20 = Gp::make_r64(20); +static constexpr Gp x21 = Gp::make_r64(21); +static constexpr Gp x22 = Gp::make_r64(22); +static constexpr Gp x23 = Gp::make_r64(23); +static constexpr Gp x24 = Gp::make_r64(24); +static constexpr Gp x25 = Gp::make_r64(25); +static constexpr Gp x26 = Gp::make_r64(26); +static constexpr Gp x27 = Gp::make_r64(27); +static constexpr Gp x28 = Gp::make_r64(28); +static constexpr Gp x29 = Gp::make_r64(29); +static constexpr Gp x30 = Gp::make_r64(30); +static constexpr Gp xzr = Gp::make_r64(Gp::kIdZr); +static constexpr Gp sp = Gp::make_r64(Gp::kIdSp); + +static constexpr Vec b0 = Vec::make_v8(0); +static constexpr Vec b1 = Vec::make_v8(1); +static constexpr Vec b2 = Vec::make_v8(2); +static constexpr Vec b3 = Vec::make_v8(3); +static constexpr Vec b4 = Vec::make_v8(4); +static constexpr Vec b5 = Vec::make_v8(5); +static constexpr Vec b6 = Vec::make_v8(6); +static constexpr Vec b7 = Vec::make_v8(7); +static constexpr Vec b8 = Vec::make_v8(8); +static constexpr Vec b9 = Vec::make_v8(9); +static constexpr Vec b10 = Vec::make_v8(10); +static constexpr Vec b11 = Vec::make_v8(11); +static constexpr Vec b12 = Vec::make_v8(12); +static constexpr Vec b13 = Vec::make_v8(13); +static constexpr Vec b14 = Vec::make_v8(14); +static constexpr Vec b15 = Vec::make_v8(15); +static constexpr Vec b16 = Vec::make_v8(16); +static constexpr Vec b17 = Vec::make_v8(17); +static constexpr Vec b18 = Vec::make_v8(18); +static constexpr Vec b19 = Vec::make_v8(19); +static constexpr Vec b20 = Vec::make_v8(20); +static constexpr Vec b21 = Vec::make_v8(21); +static constexpr Vec b22 = Vec::make_v8(22); +static constexpr Vec b23 = Vec::make_v8(23); +static constexpr Vec b24 = Vec::make_v8(24); +static constexpr Vec b25 = Vec::make_v8(25); +static constexpr Vec b26 = Vec::make_v8(26); +static constexpr Vec b27 = Vec::make_v8(27); +static constexpr Vec b28 = Vec::make_v8(28); +static constexpr Vec b29 = Vec::make_v8(29); +static constexpr Vec b30 = Vec::make_v8(30); +static constexpr Vec b31 = Vec::make_v8(31); + +static constexpr Vec h0 = Vec::make_v16(0); +static constexpr Vec h1 = Vec::make_v16(1); +static constexpr Vec h2 = Vec::make_v16(2); +static constexpr Vec h3 = Vec::make_v16(3); +static constexpr Vec h4 = Vec::make_v16(4); +static constexpr Vec h5 = Vec::make_v16(5); +static constexpr Vec h6 = Vec::make_v16(6); +static constexpr Vec h7 = Vec::make_v16(7); +static constexpr Vec h8 = Vec::make_v16(8); +static constexpr Vec h9 = Vec::make_v16(9); +static constexpr Vec h10 = Vec::make_v16(10); +static constexpr Vec h11 = Vec::make_v16(11); +static constexpr Vec h12 = Vec::make_v16(12); +static constexpr Vec h13 = Vec::make_v16(13); +static constexpr Vec h14 = Vec::make_v16(14); +static constexpr Vec h15 = Vec::make_v16(15); +static constexpr Vec h16 = Vec::make_v16(16); +static constexpr Vec h17 = Vec::make_v16(17); +static constexpr Vec h18 = Vec::make_v16(18); +static constexpr Vec h19 = Vec::make_v16(19); +static constexpr Vec h20 = Vec::make_v16(20); +static constexpr Vec h21 = Vec::make_v16(21); +static constexpr Vec h22 = Vec::make_v16(22); +static constexpr Vec h23 = Vec::make_v16(23); +static constexpr Vec h24 = Vec::make_v16(24); +static constexpr Vec h25 = Vec::make_v16(25); +static constexpr Vec h26 = Vec::make_v16(26); +static constexpr Vec h27 = Vec::make_v16(27); +static constexpr Vec h28 = Vec::make_v16(28); +static constexpr Vec h29 = Vec::make_v16(29); +static constexpr Vec h30 = Vec::make_v16(30); +static constexpr Vec h31 = Vec::make_v16(31); + +static constexpr Vec s0 = Vec::make_v32(0); +static constexpr Vec s1 = Vec::make_v32(1); +static constexpr Vec s2 = Vec::make_v32(2); +static constexpr Vec s3 = Vec::make_v32(3); +static constexpr Vec s4 = Vec::make_v32(4); +static constexpr Vec s5 = Vec::make_v32(5); +static constexpr Vec s6 = Vec::make_v32(6); +static constexpr Vec s7 = Vec::make_v32(7); +static constexpr Vec s8 = Vec::make_v32(8); +static constexpr Vec s9 = Vec::make_v32(9); +static constexpr Vec s10 = Vec::make_v32(10); +static constexpr Vec s11 = Vec::make_v32(11); +static constexpr Vec s12 = Vec::make_v32(12); +static constexpr Vec s13 = Vec::make_v32(13); +static constexpr Vec s14 = Vec::make_v32(14); +static constexpr Vec s15 = Vec::make_v32(15); +static constexpr Vec s16 = Vec::make_v32(16); +static constexpr Vec s17 = Vec::make_v32(17); +static constexpr Vec s18 = Vec::make_v32(18); +static constexpr Vec s19 = Vec::make_v32(19); +static constexpr Vec s20 = Vec::make_v32(20); +static constexpr Vec s21 = Vec::make_v32(21); +static constexpr Vec s22 = Vec::make_v32(22); +static constexpr Vec s23 = Vec::make_v32(23); +static constexpr Vec s24 = Vec::make_v32(24); +static constexpr Vec s25 = Vec::make_v32(25); +static constexpr Vec s26 = Vec::make_v32(26); +static constexpr Vec s27 = Vec::make_v32(27); +static constexpr Vec s28 = Vec::make_v32(28); +static constexpr Vec s29 = Vec::make_v32(29); +static constexpr Vec s30 = Vec::make_v32(30); +static constexpr Vec s31 = Vec::make_v32(31); + +static constexpr Vec d0 = Vec::make_v64(0); +static constexpr Vec d1 = Vec::make_v64(1); +static constexpr Vec d2 = Vec::make_v64(2); +static constexpr Vec d3 = Vec::make_v64(3); +static constexpr Vec d4 = Vec::make_v64(4); +static constexpr Vec d5 = Vec::make_v64(5); +static constexpr Vec d6 = Vec::make_v64(6); +static constexpr Vec d7 = Vec::make_v64(7); +static constexpr Vec d8 = Vec::make_v64(8); +static constexpr Vec d9 = Vec::make_v64(9); +static constexpr Vec d10 = Vec::make_v64(10); +static constexpr Vec d11 = Vec::make_v64(11); +static constexpr Vec d12 = Vec::make_v64(12); +static constexpr Vec d13 = Vec::make_v64(13); +static constexpr Vec d14 = Vec::make_v64(14); +static constexpr Vec d15 = Vec::make_v64(15); +static constexpr Vec d16 = Vec::make_v64(16); +static constexpr Vec d17 = Vec::make_v64(17); +static constexpr Vec d18 = Vec::make_v64(18); +static constexpr Vec d19 = Vec::make_v64(19); +static constexpr Vec d20 = Vec::make_v64(20); +static constexpr Vec d21 = Vec::make_v64(21); +static constexpr Vec d22 = Vec::make_v64(22); +static constexpr Vec d23 = Vec::make_v64(23); +static constexpr Vec d24 = Vec::make_v64(24); +static constexpr Vec d25 = Vec::make_v64(25); +static constexpr Vec d26 = Vec::make_v64(26); +static constexpr Vec d27 = Vec::make_v64(27); +static constexpr Vec d28 = Vec::make_v64(28); +static constexpr Vec d29 = Vec::make_v64(29); +static constexpr Vec d30 = Vec::make_v64(30); +static constexpr Vec d31 = Vec::make_v64(31); + +static constexpr Vec q0 = Vec::make_v128(0); +static constexpr Vec q1 = Vec::make_v128(1); +static constexpr Vec q2 = Vec::make_v128(2); +static constexpr Vec q3 = Vec::make_v128(3); +static constexpr Vec q4 = Vec::make_v128(4); +static constexpr Vec q5 = Vec::make_v128(5); +static constexpr Vec q6 = Vec::make_v128(6); +static constexpr Vec q7 = Vec::make_v128(7); +static constexpr Vec q8 = Vec::make_v128(8); +static constexpr Vec q9 = Vec::make_v128(9); +static constexpr Vec q10 = Vec::make_v128(10); +static constexpr Vec q11 = Vec::make_v128(11); +static constexpr Vec q12 = Vec::make_v128(12); +static constexpr Vec q13 = Vec::make_v128(13); +static constexpr Vec q14 = Vec::make_v128(14); +static constexpr Vec q15 = Vec::make_v128(15); +static constexpr Vec q16 = Vec::make_v128(16); +static constexpr Vec q17 = Vec::make_v128(17); +static constexpr Vec q18 = Vec::make_v128(18); +static constexpr Vec q19 = Vec::make_v128(19); +static constexpr Vec q20 = Vec::make_v128(20); +static constexpr Vec q21 = Vec::make_v128(21); +static constexpr Vec q22 = Vec::make_v128(22); +static constexpr Vec q23 = Vec::make_v128(23); +static constexpr Vec q24 = Vec::make_v128(24); +static constexpr Vec q25 = Vec::make_v128(25); +static constexpr Vec q26 = Vec::make_v128(26); +static constexpr Vec q27 = Vec::make_v128(27); +static constexpr Vec q28 = Vec::make_v128(28); +static constexpr Vec q29 = Vec::make_v128(29); +static constexpr Vec q30 = Vec::make_v128(30); +static constexpr Vec q31 = Vec::make_v128(31); + +static constexpr Vec v0 = Vec::make_v128(0); +static constexpr Vec v1 = Vec::make_v128(1); +static constexpr Vec v2 = Vec::make_v128(2); +static constexpr Vec v3 = Vec::make_v128(3); +static constexpr Vec v4 = Vec::make_v128(4); +static constexpr Vec v5 = Vec::make_v128(5); +static constexpr Vec v6 = Vec::make_v128(6); +static constexpr Vec v7 = Vec::make_v128(7); +static constexpr Vec v8 = Vec::make_v128(8); +static constexpr Vec v9 = Vec::make_v128(9); +static constexpr Vec v10 = Vec::make_v128(10); +static constexpr Vec v11 = Vec::make_v128(11); +static constexpr Vec v12 = Vec::make_v128(12); +static constexpr Vec v13 = Vec::make_v128(13); +static constexpr Vec v14 = Vec::make_v128(14); +static constexpr Vec v15 = Vec::make_v128(15); +static constexpr Vec v16 = Vec::make_v128(16); +static constexpr Vec v17 = Vec::make_v128(17); +static constexpr Vec v18 = Vec::make_v128(18); +static constexpr Vec v19 = Vec::make_v128(19); +static constexpr Vec v20 = Vec::make_v128(20); +static constexpr Vec v21 = Vec::make_v128(21); +static constexpr Vec v22 = Vec::make_v128(22); +static constexpr Vec v23 = Vec::make_v128(23); +static constexpr Vec v24 = Vec::make_v128(24); +static constexpr Vec v25 = Vec::make_v128(25); +static constexpr Vec v26 = Vec::make_v128(26); +static constexpr Vec v27 = Vec::make_v128(27); +static constexpr Vec v28 = Vec::make_v128(28); +static constexpr Vec v29 = Vec::make_v128(29); +static constexpr Vec v30 = Vec::make_v128(30); +static constexpr Vec v31 = Vec::make_v128(31); #ifndef _DOXYGEN } // {regs} @@ -305,6 +986,112 @@ static constexpr VecV v31 = VecV(31); using namespace regs; #endif +//! \name Shift Operation Construction +//! \{ + +//! Constructs a `UXTB #value` extend and shift (unsigned byte extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift uxtb(uint32_t value) noexcept { return Shift(ShiftOp::kUXTB, value); } + +//! Constructs a `UXTH #value` extend and shift (unsigned hword extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift uxth(uint32_t value) noexcept { return Shift(ShiftOp::kUXTH, value); } + +//! Constructs a `UXTW #value` extend and shift (unsigned word extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift uxtw(uint32_t value) noexcept { return Shift(ShiftOp::kUXTW, value); } + +//! Constructs a `UXTX #value` extend and shift (unsigned dword extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift uxtx(uint32_t value) noexcept { return Shift(ShiftOp::kUXTX, value); } + +//! Constructs a `SXTB #value` extend and shift (signed byte extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift sxtb(uint32_t value) noexcept { return Shift(ShiftOp::kSXTB, value); } + +//! Constructs a `SXTH #value` extend and shift (signed hword extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift sxth(uint32_t value) noexcept { return Shift(ShiftOp::kSXTH, value); } + +//! Constructs a `SXTW #value` extend and shift (signed word extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift sxtw(uint32_t value) noexcept { return Shift(ShiftOp::kSXTW, value); } + +//! Constructs a `SXTX #value` extend and shift (signed dword extend) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Shift sxtx(uint32_t value) noexcept { return Shift(ShiftOp::kSXTX, value); } + +//! \} + +//! \name Memory Operand Construction +//! \{ + +//! Creates `[base, offset]` memory operand (offset mode) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr(const Gp& base, int32_t offset = 0) noexcept { + return Mem(base, offset); +} + +//! Creates `[base, offset]!` memory operand (pre-index mode) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr_pre(const Gp& base, int32_t offset = 0) noexcept { + return Mem(base, offset, OperandSignature::fromValue(OffsetMode::kPreIndex)); +} + +//! Creates `[base], offset` memory operand (post-index mode) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr_post(const Gp& base, int32_t offset = 0) noexcept { + return Mem(base, offset, OperandSignature::fromValue(OffsetMode::kPostIndex)); +} + +//! Creates `[base, index]` memory operand (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr(const Gp& base, const Gp& index) noexcept { + return Mem(base, index); +} + +//! Creates `[base, index]!` memory operand (pre-index mode) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr_pre(const Gp& base, const Gp& index) noexcept { + return Mem(base, index, OperandSignature::fromValue(OffsetMode::kPreIndex)); +} + +//! Creates `[base], index` memory operand (post-index mode) (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr_post(const Gp& base, const Gp& index) noexcept { + return Mem(base, index, OperandSignature::fromValue(OffsetMode::kPostIndex)); +} + +//! Creates `[base, index, SHIFT_OP #shift]` memory operand (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr(const Gp& base, const Gp& index, const Shift& shift) noexcept { + return Mem(base, index, shift); +} + +//! Creates `[base, offset]` memory operand (AArch64). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Mem ptr(const Label& base, int32_t offset = 0) noexcept { + return Mem(base, offset); +} + +//! Creates `[base]` absolute memory operand (AArch32 or AArch64). +//! +//! \note The concept of absolute memory operands doesn't exist on ARM, the ISA only provides PC relative addressing. +//! Absolute memory operands can only be used if it's known that the PC relative offset is encodable and that it +//! would be within the limits. Absolute address is also often output from disassemblers, so AsmJit supports it to +//! make it possible to assemble such output back. +static ASMJIT_INLINE_CONSTEXPR Mem ptr(uint64_t base) noexcept { return Mem(base); } + +// TODO: [ARM] PC + offset address. +#if 0 +//! Creates `[PC + offset]` (relative) memory operand. +static ASMJIT_INLINE_CONSTEXPR Mem ptr(const PC& pc, int32_t offset = 0) noexcept { + return Mem(pc, offset); +} +#endif + +//! \} + //! \} ASMJIT_END_SUB_NAMESPACE diff --git a/pe-packer/asmjit/arm/a64rapass.cpp b/pe-packer/asmjit/arm/a64rapass.cpp index aaec1c9..d681214 100644 --- a/pe-packer/asmjit/arm/a64rapass.cpp +++ b/pe-packer/asmjit/arm/a64rapass.cpp @@ -1,12 +1,13 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #if !defined(ASMJIT_NO_AARCH64) && !defined(ASMJIT_NO_COMPILER) #include "../core/cpuinfo.h" +#include "../core/formatter_p.h" #include "../core/support.h" #include "../core/type.h" #include "../arm/a64assembler.h" @@ -22,7 +23,7 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) // ======================== // TODO: [ARM] These should be shared with all backends. -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static inline uint64_t raImmMaskFromSize(uint32_t size) noexcept { ASMJIT_ASSERT(size > 0 && size < 256); static const uint64_t masks[] = { @@ -47,6 +48,7 @@ static const RegMask raConsecutiveLeadCountToRegMaskFilter[5] = { 0x1FFFFFFFu // [4] 4 consecutive registers. }; +[[nodiscard]] static inline RATiedFlags raUseOutFlagsFromRWFlags(OpRWFlags rwFlags) noexcept { static constexpr RATiedFlags map[] = { RATiedFlags::kNone, @@ -58,15 +60,18 @@ static inline RATiedFlags raUseOutFlagsFromRWFlags(OpRWFlags rwFlags) noexcept { return map[uint32_t(rwFlags & OpRWFlags::kRW)]; } +[[nodiscard]] static inline RATiedFlags raRegRwFlags(OpRWFlags flags) noexcept { return raUseOutFlagsFromRWFlags(flags); } +[[nodiscard]] static inline RATiedFlags raMemBaseRwFlags(OpRWFlags flags) noexcept { constexpr uint32_t shift = Support::ConstCTZ::value; return raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> shift) & OpRWFlags::kRW); } +[[nodiscard]] static inline RATiedFlags raMemIndexRwFlags(OpRWFlags flags) noexcept { constexpr uint32_t shift = Support::ConstCTZ::value; return raUseOutFlagsFromRWFlags(OpRWFlags(uint32_t(flags) >> shift) & OpRWFlags::kRW); @@ -82,18 +87,31 @@ public: : RACFGBuilderT(pass), _arch(pass->cc()->arch()) {} + [[nodiscard]] inline Compiler* cc() const noexcept { return static_cast(_cc); } + [[nodiscard]] Error onInst(InstNode* inst, InstControlFlow& controlType, RAInstBuilder& ib) noexcept; + [[nodiscard]] Error onBeforeInvoke(InvokeNode* invokeNode) noexcept; + + [[nodiscard]] Error onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept; - Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept; - Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept; - Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept; + [[nodiscard]] + Error moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, Reg* out) noexcept; + [[nodiscard]] + Error moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept; + + [[nodiscard]] + Error moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Reg& reg) noexcept; + + [[nodiscard]] Error onBeforeRet(FuncRetNode* funcRet) noexcept; + + [[nodiscard]] Error onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept; }; @@ -105,20 +123,26 @@ static InstControlFlow getControlFlowType(InstId instId) noexcept { switch (BaseInst::extractRealId(instId)) { case Inst::kIdB: case Inst::kIdBr: - if (BaseInst::extractARMCondCode(instId) == CondCode::kAL) + if (BaseInst::extractARMCondCode(instId) == CondCode::kAL) { return InstControlFlow::kJump; - else + } + else { return InstControlFlow::kBranch; + } + case Inst::kIdBl: case Inst::kIdBlr: return InstControlFlow::kCall; + case Inst::kIdCbz: case Inst::kIdCbnz: case Inst::kIdTbz: case Inst::kIdTbnz: return InstControlFlow::kBranch; + case Inst::kIdRet: return InstControlFlow::kReturn; + default: return InstControlFlow::kRegular; } @@ -131,7 +155,7 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB InstId instId = inst->id(); uint32_t opCount = inst->opCount(); const Operand* opArray = inst->operands(); - ASMJIT_PROPAGATE(InstInternal::queryRWInfo(_arch, inst->baseInst(), opArray, opCount, &rwInfo)); + ASMJIT_PROPAGATE(InstInternal::queryRWInfo(inst->baseInst(), opArray, opCount, &rwInfo)); const InstDB::InstInfo& instInfo = InstDB::infoById(instId); uint32_t singleRegOps = 0; @@ -172,20 +196,22 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask useRegs = _pass->_availableRegs[group]; RegMask outRegs = useRegs; - uint32_t useId = BaseReg::kIdBad; - uint32_t outId = BaseReg::kIdBad; + uint32_t useId = Reg::kIdBad; + uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; if (opRwInfo.consecutiveLeadCount()) { // There must be a single consecutive register lead, otherwise the RW data is invalid. - if (consecutiveOffset != 0xFFFFFFFFu) + if (consecutiveOffset != 0xFFFFFFFFu) { return DebugUtils::errored(kErrorInvalidState); + } // A consecutive lead register cannot be used as a consecutive +1/+2/+3 register, the registers must be distinct. - if (RATiedReg::consecutiveDataFromFlags(flags) != 0) + if (RATiedReg::consecutiveDataFromFlags(flags) != 0) { return DebugUtils::errored(kErrorNotConsecutiveRegs); + } flags |= RATiedFlags::kLeadConsecutive | RATiedReg::consecutiveDataToFlags(opRwInfo.consecutiveLeadCount() - 1); consecutiveOffset = 0; @@ -202,26 +228,28 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB } if (Support::test(flags, RATiedFlags::kUse)) { - useRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId)); + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(®._baseId)); if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) { useId = opRwInfo.physId(); flags |= RATiedFlags::kUseFixed; } else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) { - if (consecutiveOffset == 0xFFFFFFFFu) + if (consecutiveOffset == 0xFFFFFFFFu) { return DebugUtils::errored(kErrorInvalidState); + } flags |= RATiedFlags::kUseConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset); } } else { - outRewriteMask = Support::bitMask(inst->getRewriteIndex(®._baseId)); + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(®._baseId)); if (opRwInfo.hasOpFlag(OpRWFlags::kRegPhysId)) { outId = opRwInfo.physId(); flags |= RATiedFlags::kOutFixed; } else if (opRwInfo.hasOpFlag(OpRWFlags::kConsecutive)) { - if (consecutiveOffset == 0xFFFFFFFFu) + if (consecutiveOffset == 0xFFFFFFFFu) { return DebugUtils::errored(kErrorInvalidState); + } flags |= RATiedFlags::kOutConsecutive | RATiedReg::consecutiveDataToFlags(++consecutiveOffset); } } @@ -230,20 +258,24 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB if (reg.as().hasElementIndex()) { // Only the first 0..15 registers can be used if the register uses // element accessor that accesses half-words (h[0..7] elements). - if (instInfo.hasFlag(InstDB::kInstFlagVH0_15) && reg.as().elementType() == Vec::kElementTypeH) { - if (Support::test(flags, RATiedFlags::kUse)) + if (instInfo.hasFlag(InstDB::kInstFlagVH0_15) && reg.as().elementType() == VecElementType::kH) { + if (Support::test(flags, RATiedFlags::kUse)) { useId &= 0x0000FFFFu; - else + } + else { outId &= 0x0000FFFFu; + } } } ASMJIT_PROPAGATE(ib.add(workReg, flags, useRegs, useId, useRewriteMask, outRegs, outId, outRewriteMask, opRwInfo.rmSize(), consecutiveParent)); - if (singleRegOps == i) + if (singleRegOps == i) { singleRegOps++; + } - if (Support::test(flags, RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive)) + if (Support::test(flags, RATiedFlags::kLeadConsecutive | RATiedFlags::kUseConsecutive | RATiedFlags::kOutConsecutive)) { consecutiveParent = workReg->workId(); + } } } else if (op.isMem()) { @@ -254,7 +286,9 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB if (mem.isRegHome()) { RAWorkReg* workReg; ASMJIT_PROPAGATE(_pass->virtIndexAsWorkReg(Operand::virtIdToIndex(mem.baseId()), &workReg)); - _pass->getOrCreateStackSlot(workReg); + if (ASMJIT_UNLIKELY(!_pass->getOrCreateStackSlot(workReg))) { + return DebugUtils::errored(kErrorOutOfMemory); + } } else if (mem.hasBaseReg()) { uint32_t vIndex = Operand::virtIdToIndex(mem.baseId()); @@ -267,16 +301,18 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask allocable = _pass->_availableRegs[group]; // Base registers have never fixed id on ARM. - const uint32_t useId = BaseReg::kIdBad; - const uint32_t outId = BaseReg::kIdBad; + const uint32_t useId = Reg::kIdBad; + const uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; - if (Support::test(flags, RATiedFlags::kUse)) - useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId)); - else - outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._baseId)); + if (Support::test(flags, RATiedFlags::kUse)) { + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._baseId)); + } + else { + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._baseId)); + } ASMJIT_PROPAGATE(ib.add(workReg, flags, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask)); } @@ -293,16 +329,18 @@ Error RACFGBuilder::onInst(InstNode* inst, InstControlFlow& controlType, RAInstB RegMask allocable = _pass->_availableRegs[group]; // Index registers have never fixed id on ARM. - const uint32_t useId = BaseReg::kIdBad; - const uint32_t outId = BaseReg::kIdBad; + const uint32_t useId = Reg::kIdBad; + const uint32_t outId = Reg::kIdBad; uint32_t useRewriteMask = 0; uint32_t outRewriteMask = 0; - if (Support::test(flags, RATiedFlags::kUse)) - useRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); - else - outRewriteMask = Support::bitMask(inst->getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); + if (Support::test(flags, RATiedFlags::kUse)) { + useRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); + } + else { + outRewriteMask = Support::bitMask(inst->_getRewriteIndex(&mem._data[Operand::kDataMemIndexId])); + } ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, useId, useRewriteMask, allocable, outId, outRewriteMask)); } @@ -345,7 +383,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { if (arg.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup argGroup = Reg::groupOf(arg.regType()); + RegGroup argGroup = RegUtils::groupOf(arg.regType()); if (regGroup != argGroup) { // TODO: [ARM] Conversion is not supported. @@ -358,7 +396,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { } else if (op.isImm()) { if (arg.isReg()) { - BaseReg reg; + Reg reg; ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, op.as(), ®)); invokeNode->_args[argIndex][valueIndex] = reg; } @@ -374,8 +412,9 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { if (fd.hasRet()) { for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) { const FuncValue& ret = fd.ret(valueIndex); - if (!ret) + if (!ret) { break; + } const Operand& op = invokeNode->ret(valueIndex); if (op.isReg()) { @@ -385,7 +424,7 @@ Error RACFGBuilder::onBeforeInvoke(InvokeNode* invokeNode) noexcept { if (ret.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup retGroup = Reg::groupOf(ret.regType()); + RegGroup retGroup = RegUtils::groupOf(ret.regType()); if (regGroup != retGroup) { // TODO: [ARM] Conversion is not supported. @@ -411,14 +450,16 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) { const FuncValuePack& argPack = fd.argPack(argIndex); for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) { - if (!argPack[valueIndex]) + if (!argPack[valueIndex]) { continue; + } const FuncValue& arg = argPack[valueIndex]; const Operand& op = invokeNode->arg(argIndex, valueIndex); - if (op.isNone()) + if (op.isNone()) { continue; + } if (op.isReg()) { const Reg& reg = op.as(); @@ -427,13 +468,14 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept if (arg.isIndirect()) { RegGroup regGroup = workReg->group(); - if (regGroup != RegGroup::kGp) + if (regGroup != RegGroup::kGp) { return DebugUtils::errored(kErrorInvalidState); + } ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId())); } else if (arg.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup argGroup = Reg::groupOf(arg.regType()); + RegGroup argGroup = RegUtils::groupOf(arg.regType()); if (regGroup == argGroup) { ASMJIT_PROPAGATE(ib.addCallArg(workReg, arg.regId())); @@ -445,8 +487,9 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept for (uint32_t retIndex = 0; retIndex < Globals::kMaxValuePack; retIndex++) { const FuncValue& ret = fd.ret(retIndex); - if (!ret) + if (!ret) { break; + } const Operand& op = invokeNode->ret(retIndex); if (op.isReg()) { @@ -456,7 +499,7 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept if (ret.isReg()) { RegGroup regGroup = workReg->group(); - RegGroup retGroup = Reg::groupOf(ret.regType()); + RegGroup retGroup = RegUtils::groupOf(ret.regType()); if (regGroup == retGroup) { ASMJIT_PROPAGATE(ib.addCallRet(workReg, ret.regId())); @@ -480,7 +523,7 @@ Error RACFGBuilder::onInvoke(InvokeNode* invokeNode, RAInstBuilder& ib) noexcept // a64::RACFGBuilder - MoveImmToRegArg // =================================== -Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, BaseReg* out) noexcept { +Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_, Reg* out) noexcept { DebugUtils::unused(invokeNode); ASMJIT_ASSERT(arg.isReg()); @@ -510,7 +553,7 @@ Error RACFGBuilder::moveImmToRegArg(InvokeNode* invokeNode, const FuncValue& arg // ===================================== Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Imm& imm_) noexcept { - BaseReg reg; + Reg reg; ASMJIT_PROPAGATE(moveImmToRegArg(invokeNode, arg, imm_, ®)); ASMJIT_PROPAGATE(moveRegToStackArg(invokeNode, arg, reg)); @@ -521,15 +564,17 @@ Error RACFGBuilder::moveImmToStackArg(InvokeNode* invokeNode, const FuncValue& a // a64::RACFGBuilder - MoveRegToStackArg // ===================================== -Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const BaseReg& reg) noexcept { +Error RACFGBuilder::moveRegToStackArg(InvokeNode* invokeNode, const FuncValue& arg, const Reg& reg) noexcept { DebugUtils::unused(invokeNode); Mem stackPtr = ptr(_pass->_sp.as(), arg.stackOffset()); - if (reg.isGp()) + if (reg.isGp()) { return cc()->str(reg.as(), stackPtr); + } - if (reg.isVec()) + if (reg.isVec()) { return cc()->str(reg.as(), stackPtr); + } return DebugUtils::errored(kErrorInvalidState); } @@ -549,11 +594,14 @@ Error RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept { for (uint32_t i = 0; i < opCount; i++) { const Operand& op = opArray[i]; - if (op.isNone()) continue; + if (op.isNone()) { + continue; + } const FuncValue& ret = funcDetail.ret(i); - if (ASMJIT_UNLIKELY(!ret.isReg())) + if (ASMJIT_UNLIKELY(!ret.isReg())) { return DebugUtils::errored(kErrorInvalidAssignment); + } if (op.isReg()) { // Register return value. @@ -566,7 +614,7 @@ Error RACFGBuilder::onRet(FuncRetNode* funcRet, RAInstBuilder& ib) noexcept { RegGroup group = workReg->group(); RegMask allocable = _pass->_availableRegs[group]; - ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, ret.regId(), 0, 0, BaseReg::kIdBad, 0)); + ASMJIT_PROPAGATE(ib.add(workReg, RATiedFlags::kUse | RATiedFlags::kRead, allocable, ret.regId(), 0, 0, Reg::kIdBad, 0)); } } else { @@ -595,28 +643,32 @@ void ARMRAPass::onInit() noexcept { _archTraits = &ArchTraits::byArch(arch); _physRegCount.set(RegGroup::kGp, 32); _physRegCount.set(RegGroup::kVec, 32); - _physRegCount.set(RegGroup::kExtraVirt2, 0); + _physRegCount.set(RegGroup::kMask, 0); _physRegCount.set(RegGroup::kExtraVirt3, 0); _buildPhysIndex(); - _availableRegCount = _physRegCount; _availableRegs[RegGroup::kGp] = Support::lsbMask(_physRegCount.get(RegGroup::kGp)); _availableRegs[RegGroup::kVec] = Support::lsbMask(_physRegCount.get(RegGroup::kVec)); - _availableRegs[RegGroup::kExtraVirt3] = Support::lsbMask(_physRegCount.get(RegGroup::kExtraVirt2)); + _availableRegs[RegGroup::kMask] = Support::lsbMask(_physRegCount.get(RegGroup::kMask)); _availableRegs[RegGroup::kExtraVirt3] = Support::lsbMask(_physRegCount.get(RegGroup::kExtraVirt3)); _scratchRegIndexes[0] = uint8_t(27); _scratchRegIndexes[1] = uint8_t(28); + const FuncFrame& frame = _func->frame(); + // The architecture specific setup makes implicitly all registers available. So // make unavailable all registers that are special and cannot be used in general. - bool hasFP = _func->frame().hasPreservedFP(); + bool hasFP = frame.hasPreservedFP(); - if (hasFP) + // Apple ABI requires that the frame-pointer register is not changed by leaf functions and properly updated + // by non-leaf functions. So, let's make this register unavailable as it's just not safe to update it. + if (hasFP || cc()->environment().isDarwin()) { makeUnavailable(RegGroup::kGp, Gp::kIdFp); - + } makeUnavailable(RegGroup::kGp, Gp::kIdSp); makeUnavailable(RegGroup::kGp, Gp::kIdOs); // OS-specific use, usually TLS. + makeUnavailable(frame._unavailableRegs); _sp = sp; _fp = x29; @@ -661,13 +713,17 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no Support::BitWordIterator useIt(tiedReg->useRewriteMask()); uint32_t useId = tiedReg->useId(); - while (useIt.hasNext()) - inst->rewriteIdAtIndex(useIt.next(), useId); + + while (useIt.hasNext()) { + inst->_rewriteIdAtIndex(useIt.next(), useId); + } Support::BitWordIterator outIt(tiedReg->outRewriteMask()); uint32_t outId = tiedReg->outId(); - while (outIt.hasNext()) - inst->rewriteIdAtIndex(outIt.next(), outId); + + while (outIt.hasNext()) { + inst->_rewriteIdAtIndex(outIt.next(), outId); + } } // This data is allocated by Zone passed to `runOnFunction()`, which @@ -701,8 +757,9 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no BaseMem& mem = op.as(); if (mem.isRegHome()) { uint32_t virtIndex = Operand::virtIdToIndex(mem.baseId()); - if (ASMJIT_UNLIKELY(virtIndex >= virtCount)) + if (ASMJIT_UNLIKELY(virtIndex >= virtCount)) { return DebugUtils::errored(kErrorInvalidVirtId); + } VirtReg* virtReg = cc()->virtRegByIndex(virtIndex); RAWorkReg* workReg = virtReg->workReg(); @@ -711,7 +768,7 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no RAStackSlot* slot = workReg->stackSlot(); int32_t offset = slot->offset(); - mem._setBase(_sp.type(), slot->baseRegId()); + mem._setBase(_sp.regType(), slot->baseRegId()); mem.clearRegHome(); mem.addOffsetLo32(offset); } @@ -728,11 +785,12 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no inst->setOp(1, Imm(offset)); } else { - if (mem.hasIndex()) + if (mem.hasIndex()) { return DebugUtils::errored(kErrorInvalidAddressIndex); + } - GpX dst(inst->op(0).as().id()); - GpX base(mem.baseId()); + Gp dst = Gp::make_r64(inst->op(0).as().id()); + Gp base = Gp::make_r64(mem.baseId()); InstId arithInstId = offset < 0 ? Inst::kIdSub : Inst::kIdAdd; uint64_t absOffset = offset < 0 ? Support::neg(uint64_t(offset)) : uint64_t(offset); @@ -773,8 +831,9 @@ ASMJIT_FAVOR_SPEED Error ARMRAPass::_rewrite(BaseNode* first, BaseNode* stop) no // ================================ Error ARMRAPass::updateStackFrame() noexcept { - if (_func->frame().hasFuncCalls()) + if (_func->frame().hasFuncCalls()) { _func->frame().addDirtyRegs(RegGroup::kGp, Support::bitMask(Gp::kIdLr)); + } return BaseRAPass::updateStackFrame(); } @@ -784,14 +843,15 @@ Error ARMRAPass::updateStackFrame() noexcept { Error ARMRAPass::emitMove(uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); - BaseReg dst(wReg->signature(), dstPhysId); - BaseReg src(wReg->signature(), srcPhysId); + Reg dst(wReg->signature(), dstPhysId); + Reg src(wReg->signature(), srcPhysId); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif @@ -806,14 +866,15 @@ Error ARMRAPass::emitSwap(uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, Error ARMRAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); - BaseReg dstReg(wReg->signature(), dstPhysId); + Reg dstReg(wReg->signature(), dstPhysId); BaseMem srcMem(workRegAsMem(wReg)); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif @@ -824,13 +885,14 @@ Error ARMRAPass::emitLoad(uint32_t workId, uint32_t dstPhysId) noexcept { Error ARMRAPass::emitSave(uint32_t workId, uint32_t srcPhysId) noexcept { RAWorkReg* wReg = workRegById(workId); BaseMem dstMem(workRegAsMem(wReg)); - BaseReg srcReg(wReg->signature(), srcPhysId); + Reg srcReg(wReg->signature(), srcPhysId); const char* comment = nullptr; #ifndef ASMJIT_NO_LOGGING if (hasDiagnosticOption(DiagnosticOptions::kRAAnnotate)) { - _tmpString.assignFormat(" %s", workRegById(workId)->name()); + _tmpString.clear(); + Formatter::formatVirtRegNameWithPrefix(_tmpString, " ", 7u, wReg->virtReg()); comment = _tmpString.data(); } #endif diff --git a/pe-packer/asmjit/arm/a64rapass_p.h b/pe-packer/asmjit/arm/a64rapass_p.h index 7313087..908f661 100644 --- a/pe-packer/asmjit/arm/a64rapass_p.h +++ b/pe-packer/asmjit/arm/a64rapass_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_A64RAPASS_P_H_INCLUDED @@ -29,10 +29,15 @@ ASMJIT_BEGIN_SUB_NAMESPACE(a64) class ARMRAPass : public BaseRAPass { public: ASMJIT_NONCOPYABLE(ARMRAPass) - typedef BaseRAPass Base; + using Base = BaseRAPass; + + //! \name Members + //! \{ EmitHelper _emitHelper; + //! \} + //! \name Construction & Destruction //! \{ @@ -45,9 +50,11 @@ public: //! \{ //! Returns the compiler casted to `arm::Compiler`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Compiler* cc() const noexcept { return static_cast(_cb); } //! Returns emit helper. + [[nodiscard]] ASMJIT_INLINE_NODEBUG EmitHelper* emitHelper() noexcept { return &_emitHelper; } //! \} diff --git a/pe-packer/asmjit/arm/armformatter.cpp b/pe-packer/asmjit/arm/armformatter.cpp index 86eb24b..c4e6712 100644 --- a/pe-packer/asmjit/arm/armformatter.cpp +++ b/pe-packer/asmjit/arm/armformatter.cpp @@ -1,15 +1,16 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #ifndef ASMJIT_NO_LOGGING +#include "../core/formatter_p.h" #include "../core/misc_p.h" #include "../core/support.h" #include "../arm/armformatter_p.h" -#include "../arm/armoperand.h" +#include "../arm/a64operand.h" #include "../arm/a64instapi_p.h" #include "../arm/a64instdb_p.h" @@ -31,26 +32,50 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "ARMv8a\0" "THUMB\0" "THUMBv2\0" + "ABLE\0" + "ADERR\0" "AES\0" "AFP\0" + "AIE\0" + "AMU1\0" + "AMU1_1\0" + "ANERR\0" "ASIMD\0" "BF16\0" + "BRBE\0" "BTI\0" + "BWE\0" "CCIDX\0" "CHK\0" "CLRBHB\0" + "CMOW\0" + "CMPBR\0" + "CONSTPACFIELD\0" + "CPA\0" + "CPA2\0" "CPUID\0" "CRC32\0" "CSSC\0" + "CSV2\0" + "CSV2_3\0" + "CSV3\0" "D128\0" "DGH\0" "DIT\0" "DOTPROD\0" "DPB\0" "DPB2\0" + "EBEP\0" "EBF16\0" + "ECBHB\0" "ECV\0" + "EDHSR\0" "EDSP\0" + "F8E4M3\0" + "F8E5M2\0" + "F8F16MM\0" + "F8F32MM\0" + "FAMINMAX\0" "FCMA\0" "FGT\0" "FGT2\0" @@ -61,13 +86,26 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "FP\0" "FP16\0" "FP16CONV\0" + "FP8\0" + "FP8DOT2\0" + "FP8DOT4\0" + "FP8FMA\0" + "FPMR\0" + "FPRCVT\0" "FRINTTS\0" "GCS\0" + "HACDBS\0" + "HAFDBS\0" + "HAFT\0" + "HDBSS\0" "HBC\0" "HCX\0" + "HPDS\0" + "HPDS2\0" "I8MM\0" "IDIVA\0" "IDIVT\0" + "ITE\0" "JSCVT\0" "LOR\0" "LRCPC\0" @@ -76,35 +114,60 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "LS64\0" "LS64_ACCDATA\0" "LS64_V\0" + "LS64WB\0" "LSE\0" "LSE128\0" "LSE2\0" + "LSFE\0" + "LSUI\0" + "LUT\0" + "LVA\0" + "LVA3\0" + "MEC\0" "MOPS\0" "MPAM\0" "MTE\0" "MTE2\0" "MTE3\0" "MTE4\0" + "MTE_ASYM_FAULT\0" + "MTE_ASYNC\0" + "MTE_CANONICAL_TAGS\0" + "MTE_NO_ADDRESS_TAGS\0" + "MTE_PERM_S1\0" + "MTE_STORE_ONLY\0" + "MTE_TAGGED_FAR\0" + "MTPMU\0" "NMI\0" "NV\0" "NV2\0" + "OCCMO\0" "PAN\0" "PAN2\0" "PAN3\0" "PAUTH\0" + "PFAR\0" "PMU\0" "PMULL\0" "PRFMSLC\0" "RAS\0" "RAS1_1\0" "RAS2\0" + "RASSA2\0" "RDM\0" "RME\0" "RNG\0" "RNG_TRAP\0" "RPRES\0" "RPRFM\0" + "S1PIE\0" + "S1POE\0" + "S2PIE\0" + "S2POE\0" "SB\0" + "SCTLR2\0" + "SEBEP\0" + "SEL2\0" "SHA1\0" "SHA256\0" "SHA3\0" @@ -114,6 +177,8 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SME\0" "SME2\0" "SME2_1\0" + "SME2_2\0" + "SME_AES\0" "SME_B16B16\0" "SME_B16F32\0" "SME_BI32I32\0" @@ -121,22 +186,50 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SME_F16F32\0" "SME_F32F32\0" "SME_F64F64\0" + "SME_F8F16\0" + "SME_F8F32\0" "SME_FA64\0" "SME_I16I32\0" "SME_I16I64\0" "SME_I8I32\0" + "SME_LUTv2\0" + "SME_MOP4\0" + "SME_TMOP\0" + "SPE\0" + "SPE1_1\0" + "SPE1_2\0" + "SPE1_3\0" + "SPE1_4\0" + "SPE_ALTCLK\0" + "SPE_CRR\0" + "SPE_EFT\0" + "SPE_FDS\0" + "SPE_FPF\0" + "SPE_SME\0" "SPECRES\0" "SPECRES2\0" + "SPMU\0" "SSBS\0" "SSBS2\0" + "SSVE_AES\0" + "SSVE_BITPERM\0" + "SSVE_FEXPA\0" + "SSVE_FP8DOT2\0" + "SSVE_FP8DOT4\0" + "SSVE_FP8FMA\0" "SVE\0" "SVE2\0" "SVE2_1\0" + "SVE2_2\0" "SVE_AES\0" + "SVE_AES2\0" "SVE_B16B16\0" "SVE_BF16\0" + "SVE_BFSCALE\0" "SVE_BITPERM\0" "SVE_EBF16\0" + "SVE_ELTPERM\0" + "SVE_F16MM\0" "SVE_F32MM\0" "SVE_F64MM\0" "SVE_I8MM\0" @@ -146,25 +239,37 @@ Error FormatterInternal::formatFeature(String& sb, uint32_t featureId) noexcept "SYSINSTR128\0" "SYSREG128\0" "THE\0" + "TLBIOS\0" + "TLBIRANGE\0" + "TLBIW\0" "TME\0" "TRF\0" "UAO\0" "VFP_D32\0" "VHE\0" + "VMID16\0" "WFXT\0" + "XNX\0" "XS\0" "\0"; static const uint16_t sFeatureIndex[] = { - 0, 5, 11, 17, 24, 30, 38, 42, 46, 52, 57, 61, 67, 71, 78, 84, 90, 95, 100, - 104, 108, 116, 120, 125, 131, 135, 140, 145, 149, 154, 158, 164, 171, 176, - 179, 184, 193, 201, 205, 209, 213, 218, 224, 230, 236, 240, 246, 253, 260, - 265, 278, 285, 289, 296, 301, 306, 311, 315, 320, 325, 330, 334, 337, 341, - 345, 350, 355, 361, 365, 371, 379, 383, 390, 395, 399, 403, 407, 416, 422, - 428, 431, 436, 443, 448, 455, 459, 463, 467, 472, 479, 490, 501, 513, 524, - 535, 546, 557, 566, 577, 588, 598, 606, 615, 620, 626, 630, 635, 642, 650, - 661, 670, 682, 692, 702, 712, 721, 734, 743, 751, 763, 773, 777, 781, 785, - 789, 797, 801, 806, 809 + 0, 5, 11, 17, 24, 30, 38, 43, 49, 53, 57, 61, 66, 73, 79, 85, 90, 95, 99, + 103, 109, 113, 120, 125, 131, 145, 149, 154, 160, 166, 171, 176, 183, 188, + 193, 197, 201, 209, 213, 218, 223, 229, 235, 239, 245, 250, 257, 264, 272, + 280, 289, 294, 298, 303, 307, 313, 320, 325, 328, 333, 342, 346, 354, 362, + 369, 374, 381, 389, 393, 400, 407, 412, 418, 422, 426, 431, 437, 442, 448, + 454, 458, 464, 468, 474, 481, 488, 493, 506, 513, 520, 524, 531, 536, 541, + 546, 550, 554, 559, 563, 568, 573, 577, 582, 587, 592, 607, 617, 636, 656, + 668, 683, 698, 704, 708, 711, 715, 721, 725, 730, 735, 741, 746, 750, 756, + 764, 768, 775, 780, 787, 791, 795, 799, 808, 814, 820, 826, 832, 838, 844, + 847, 854, 860, 865, 870, 877, 882, 889, 893, 897, 901, 906, 913, 920, 928, + 939, 950, 962, 973, 984, 995, 1006, 1016, 1026, 1035, 1046, 1057, 1067, 1077, + 1086, 1095, 1099, 1106, 1113, 1120, 1127, 1138, 1146, 1154, 1162, 1170, 1178, + 1186, 1195, 1200, 1205, 1211, 1220, 1233, 1244, 1257, 1270, 1282, 1286, 1291, + 1298, 1305, 1313, 1322, 1333, 1342, 1354, 1366, 1376, 1388, 1398, 1408, 1418, + 1427, 1440, 1449, 1457, 1469, 1479, 1483, 1490, 1500, 1506, 1510, 1514, 1518, + 1526, 1530, 1537, 1542, 1546, 1549 }; // @EnumStringEnd@ @@ -178,14 +283,14 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatCondCode(String& sb, CondCode c static const char condCodeData[] = "al\0" "na\0" "eq\0" "ne\0" - "cs\0" "cc\0" "mi\0" "pl\0" "vs\0" "vc\0" + "hs\0" "lo\0" "mi\0" "pl\0" "vs\0" "vc\0" "hi\0" "ls\0" "ge\0" "lt\0" "gt\0" "le\0" ""; return sb.append(condCodeData + Support::min(uint32_t(cc), 16u) * 3); } ASMJIT_FAVOR_SIZE Error FormatterInternal::formatShiftOp(String& sb, ShiftOp shiftOp) noexcept { - const char* str = ""; + const char* str = nullptr; switch (shiftOp) { case ShiftOp::kLSL: str = "lsl"; break; case ShiftOp::kLSR: str = "lsr"; break; @@ -201,6 +306,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatShiftOp(String& sb, ShiftOp shi case ShiftOp::kSXTH: str = "sxth"; break; case ShiftOp::kSXTW: str = "sxtw"; break; case ShiftOp::kSXTX: str = "sxtx"; break; + default: str = ""; break; } return sb.append(str); } @@ -208,6 +314,25 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatShiftOp(String& sb, ShiftOp shi // arm::FormatterInternal - Format Register // ======================================== +struct FormatElementData { + char letter; + uint8_t elementCount; + uint8_t onlyIndex; + uint8_t reserved; +}; + +static constexpr FormatElementData formatElementDataTable[9] = { + { '?' , 0 , 0, 0 }, // None + { 'b' , 16, 0, 0 }, // bX or b[index] + { 'h' , 8 , 0, 0 }, // hX or h[index] + { 's' , 4 , 0, 0 }, // sX or s[index] + { 'd' , 2 , 0, 0 }, // dX or d[index] + { 'b' , 4 , 1, 0 }, // ?? or b4[index] + { 'h' , 2 , 1, 0 }, // ?? or h2[index] + { '?' , 0 , 0, 0 }, // invalid (possibly stored in Operand) + { '?' , 0 , 0, 0 } // invalid (never stored in Operand, bug...) +}; + ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( String& sb, FormatFlags flags, @@ -233,12 +358,7 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( VirtReg* vReg = cc->virtRegById(rId); ASMJIT_ASSERT(vReg != nullptr); - const char* name = vReg->name(); - if (name && name[0] != '\0') - ASMJIT_PROPAGATE(sb.append(name)); - else - ASMJIT_PROPAGATE(sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(rId)))); - + ASMJIT_PROPAGATE(Formatter::formatVirtRegName(sb, vReg)); virtRegFormatted = true; } } @@ -250,56 +370,53 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( if (!virtRegFormatted) { char letter = '\0'; switch (regType) { - case RegType::kARM_VecB: - case RegType::kARM_VecH: - case RegType::kARM_VecS: - case RegType::kARM_VecD: - case RegType::kARM_VecV: - letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kARM_VecB)]; - if (elementType) + case RegType::kVec8: + case RegType::kVec16: + case RegType::kVec32: + case RegType::kVec64: + case RegType::kVec128: + letter = bhsdq[uint32_t(regType) - uint32_t(RegType::kVec8)]; + if (elementType) { letter = 'v'; + } break; - case RegType::kARM_GpW: + case RegType::kGp32: if (Environment::is64Bit(arch)) { letter = 'w'; - if (rId == Gp::kIdZr) + if (rId == a64::Gp::kIdZr) { return sb.append("wzr", 3); + } - if (rId == Gp::kIdSp) + if (rId == a64::Gp::kIdSp) { return sb.append("wsp", 3); + } } else { letter = 'r'; - - if (rId == 13) - return sb.append("sp", 2); - - if (rId == 14) - return sb.append("lr", 2); - - if (rId == 15) - return sb.append("pc", 2); } break; - case RegType::kARM_GpX: + case RegType::kGp64: if (Environment::is64Bit(arch)) { - if (rId == Gp::kIdZr) + if (rId == a64::Gp::kIdZr) { return sb.append("xzr", 3); - if (rId == Gp::kIdSp) + } + + if (rId == a64::Gp::kIdSp) { return sb.append("sp", 2); + } letter = 'x'; break; } // X registers are undefined in 32-bit mode. - ASMJIT_FALLTHROUGH; + [[fallthrough]]; default: - ASMJIT_PROPAGATE(sb.appendFormat("?$u", uint32_t(regType), rId)); + ASMJIT_PROPAGATE(sb.appendFormat("?%u", uint32_t(regType), rId)); break; } @@ -307,54 +424,70 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegister( ASMJIT_PROPAGATE(sb.appendFormat("%c%u", letter, rId)); } + constexpr uint32_t kElementTypeCount = uint32_t(a64::VecElementType::kMaxValue) + 1; if (elementType) { - char elementLetter = '\0'; - uint32_t elementCount = 0; + elementType = Support::min(elementType, kElementTypeCount); - switch (elementType) { - case Vec::kElementTypeB: - elementLetter = 'b'; - elementCount = 16; - break; + FormatElementData elementData = formatElementDataTable[elementType]; + uint32_t elementCount = elementData.elementCount; - case Vec::kElementTypeH: - elementLetter = 'h'; - elementCount = 8; - break; - - case Vec::kElementTypeS: - elementLetter = 's'; - elementCount = 4; - break; - - case Vec::kElementTypeD: - elementLetter = 'd'; - elementCount = 2; - break; - - default: - return sb.append("."); + if (regType == RegType::kVec64) { + elementCount /= 2u; } - if (elementLetter) { - if (elementIndex == 0xFFFFFFFFu) { - if (regType == RegType::kARM_VecD) - elementCount /= 2u; - ASMJIT_PROPAGATE(sb.appendFormat(".%u%c", elementCount, elementLetter)); - } - else { - ASMJIT_PROPAGATE(sb.appendFormat(".%c[%u]", elementLetter, elementIndex)); - } + ASMJIT_PROPAGATE(sb.append('.')); + if (elementCount) { + ASMJIT_PROPAGATE(sb.appendUInt(elementCount)); } + ASMJIT_PROPAGATE(sb.append(elementData.letter)); } - else if (elementIndex != 0xFFFFFFFFu) { - // This should only be used by AArch32 - AArch64 requires an additional elementType in index[]. + + if (elementIndex != 0xFFFFFFFFu) { ASMJIT_PROPAGATE(sb.appendFormat("[%u]", elementIndex)); } return kErrorOk; } +ASMJIT_FAVOR_SIZE Error FormatterInternal::formatRegisterList( + String& sb, + FormatFlags flags, + const BaseEmitter* emitter, + Arch arch, + RegType regType, + uint32_t rMask) noexcept { + + bool first = true; + + ASMJIT_PROPAGATE(sb.append('{')); + while (rMask != 0u) { + uint32_t start = Support::ctz(rMask); + uint32_t count = 0u; + + uint32_t mask = 1u << start; + do { + rMask &= ~mask; + mask <<= 1u; + count++; + } while (rMask & mask); + + if (!first) { + ASMJIT_PROPAGATE(sb.append(", ")); + } + + ASMJIT_PROPAGATE(formatRegister(sb, flags, emitter, arch, regType, start, 0, 0xFFFFFFFFu)); + if (count >= 2u) { + ASMJIT_PROPAGATE(sb.append('-')); + ASMJIT_PROPAGATE(formatRegister(sb, flags, emitter, arch, regType, start + count - 1, 0, 0xFFFFFFFFu)); + } + + first = false; + } + ASMJIT_PROPAGATE(sb.append('}')); + + return kErrorOk; +} + // a64::FormatterInternal - Format Operand // ======================================= @@ -366,19 +499,20 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( const Operand_& op) noexcept { if (op.isReg()) { - const BaseReg& reg = op.as(); + const Reg& reg = op.as(); - uint32_t elementType = op.as().elementType(); - uint32_t elementIndex = op.as().elementIndex(); + uint32_t elementType = op._signature.getField(); + uint32_t elementIndex = op.as().elementIndex(); - if (!op.as().hasElementIndex()) + if (!op.as().hasElementIndex()) { elementIndex = 0xFFFFFFFFu; + } - return formatRegister(sb, flags, emitter, arch, reg.type(), reg.id(), elementType, elementIndex); + return formatRegister(sb, flags, emitter, arch, reg.regType(), reg.id(), elementType, elementIndex); } if (op.isMem()) { - const Mem& m = op.as(); + const a64::Mem& m = op.as(); ASMJIT_PROPAGATE(sb.append('[')); if (m.hasBase()) { @@ -418,8 +552,9 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( int64_t off = int64_t(m.offset()); uint32_t base = 10; - if (Support::test(flags, FormatFlags::kHexOffsets) && uint64_t(off) > 9) + if (Support::test(flags, FormatFlags::kHexOffsets) && uint64_t(off) > 9) { base = 16; + } if (base == 10) { ASMJIT_PROPAGATE(sb.appendInt(off, base)); @@ -432,16 +567,19 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( if (m.hasShift()) { ASMJIT_PROPAGATE(sb.append(' ')); - if (!m.isPreOrPost()) - ASMJIT_PROPAGATE(formatShiftOp(sb, (ShiftOp)m.predicate())); + if (!m.isPreOrPost()) { + ASMJIT_PROPAGATE(formatShiftOp(sb, m.shiftOp())); + } ASMJIT_PROPAGATE(sb.appendFormat(" %u", m.shift())); } - if (!m.isPostIndex()) + if (!m.isPostIndex()) { ASMJIT_PROPAGATE(sb.append(']')); + } - if (m.isPreIndex()) + if (m.isPreIndex()) { ASMJIT_PROPAGATE(sb.append('!')); + } return kErrorOk; } @@ -449,6 +587,12 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( if (op.isImm()) { const Imm& i = op.as(); int64_t val = i.value(); + uint32_t predicate = i.predicate(); + + if (predicate) { + ASMJIT_PROPAGATE(formatShiftOp(sb, ShiftOp(predicate))); + ASMJIT_PROPAGATE(sb.append(' ')); + } if (Support::test(flags, FormatFlags::kHexImms) && uint64_t(val) > 9) { ASMJIT_PROPAGATE(sb.append("0x")); @@ -463,6 +607,11 @@ ASMJIT_FAVOR_SIZE Error FormatterInternal::formatOperand( return Formatter::formatLabel(sb, flags, emitter, op.id()); } + if (op.isRegList()) { + const BaseRegList& regList = op.as(); + return formatRegisterList(sb, flags, emitter, arch, regList.regType(), regList.list()); + } + return sb.append(""); } diff --git a/pe-packer/asmjit/arm/armformatter_p.h b/pe-packer/asmjit/arm/armformatter_p.h index ccd2bd6..d75fd91 100644 --- a/pe-packer/asmjit/arm/armformatter_p.h +++ b/pe-packer/asmjit/arm/armformatter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMFORMATTER_P_H_INCLUDED @@ -43,6 +43,14 @@ Error ASMJIT_CDECL formatRegister( uint32_t elementType = 0, uint32_t elementIndex = 0xFFFFFFFF) noexcept; +Error ASMJIT_CDECL formatRegisterList( + String& sb, + FormatFlags flags, + const BaseEmitter* emitter, + Arch arch, + RegType regType, + uint32_t rMask) noexcept; + Error ASMJIT_CDECL formatOperand( String& sb, FormatFlags flags, diff --git a/pe-packer/asmjit/arm/armglobals.h b/pe-packer/asmjit/arm/armglobals.h index 506646f..e05ee74 100644 --- a/pe-packer/asmjit/arm/armglobals.h +++ b/pe-packer/asmjit/arm/armglobals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMGLOBALS_H_INCLUDED @@ -14,8 +14,4 @@ //! //! API shared between AArch32 & AArch64 backends. -ASMJIT_BEGIN_SUB_NAMESPACE(arm) - -ASMJIT_END_SUB_NAMESPACE - #endif // ASMJIT_ARM_ARMGLOBALS_H_INCLUDED diff --git a/pe-packer/asmjit/arm/armoperand.h b/pe-packer/asmjit/arm/armoperand.h deleted file mode 100644 index 5e03e64..0000000 --- a/pe-packer/asmjit/arm/armoperand.h +++ /dev/null @@ -1,626 +0,0 @@ -// This file is part of AsmJit project -// -// See asmjit.h or LICENSE.md for license and copyright information -// SPDX-License-Identifier: Zlib - -#ifndef ASMJIT_ARM_ARMOPERAND_H_INCLUDED -#define ASMJIT_ARM_ARMOPERAND_H_INCLUDED - -#include "../core/archtraits.h" -#include "../core/operand.h" -#include "../core/type.h" -#include "../arm/armglobals.h" - -ASMJIT_BEGIN_SUB_NAMESPACE(arm) - -//! \addtogroup asmjit_arm -//! \{ - -class Reg; -class Mem; - -class Gp; -class GpW; -class GpX; - -class Vec; -class VecB; -class VecH; -class VecS; -class VecD; -class VecV; - -//! Register traits (ARM/AArch64). -//! -//! Register traits contains information about a particular register type. It's used by asmjit to setup register -//! information on-the-fly and to populate tables that contain register information (this way it's possible to -//! change register types and groups without having to reorder these tables). -template -struct RegTraits : public BaseRegTraits {}; - -//! \cond -// <--------------------+-----+-------------------------+------------------------+---+---+------------------+ -// | Reg | Reg-Type | Reg-Group |Sz |Cnt| TypeId | -// <--------------------+-----+-------------------------+------------------------+---+---+------------------+ -ASMJIT_DEFINE_REG_TRAITS(GpW , RegType::kARM_GpW , RegGroup::kGp , 4 , 32, TypeId::kInt32 ); -ASMJIT_DEFINE_REG_TRAITS(GpX , RegType::kARM_GpX , RegGroup::kGp , 8 , 32, TypeId::kInt64 ); -ASMJIT_DEFINE_REG_TRAITS(VecB , RegType::kARM_VecB , RegGroup::kVec , 1 , 32, TypeId::kVoid ); -ASMJIT_DEFINE_REG_TRAITS(VecH , RegType::kARM_VecH , RegGroup::kVec , 2 , 32, TypeId::kVoid ); -ASMJIT_DEFINE_REG_TRAITS(VecS , RegType::kARM_VecS , RegGroup::kVec , 4 , 32, TypeId::kInt32x1 ); -ASMJIT_DEFINE_REG_TRAITS(VecD , RegType::kARM_VecD , RegGroup::kVec , 8 , 32, TypeId::kInt32x2 ); -ASMJIT_DEFINE_REG_TRAITS(VecV , RegType::kARM_VecV , RegGroup::kVec , 16, 32, TypeId::kInt32x4 ); -//! \endcond - -//! Register (ARM). -class Reg : public BaseReg { -public: - ASMJIT_DEFINE_ABSTRACT_REG(Reg, BaseReg) - - //! Gets whether the register is a `R|W` register (32-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isGpW() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is an `X` register (64-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isGpX() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-B register (8-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecB() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-H register (16-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecH() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-S register (32-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecS() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-D register (64-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecD() const noexcept { return baseSignature() == RegTraits::kSignature; } - //! Gets whether the register is a VEC-Q register (128-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecQ() const noexcept { return baseSignature() == RegTraits::kSignature; } - - //! Gets whether the register is either VEC-D (64-bit) or VEC-Q (128-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecDOrQ() const noexcept { return uint32_t(type()) - uint32_t(RegType::kARM_VecD) <= 1u; } - - //! Gets whether the register is a VEC-V register (128-bit). - ASMJIT_INLINE_NODEBUG constexpr bool isVecV() const noexcept { return baseSignature() == RegTraits::kSignature; } - - template - ASMJIT_INLINE_NODEBUG void setRegT(uint32_t id) noexcept { - setSignature(RegTraits::kSignature); - setId(id); - } - - ASMJIT_INLINE_NODEBUG void setTypeAndId(RegType type, uint32_t id) noexcept { - setSignature(signatureOf(type)); - setId(id); - } - - static ASMJIT_INLINE_NODEBUG RegGroup groupOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToGroup(type); } - static ASMJIT_INLINE_NODEBUG TypeId typeIdOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToTypeId(type); } - static ASMJIT_INLINE_NODEBUG OperandSignature signatureOf(RegType type) noexcept { return ArchTraits::byArch(Arch::kAArch64).regTypeToSignature(type); } - - template - static ASMJIT_INLINE_NODEBUG RegGroup groupOfT() noexcept { return RegTraits::kGroup; } - - template - static ASMJIT_INLINE_NODEBUG TypeId typeIdOfT() noexcept { return RegTraits::kTypeId; } - - template - static ASMJIT_INLINE_NODEBUG OperandSignature signatureOfT() noexcept { return RegTraits::kSignature; } - - static ASMJIT_INLINE_NODEBUG bool isGpW(const Operand_& op) noexcept { return op.as().isGpW(); } - static ASMJIT_INLINE_NODEBUG bool isGpX(const Operand_& op) noexcept { return op.as().isGpX(); } - static ASMJIT_INLINE_NODEBUG bool isVecB(const Operand_& op) noexcept { return op.as().isVecB(); } - static ASMJIT_INLINE_NODEBUG bool isVecH(const Operand_& op) noexcept { return op.as().isVecH(); } - static ASMJIT_INLINE_NODEBUG bool isVecS(const Operand_& op) noexcept { return op.as().isVecS(); } - static ASMJIT_INLINE_NODEBUG bool isVecD(const Operand_& op) noexcept { return op.as().isVecD(); } - static ASMJIT_INLINE_NODEBUG bool isVecQ(const Operand_& op) noexcept { return op.as().isVecQ(); } - static ASMJIT_INLINE_NODEBUG bool isVecV(const Operand_& op) noexcept { return op.as().isVecV(); } - - static ASMJIT_INLINE_NODEBUG bool isGpW(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGpW(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isGpX(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGpX(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecB(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecB(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecH(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecH(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecS(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecS(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecD(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecD(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecQ(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecQ(op)) & unsigned(op.id() == id)); } - static ASMJIT_INLINE_NODEBUG bool isVecV(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVecV(op)) & unsigned(op.id() == id)); } -}; - -//! General purpose register (ARM). -class Gp : public Reg { -public: - ASMJIT_DEFINE_ABSTRACT_REG(Gp, Reg) - - //! Special register id. - enum Id : uint32_t { - //! Register that depends on OS, could be used as TLS offset. - kIdOs = 18, - //! Frame pointer. - kIdFp = 29, - //! Link register. - kIdLr = 30, - //! Stack register id. - kIdSp = 31, - //! Zero register id. - //! - //! Although zero register has the same id as stack register it has a special treatment, because we need to be - //! able to distinguish between these two at API level. Some intructions were designed to be used with SP and - //! some other with ZR - so we need a way to distinguish these two to make sure we emit the right thing. - //! - //! The number 63 is not random, when you perform `id & 31` you would always get 31 for both SP and ZR inputs, - //! which is the identifier used by AArch64 ISA to encode either SP or ZR depending on the instruction. - kIdZr = 63 - }; - - ASMJIT_INLINE_NODEBUG constexpr bool isZR() const noexcept { return id() == kIdZr; } - ASMJIT_INLINE_NODEBUG constexpr bool isSP() const noexcept { return id() == kIdSp; } - - //! Cast this register to a 32-bit R|W. - ASMJIT_INLINE_NODEBUG GpW w() const noexcept; - //! Cast this register to a 64-bit X. - ASMJIT_INLINE_NODEBUG GpX x() const noexcept; -}; - -//! Vector register (ARM). -class Vec : public Reg { -public: - ASMJIT_DEFINE_ABSTRACT_REG(Vec, Reg) - - //! Additional signature bits used by arm::Vec. - enum AdditionalBits : uint32_t { - // Register element type (3 bits). - // |........|........|.XXX....|........| - kSignatureRegElementTypeShift = 12, - kSignatureRegElementTypeMask = 0x07 << kSignatureRegElementTypeShift, - - // Register has element index (1 bit). - // |........|........|X.......|........| - kSignatureRegElementFlagShift = 15, - kSignatureRegElementFlagMask = 0x01 << kSignatureRegElementFlagShift, - - // Register element index (4 bits). - // |........|....XXXX|........|........| - kSignatureRegElementIndexShift = 16, - kSignatureRegElementIndexMask = 0x0F << kSignatureRegElementIndexShift - }; - - //! Element type (AArch64 only). - enum ElementType : uint32_t { - //! No element type specified. - kElementTypeNone = 0, - //! Byte elements (B8 or B16). - kElementTypeB, - //! Halfword elements (H4 or H8). - kElementTypeH, - //! Singleword elements (S2 or S4). - kElementTypeS, - //! Doubleword elements (D2). - kElementTypeD, - //! Byte elements grouped by 4 bytes (B4). - //! - //! \note This element-type is only used by few instructions. - kElementTypeB4, - //! Halfword elements grouped by 2 halfwords (H2). - //! - //! \note This element-type is only used by few instructions. - kElementTypeH2, - - //! Count of element types. - kElementTypeCount - }; - - //! \cond - //! Shortcuts. - enum SignatureReg : uint32_t { - kSignatureElementB = kElementTypeB << kSignatureRegElementTypeShift, - kSignatureElementH = kElementTypeH << kSignatureRegElementTypeShift, - kSignatureElementS = kElementTypeS << kSignatureRegElementTypeShift, - kSignatureElementD = kElementTypeD << kSignatureRegElementTypeShift, - kSignatureElementB4 = kElementTypeB4 << kSignatureRegElementTypeShift, - kSignatureElementH2 = kElementTypeH2 << kSignatureRegElementTypeShift - }; - //! \endcond - - //! Returns whether the register has associated an element type. - ASMJIT_INLINE_NODEBUG constexpr bool hasElementType() const noexcept { return _signature.hasField(); } - //! Returns whether the register has element index (it's an element index access). - ASMJIT_INLINE_NODEBUG constexpr bool hasElementIndex() const noexcept { return _signature.hasField(); } - //! Returns whether the reggister has element type or element index (or both). - ASMJIT_INLINE_NODEBUG constexpr bool hasElementTypeOrIndex() const noexcept { return _signature.hasField(); } - - //! Returns element type of the register. - ASMJIT_INLINE_NODEBUG constexpr uint32_t elementType() const noexcept { return _signature.getField(); } - //! Sets element type of the register to `elementType`. - ASMJIT_INLINE_NODEBUG void setElementType(uint32_t elementType) noexcept { _signature.setField(elementType); } - //! Resets element type to none. - ASMJIT_INLINE_NODEBUG void resetElementType() noexcept { _signature.setField(0); } - - //! Returns element index of the register. - ASMJIT_INLINE_NODEBUG constexpr uint32_t elementIndex() const noexcept { return _signature.getField(); } - //! Sets element index of the register to `elementType`. - ASMJIT_INLINE_NODEBUG void setElementIndex(uint32_t elementIndex) noexcept { - _signature |= kSignatureRegElementFlagMask; - _signature.setField(elementIndex); - } - //! Resets element index of the register. - ASMJIT_INLINE_NODEBUG void resetElementIndex() noexcept { - _signature &= ~(kSignatureRegElementFlagMask | kSignatureRegElementIndexMask); - } - - ASMJIT_INLINE_NODEBUG constexpr bool isVecB8() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecH4() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecS2() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecD1() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature); } - - ASMJIT_INLINE_NODEBUG constexpr bool isVecB16() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecH8() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecS4() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementS); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecD2() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementD); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecB4x4() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementB4); } - ASMJIT_INLINE_NODEBUG constexpr bool isVecH2x4() const noexcept { return _signature.subset(kBaseSignatureMask | kSignatureRegElementTypeMask) == (RegTraits::kSignature | kSignatureElementH2); } - - //! Creates a cloned register with element access. - ASMJIT_INLINE_NODEBUG Vec at(uint32_t elementIndex) const noexcept { - return Vec((signature() & ~kSignatureRegElementIndexMask) | (elementIndex << kSignatureRegElementIndexShift) | kSignatureRegElementFlagMask, id()); - } - - //! Cast this register to an 8-bit B register (AArch64 only). - ASMJIT_INLINE_NODEBUG VecB b() const noexcept; - //! Cast this register to a 16-bit H register (AArch64 only). - ASMJIT_INLINE_NODEBUG VecH h() const noexcept; - //! Cast this register to a 32-bit S register. - ASMJIT_INLINE_NODEBUG VecS s() const noexcept; - //! Cast this register to a 64-bit D register. - ASMJIT_INLINE_NODEBUG VecD d() const noexcept; - //! Cast this register to a 128-bit Q register. - ASMJIT_INLINE_NODEBUG VecV q() const noexcept; - //! Cast this register to a 128-bit V register. - ASMJIT_INLINE_NODEBUG VecV v() const noexcept; - - //! Cast this register to a 128-bit V.B[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV b(uint32_t elementIndex) const noexcept; - //! Cast this register to a 128-bit V.H[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV h(uint32_t elementIndex) const noexcept; - //! Cast this register to a 128-bit V.S[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV s(uint32_t elementIndex) const noexcept; - //! Cast this register to a 128-bit V.D[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV d(uint32_t elementIndex) const noexcept; - //! Cast this register to a 128-bit V.H2[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV h2(uint32_t elementIndex) const noexcept; - //! Cast this register to a 128-bit V.B4[elementIndex] register. - ASMJIT_INLINE_NODEBUG VecV b4(uint32_t elementIndex) const noexcept; - - //! Cast this register to V.8B. - ASMJIT_INLINE_NODEBUG VecD b8() const noexcept; - //! Cast this register to V.16B. - ASMJIT_INLINE_NODEBUG VecV b16() const noexcept; - //! Cast this register to V.2H. - ASMJIT_INLINE_NODEBUG VecS h2() const noexcept; - //! Cast this register to V.4H. - ASMJIT_INLINE_NODEBUG VecD h4() const noexcept; - //! Cast this register to V.8H. - ASMJIT_INLINE_NODEBUG VecV h8() const noexcept; - //! Cast this register to V.2S. - ASMJIT_INLINE_NODEBUG VecD s2() const noexcept; - //! Cast this register to V.4S. - ASMJIT_INLINE_NODEBUG VecV s4() const noexcept; - //! Cast this register to V.2D. - ASMJIT_INLINE_NODEBUG VecV d2() const noexcept; - - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature _makeElementAccessSignature(uint32_t elementType, uint32_t elementIndex) noexcept { - return OperandSignature{ - uint32_t(RegTraits::kSignature) | - uint32_t(kSignatureRegElementFlagMask) | - uint32_t(elementType << kSignatureRegElementTypeShift) | - uint32_t(elementIndex << kSignatureRegElementIndexShift)}; - } -}; - -//! 32-bit GPW (AArch64) and/or GPR (ARM/AArch32) register. -class GpW : public Gp { ASMJIT_DEFINE_FINAL_REG(GpW, Gp, RegTraits) }; -//! 64-bit GPX (AArch64) register. -class GpX : public Gp { ASMJIT_DEFINE_FINAL_REG(GpX, Gp, RegTraits) }; - -//! 8-bit view (S) of VFP/SIMD register. -class VecB : public Vec { ASMJIT_DEFINE_FINAL_REG(VecB, Vec, RegTraits) }; -//! 16-bit view (S) of VFP/SIMD register. -class VecH : public Vec { ASMJIT_DEFINE_FINAL_REG(VecH, Vec, RegTraits) }; -//! 32-bit view (S) of VFP/SIMD register. -class VecS : public Vec { ASMJIT_DEFINE_FINAL_REG(VecS, Vec, RegTraits) }; -//! 64-bit view (D) of VFP/SIMD register. -class VecD : public Vec { ASMJIT_DEFINE_FINAL_REG(VecD, Vec, RegTraits) }; -//! 128-bit vector register (Q or V). -class VecV : public Vec { ASMJIT_DEFINE_FINAL_REG(VecV, Vec, RegTraits) }; - -ASMJIT_INLINE_NODEBUG GpW Gp::w() const noexcept { return GpW(id()); } -ASMJIT_INLINE_NODEBUG GpX Gp::x() const noexcept { return GpX(id()); } - -ASMJIT_INLINE_NODEBUG VecB Vec::b() const noexcept { return VecB(id()); } -ASMJIT_INLINE_NODEBUG VecH Vec::h() const noexcept { return VecH(id()); } -ASMJIT_INLINE_NODEBUG VecS Vec::s() const noexcept { return VecS(id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::d() const noexcept { return VecD(id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::q() const noexcept { return VecV(id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::v() const noexcept { return VecV(id()); } - -ASMJIT_INLINE_NODEBUG VecV Vec::b(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeB, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeH, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::s(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeS, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::d(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeD, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h2(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeH2, elementIndex), id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::b4(uint32_t elementIndex) const noexcept { return VecV(_makeElementAccessSignature(kElementTypeB4, elementIndex), id()); } - -ASMJIT_INLINE_NODEBUG VecD Vec::b8() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementB}, id()); } -ASMJIT_INLINE_NODEBUG VecS Vec::h2() const noexcept { return VecS(OperandSignature{VecS::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::h4() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecD Vec::s2() const noexcept { return VecD(OperandSignature{VecD::kSignature | kSignatureElementS}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::b16() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementB}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::h8() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementH}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::s4() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementS}, id()); } -ASMJIT_INLINE_NODEBUG VecV Vec::d2() const noexcept { return VecV(OperandSignature{VecV::kSignature | kSignatureElementD}, id()); } - -#ifndef _DOXYGEN -namespace regs { -#endif - -//! Creates a 32-bit W register operand (ARM/AArch64). -static ASMJIT_INLINE_NODEBUG constexpr GpW w(uint32_t id) noexcept { return GpW(id); } -//! Creates a 64-bit X register operand (AArch64). -static ASMJIT_INLINE_NODEBUG constexpr GpX x(uint32_t id) noexcept { return GpX(id); } -//! Creates a 32-bit S register operand (ARM/AArch64). -static ASMJIT_INLINE_NODEBUG constexpr VecS s(uint32_t id) noexcept { return VecS(id); } -//! Creates a 64-bit D register operand (ARM/AArch64). -static ASMJIT_INLINE_NODEBUG constexpr VecD d(uint32_t id) noexcept { return VecD(id); } -//! Creates a 1282-bit V register operand (ARM/AArch64). -static ASMJIT_INLINE_NODEBUG constexpr VecV v(uint32_t id) noexcept { return VecV(id); } - -#ifndef _DOXYGEN -} // {regs} - -// Make `arm::regs` accessible through `arm` namespace as well. -using namespace regs; -#endif - -//! Memory operand (ARM). -class Mem : public BaseMem { -public: - //! \cond INTERNAL - //! Additional bits of operand's signature used by `arm::Mem`. - enum AdditionalBits : uint32_t { - // Index shift value (5 bits). - // |........|.....XXX|XX......|........| - kSignatureMemShiftValueShift = 14, - kSignatureMemShiftValueMask = 0x1Fu << kSignatureMemShiftValueShift, - - // Shift operation type (4 bits). - // |........|XXXX....|........|........| - kSignatureMemPredicateShift = 20, - kSignatureMemPredicateMask = 0x0Fu << kSignatureMemPredicateShift - }; - //! \endcond - - //! Memory offset mode. - //! - //! Additional constants that can be used with the `predicate`. - enum OffsetMode : uint32_t { - //! Pre-index "[BASE, #Offset {, }]!" with write-back. - kOffsetPreIndex = 0xE, - //! Post-index "[BASE], #Offset {, }" with write-back. - kOffsetPostIndex = 0xF - }; - - //! \name Construction & Destruction - //! \{ - - //! Construct a default `Mem` operand, that points to [0]. - ASMJIT_INLINE_NODEBUG constexpr Mem() noexcept - : BaseMem() {} - - ASMJIT_INLINE_NODEBUG constexpr Mem(const Mem& other) noexcept - : BaseMem(other) {} - - ASMJIT_INLINE_NODEBUG explicit Mem(Globals::NoInit_) noexcept - : BaseMem(Globals::NoInit) {} - - ASMJIT_INLINE_NODEBUG constexpr Mem(const Signature& signature, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept - : BaseMem(signature, baseId, indexId, offset) {} - - ASMJIT_INLINE_NODEBUG constexpr explicit Mem(const Label& base, int32_t off = 0, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(RegType::kLabelTag) | - signature, base.id(), 0, off) {} - - ASMJIT_INLINE_NODEBUG constexpr explicit Mem(const BaseReg& base, int32_t off = 0, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - signature, base.id(), 0, off) {} - - ASMJIT_INLINE_NODEBUG constexpr Mem(const BaseReg& base, const BaseReg& index, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - Signature::fromMemIndexType(index.type()) | - signature, base.id(), index.id(), 0) {} - - ASMJIT_INLINE_NODEBUG constexpr Mem(const BaseReg& base, const BaseReg& index, const Shift& shift, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - Signature::fromMemBaseType(base.type()) | - Signature::fromMemIndexType(index.type()) | - Signature::fromValue(uint32_t(shift.op())) | - Signature::fromValue(shift.value()) | - signature, base.id(), index.id(), 0) {} - - ASMJIT_INLINE_NODEBUG constexpr Mem(uint64_t base, Signature signature = Signature{0}) noexcept - : BaseMem(Signature::fromOpType(OperandType::kMem) | - signature, uint32_t(base >> 32), 0, int32_t(uint32_t(base & 0xFFFFFFFFu))) {} - - //! \} - - //! \name Overloaded Operators - //! \{ - - ASMJIT_INLINE_NODEBUG Mem& operator=(const Mem& other) noexcept = default; - - //! \} - - //! \name Clone - //! \{ - - //! Clones the memory operand. - ASMJIT_INLINE_NODEBUG constexpr Mem clone() const noexcept { return Mem(*this); } - - //! Gets new memory operand adjusted by `off`. - ASMJIT_INLINE_NODEBUG Mem cloneAdjusted(int64_t off) const noexcept { - Mem result(*this); - result.addOffset(off); - return result; - } - - //! Clones the memory operand and makes it pre-index. - ASMJIT_INLINE_NODEBUG Mem pre() const noexcept { - Mem result(*this); - result.setPredicate(kOffsetPreIndex); - return result; - } - - //! Clones the memory operand, applies a given offset `off` and makes it pre-index. - ASMJIT_INLINE_NODEBUG Mem pre(int64_t off) const noexcept { - Mem result(*this); - result.setPredicate(kOffsetPreIndex); - result.addOffset(off); - return result; - } - - //! Clones the memory operand and makes it post-index. - ASMJIT_INLINE_NODEBUG Mem post() const noexcept { - Mem result(*this); - result.setPredicate(kOffsetPostIndex); - return result; - } - - //! Clones the memory operand, applies a given offset `off` and makes it post-index. - ASMJIT_INLINE_NODEBUG Mem post(int64_t off) const noexcept { - Mem result(*this); - result.setPredicate(kOffsetPostIndex); - result.addOffset(off); - return result; - } - - //! \} - - //! \name Base & Index - //! \{ - - //! Converts memory `baseType` and `baseId` to `arm::Reg` instance. - //! - //! The memory must have a valid base register otherwise the result will be wrong. - ASMJIT_INLINE_NODEBUG Reg baseReg() const noexcept { return Reg::fromTypeAndId(baseType(), baseId()); } - - //! Converts memory `indexType` and `indexId` to `arm::Reg` instance. - //! - //! The memory must have a valid index register otherwise the result will be wrong. - ASMJIT_INLINE_NODEBUG Reg indexReg() const noexcept { return Reg::fromTypeAndId(indexType(), indexId()); } - - using BaseMem::setIndex; - - ASMJIT_INLINE_NODEBUG void setIndex(const BaseReg& index, uint32_t shift) noexcept { - setIndex(index); - setShift(shift); - } - - //! \} - - //! \name ARM Specific Features - //! \{ - - //! Gets whether the memory operand has shift (aka scale) constant. - ASMJIT_INLINE_NODEBUG constexpr bool hasShift() const noexcept { return _signature.hasField(); } - //! Gets the memory operand's shift (aka scale) constant. - ASMJIT_INLINE_NODEBUG constexpr uint32_t shift() const noexcept { return _signature.getField(); } - //! Sets the memory operand's shift (aka scale) constant. - ASMJIT_INLINE_NODEBUG void setShift(uint32_t shift) noexcept { _signature.setField(shift); } - //! Resets the memory operand's shift (aka scale) constant to zero. - ASMJIT_INLINE_NODEBUG void resetShift() noexcept { _signature.setField(0); } - - //! Gets memory predicate (shift mode or offset mode), see \ref ShiftOp and \ref OffsetMode. - ASMJIT_INLINE_NODEBUG constexpr uint32_t predicate() const noexcept { return _signature.getField(); } - //! Sets memory predicate to `predicate`, see `Mem::ShiftOp`. - ASMJIT_INLINE_NODEBUG void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } - //! Resets shift mode to LSL (default). - ASMJIT_INLINE_NODEBUG void resetPredicate() noexcept { _signature.setField(0); } - - ASMJIT_INLINE_NODEBUG constexpr bool isFixedOffset() const noexcept { return predicate() < kOffsetPreIndex; } - ASMJIT_INLINE_NODEBUG constexpr bool isPreOrPost() const noexcept { return predicate() >= kOffsetPreIndex; } - ASMJIT_INLINE_NODEBUG constexpr bool isPreIndex() const noexcept { return predicate() == kOffsetPreIndex; } - ASMJIT_INLINE_NODEBUG constexpr bool isPostIndex() const noexcept { return predicate() == kOffsetPostIndex; } - - ASMJIT_INLINE_NODEBUG void resetToFixedOffset() noexcept { resetPredicate(); } - ASMJIT_INLINE_NODEBUG void makePreIndex() noexcept { setPredicate(kOffsetPreIndex); } - ASMJIT_INLINE_NODEBUG void makePostIndex() noexcept { setPredicate(kOffsetPostIndex); } - - //! \} -}; - -//! Creates `[base, offset]` memory operand (offset mode). -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(const Gp& base, int32_t offset = 0) noexcept { - return Mem(base, offset); -} - -//! Creates `[base, offset]!` memory operand (pre-index mode). -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr_pre(const Gp& base, int32_t offset = 0) noexcept { - return Mem(base, offset, OperandSignature::fromValue(Mem::kOffsetPreIndex)); -} - -//! Creates `[base], offset` memory operand (post-index mode). -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr_post(const Gp& base, int32_t offset = 0) noexcept { - return Mem(base, offset, OperandSignature::fromValue(Mem::kOffsetPostIndex)); -} - -//! Creates `[base, index]` memory operand. -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(const Gp& base, const Gp& index) noexcept { - return Mem(base, index); -} - -//! Creates `[base, index]!` memory operand (pre-index mode). -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr_pre(const Gp& base, const Gp& index) noexcept { - return Mem(base, index, OperandSignature::fromValue(Mem::kOffsetPreIndex)); -} - -//! Creates `[base], index` memory operand (post-index mode). -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr_post(const Gp& base, const Gp& index) noexcept { - return Mem(base, index, OperandSignature::fromValue(Mem::kOffsetPostIndex)); -} - -//! Creates `[base, index, SHIFT_OP #shift]` memory operand. -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(const Gp& base, const Gp& index, const Shift& shift) noexcept { - return Mem(base, index, shift); -} - -//! Creates `[base, offset]` memory operand. -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(const Label& base, int32_t offset = 0) noexcept { - return Mem(base, offset); -} - -// TODO: [ARM] PC + offset address. -#if 0 -//! Creates `[PC + offset]` (relative) memory operand. -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(const PC& pc, int32_t offset = 0) noexcept { - return Mem(pc, offset); -} -#endif - -//! Creates `[base]` absolute memory operand. -//! -//! \note The concept of absolute memory operands doesn't exist on ARM, the ISA only provides PC relative addressing. -//! Absolute memory operands can only be used if it's known that the PC relative offset is encodable and that it -//! would be within the limits. Absolute address is also often output from disassemblers, so AsmJit supports it to -//! make it possible to assemble such output back. -static ASMJIT_INLINE_NODEBUG constexpr Mem ptr(uint64_t base) noexcept { return Mem(base); } - -//! \} - -ASMJIT_END_SUB_NAMESPACE - -//! \cond INTERNAL -ASMJIT_BEGIN_NAMESPACE -ASMJIT_DEFINE_TYPE_ID(arm::GpW, TypeId::kInt32); -ASMJIT_DEFINE_TYPE_ID(arm::GpX, TypeId::kInt64); -ASMJIT_DEFINE_TYPE_ID(arm::VecS, TypeId::kFloat32x1); -ASMJIT_DEFINE_TYPE_ID(arm::VecD, TypeId::kFloat64x1); -ASMJIT_DEFINE_TYPE_ID(arm::VecV, TypeId::kInt32x4); -ASMJIT_END_NAMESPACE -//! \endcond - -#endif // ASMJIT_ARM_ARMOPERAND_H_INCLUDED diff --git a/pe-packer/asmjit/arm/armutils.h b/pe-packer/asmjit/arm/armutils.h index a32256c..b81f72f 100644 --- a/pe-packer/asmjit/arm/armutils.h +++ b/pe-packer/asmjit/arm/armutils.h @@ -1,11 +1,12 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_ARM_ARMUTILS_H_INCLUDED #define ASMJIT_ARM_ARMUTILS_H_INCLUDED +#include "../core/support.h" #include "../arm/armglobals.h" ASMJIT_BEGIN_SUB_NAMESPACE(arm) @@ -16,6 +17,38 @@ ASMJIT_BEGIN_SUB_NAMESPACE(arm) //! Public utilities and helpers for targeting AArch32 and AArch64 architectures. namespace Utils { +//! Encodes a 12-bit immediate part of opcode that ise used by a standard 32-bit ARM encoding. +[[maybe_unused]] +static inline bool encodeAArch32Imm(uint64_t imm, uint32_t* encodedImmOut) noexcept { + if (imm & 0xFFFFFFFF00000000u) + return false; + + uint32_t v = uint32_t(imm); + uint32_t r = 0; + + if (v <= 0xFFu) { + *encodedImmOut = v; + return true; + } + + // Rotate if there are bits on both ends (LSB and MSB) + // (otherwise we would not be able to calculate the rotation with ctz). + if (v & 0xFF0000FFu) { + v = Support::ror(v, 16); + r = 16u; + } + + uint32_t n = Support::ctz(v) & ~0x1u; + r = (r - n) & 0x1Eu; + v = Support::ror(v, n); + + if (v > 0xFFu) + return false; + + *encodedImmOut = v | (r << 7); + return true; +} + //! Decomposed fields of a logical immediate value. struct LogicalImm { uint32_t n; @@ -40,7 +73,7 @@ struct LogicalImm { //! | 0 | 11110s | .....r | 2 | //! +---+--------+--------+------+ //! ``` -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static bool encodeLogicalImm(uint64_t imm, uint32_t width, LogicalImm* out) noexcept { // Determine the element width, which must be 2, 4, 8, 16, 32, or 64 bits. do { @@ -88,12 +121,19 @@ static bool encodeLogicalImm(uint64_t imm, uint32_t width, LogicalImm* out) noex //! Returns true if the given `imm` value is encodable as a logical immediate. The `width` argument describes the //! width of the operation, and must be either 32 or 64. This function can be used to test whether an immediate //! value can be used with AND, ANDS, BIC, BICS, EON, EOR, ORN, and ORR instruction. -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_INLINE_NODEBUG bool isLogicalImm(uint64_t imm, uint32_t width) noexcept { LogicalImm dummy; return encodeLogicalImm(imm, width, &dummy); } +//! Returns true if the given `imm` value is encodable as an immediate with `add` and `sub` instructions on AArch64. +//! These two instructions can encode 12-bit immediate value optionally shifted left by 12 bits. +[[maybe_unused]] +static ASMJIT_INLINE_NODEBUG bool isAddSubImm(uint64_t imm) noexcept { + return imm <= 0xFFFu || (imm & ~uint64_t(0xFFFu << 12)) == 0; +} + //! Returns true if the given `imm` value is a byte mask. Byte mask has each byte part of the value set to either //! 0x00 or 0xFF. Some ARM instructions accept immediates that form a byte-mask and this function can be used to //! verify that the immediate is encodable before using the value. @@ -113,7 +153,7 @@ static ASMJIT_INLINE_NODEBUG uint32_t encodeImm64ByteMaskToImm8(uint64_t imm) no //! \cond //! A generic implementation that checjs whether a floating point value can be converted to ARM Imm8. template -static ASMJIT_FORCE_INLINE bool isFPImm8Generic(T val) noexcept { +static ASMJIT_INLINE bool isFPImm8Generic(T val) noexcept { constexpr uint32_t kAllBsMask = Support::lsbMask(kNumBBits); constexpr uint32_t kB0Pattern = Support::bitMask(kNumBBits - 1); constexpr uint32_t kB1Pattern = kAllBsMask ^ kB0Pattern; diff --git a/pe-packer/asmjit/asmjit-scope-begin.h b/pe-packer/asmjit/asmjit-scope-begin.h index 93397b5..78ba411 100644 --- a/pe-packer/asmjit/asmjit-scope-begin.h +++ b/pe-packer/asmjit/asmjit-scope-begin.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifdef _WIN32 diff --git a/pe-packer/asmjit/asmjit-scope-end.h b/pe-packer/asmjit/asmjit-scope-end.h index 702cef4..b11ca22 100644 --- a/pe-packer/asmjit/asmjit-scope-end.h +++ b/pe-packer/asmjit/asmjit-scope-end.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifdef _WIN32 diff --git a/pe-packer/asmjit/asmjit.h b/pe-packer/asmjit/asmjit.h index 1cd0651..b95350a 100644 --- a/pe-packer/asmjit/asmjit.h +++ b/pe-packer/asmjit/asmjit.h @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Zlib // Official GitHub Repository: https://github.com/asmjit/asmjit // -// Copyright (c) 2008-2021 The AsmJit Authors +// Copyright (c) 2008-2024 The AsmJit Authors // // This software is provided 'as-is', without any express or implied // warranty. In no event will the authors be held liable for any damages @@ -24,6 +24,8 @@ #ifndef ASMJIT_ASMJIT_H_INCLUDED #define ASMJIT_ASMJIT_H_INCLUDED +#pragma message("asmjit/asmjit.h is deprecated! Please use asmjit/[core|x86|a64|host].h instead.") + #include "./core.h" #ifndef ASMJIT_NO_X86 diff --git a/pe-packer/asmjit/core.h b/pe-packer/asmjit/core.h index 19092ae..216fa79 100644 --- a/pe-packer/asmjit/core.h +++ b/pe-packer/asmjit/core.h @@ -1,7 +1,26 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib +// Official GitHub Repository: https://github.com/asmjit/asmjit +// +// Copyright (c) 2008-2025 The AsmJit Authors +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + #ifndef ASMJIT_CORE_H_INCLUDED #define ASMJIT_CORE_H_INCLUDED @@ -17,7 +36,7 @@ namespace asmjit { //! functionality is within \ref asmjit namespace and architecture specific functionality is always in its own //! namespace. For example \ref asmjit::x86 provides both 32-bit and 64-bit X86 code generation. //! -//! \section main_groups Documentation Groups +//! \section doc_groups Documentation Groups //! //! AsmJit documentation is structured into groups. Groups can be followed in order to learn AsmJit, but knowledge //! from multiple groups is required to use AsmJit properly: @@ -78,7 +97,7 @@ namespace asmjit { //! output from \ref Logger is always necessary when filling bug reports. In other words, using logging and proper error //! handling can save a lot of time during the development and can also save users from submitting issues. //! -//! \section main_other Other Pages +//! \section other_pages Other Pages //! //! - Class List - List of classes sorted alphabetically //! - AsmJit Namespace - List of symbols provided by `asmjit` namespace @@ -96,35 +115,37 @@ namespace asmjit { //! your project and to define \ref ASMJIT_STATIC. AsmJit can be just updated from time to time without any changes to //! this integration process. Do not embed AsmJit's `test` files in such case as these are used exclusively for testing. //! +//! \section supported_environment Supported Operating Systems, Compilers, and Build Tools +//! //! ### Supported C++ Compilers //! //! - Requirements: //! -//! - AsmJit won't build without C++11 enabled. If you use older GCC or Clang you would have to enable at least -//! C++11 standard through compiler flags. +//! - AsmJit won't build without C++17 enabled. If you use older GCC or Clang you would have to enable at least +//! C++17 standard through compiler flags. //! //! - Tested: //! -//! - **Clang** - Tested by GitHub Actions - Clang 10+ is officially supported and tested by CI, older Clang versions -//! having C++11 should work, but are not tested anymore due to upgraded CI images. +//! - **Clang** - Tested by GitHub Actions - Clang 14+ is officially supported and tested by CI, older Clang versions +//! having C++17 should work, but these versions are not tested anymore due to upgraded CI images. //! -//! - **GNU** - Tested by GitHub Actions - GCC 7+ is officially supported, older GCC versions from 4.8+ having C++11 -//! enabled should also work, but are not tested anymore due to upgraded CI images. +//! - **GNU** - Tested by GitHub Actions - GCC 9+ is officially supported and tested by CI, older GCC versions such +//! as GCC 7 should work, but these versions are not tested anymore due to upgraded CI images. //! -//! - **MINGW** - Reported to work, but not tested in our CI environment (help welcome). +//! - **MINGW** - Reported to work, but not tested in our CI environment (help welcome!). //! -//! - **MSVC** - Tested by GitHub Actions - VS2019+ is officially supported, VS2015 and VS2017 is reported to work, -//! but not tested by CI anymore. +//! - **MSVC** - Tested by GitHub Actions - VS2022 and onwards are officially supported and tested by CI, VS2015, +//! VS2017, and VS2019 are not tested anymore due to upgraded CI images. VS2019 should work well. //! //! ### Supported Operating Systems and Platforms //! //! - Tested: //! //! - **BSD** - FreeBSD, NetBSD, and OpenBSD tested by GitHub Actions (only recent images are tested by CI). BSD -//! runners only test BSD images with clang compiler. +//! runners only test BSD images with Clang compiler. Both X86_64 and AArch64 host builds are tested. //! //! - **Linux** - Tested by GitHub Actions (only recent Ubuntu images are tested by CI, in general any distribution -//! should be supported as AsmJit has no dependencies). +//! should be supported as AsmJit has no dependencies). Linux tests X86, X86_64, and AArch64 host builds. //! //! - **Mac OS** - Tested by GitHub Actions. //! @@ -145,10 +166,9 @@ namespace asmjit { //! ### Supported Backends / Architectures //! //! - **X86** and **X86_64** - Both 32-bit and 64-bit backends tested on CI. -//! - **AArch64** - AArch64 backend is currently only partially tested (there is no native AArch64 runner to test -//! AsmJit Builder/Compiler). +//! - **AArch64** - Tested on CI (Native Apple runners and Linux emulated via QEMU). //! -//! ### Static Builds and Embedding +//! \section build_mode Static Builds and Embedding //! //! These definitions can be used to enable static library build. Embed is used when AsmJit's source code is embedded //! directly in another project, implies static build as well. @@ -160,7 +180,31 @@ namespace asmjit { //! otherwise AsmJit would use dynamic library imports in \ref ASMJIT_API decorator. The recommendation is to define //! this macro across the whole project that uses AsmJit this way. //! -//! ### Build Configuration +//! \section cmake_integration CMake Integration +//! +//! AsmJit has a first-class CMake support. When consuming AsmJit as a cmake dependency, just use `asmjit::asmjit` +//! as a link dependency, which would instrument cmake to setup everything else, including include paths, and build +//! flags (either defining `ASMJIT_STATIC` or not, and possibly defining other AsmJit feature macros). For example +//! considering that AsmJit was fetched to `3rdparty/asmjit` directory in your project as an external dependency, +//! you can just use the following CMake snippet that integrates AsmJit with your own CMake project: +//! +//! ```cmake +//! cmake_minimum_required(VERSION 3.30) +//! +//! project(asmjit_consumer C CXX) # Both C and CXX are required. +//! set(CMAKE_CXX_STANDARD 17) # C++17 and never is supported. +//! +//! set(ASMJIT_DIR "3rdparty/asmjit") # Location of AsmJit. +//! set(ASMJIT_STATIC TRUE) # Force static build. +//! +//! add_subdirectory("${ASMJIT_DIR}") # This adds AsmJit as a part of your project. +//! +//! add_executable(asmjit_consumer asmjit_consumer.cpp) +//! target_link_libraries( +//! asmjit_consumer asmjit::asmjit) # This adds AsmJit as a dependency to your target. +//! ``` +//! +//! \section build_type Build Type Configuration //! //! These definitions control whether asserts are active or not. By default AsmJit would autodetect build configuration //! from existing pre-processor definitions, but this behavior can be overridden, for example to enable debug asserts @@ -174,43 +218,52 @@ namespace asmjit { //! were not used. We only recommend using build configuration overrides in special situations, like using AsmJit in //! release configuration with asserts enabled for whatever reason. //! -//! ### AsmJit Backends +//! \section build_backends AsmJit Backends //! -//! AsmJit currently supports only X86/X64 backend, but the plan is to add more backends in the future. By default -//! AsmJit builds only the host backend, which is auto-detected at compile-time, but this can be overridden. +//! All backends AsmJit supports are included by default. To exclude a backend use the following build-type macros: //! -//! - \ref ASMJIT_NO_X86 - Disable X86/X64 backends. -//! - \ref ASMJIT_NO_FOREIGN - Disables the support for foreign architectures. +//! - \ref ASMJIT_NO_X86 - Disables both X86 and X86_64 backends. +//! - \ref ASMJIT_NO_AARCH64 - Disables AArch64 backend. +//! - \ref ASMJIT_NO_FOREIGN - Disables the support for foreign architecture backends, only keeps a native backend. +//! For example if your target is X86, `ASMJIT_NO_FOREIGN` would disable every backend but X86. //! -//! ### Features Selection -//! -//! AsmJit builds by defaults all supported features, which includes all emitters, logging, instruction validation and -//! introspection, and JIT memory allocation. Features can be disabled at compile time by using `ASMJIT_NO_...` -//! definitions. +//! \section build_options Build Options //! //! - \ref ASMJIT_NO_DEPRECATED - Disables deprecated API at compile time so it won't be available and the //! compilation will fail if there is attempt to use such API. This includes deprecated classes, namespaces, //! enumerations, and functions. //! -//! - \ref ASMJIT_NO_BUILDER - Disables \ref asmjit_builder functionality completely. This implies \ref -//! ASMJIT_NO_COMPILER as \ref asmjit_compiler cannot be used without \ref asmjit_builder. +//! - \ref ASMJIT_NO_SHM_OPEN - Disables functionality that uses `shm_open()`. //! -//! - \ref ASMJIT_NO_COMPILER - Disables \ref asmjit_compiler functionality completely. +//! - \ref ASMJIT_NO_ABI_NAMESPACE - Disables inline ABI namespace within `asmjit` namespace. This is only provided +//! for users that control all the dependencies (even transitive ones) and that make sure that no two AsmJit +//! versions are used at the same time. This option can be debugging a little simpler as there would not be ABI +//! tag after `asmjit::` namespace. Otherwise asmjit would look like `asmjit::_abi_1_13::`, for example. //! +//! \section build_features Build Features +//! +//! AsmJit builds by default all supported features, which includes all emitters, logging, instruction validation and +//! introspection, and JIT memory allocation. Features can be disabled at compile time by using `ASMJIT_NO_...` +//! definitions. //! - \ref ASMJIT_NO_JIT - Disables JIT memory management and \ref JitRuntime. //! -//! - \ref ASMJIT_NO_LOGGING - Disables \ref Logger and \ref Formatter. -//! //! - \ref ASMJIT_NO_TEXT - Disables everything that contains string representation of AsmJit constants, should //! be used together with \ref ASMJIT_NO_LOGGING as logging doesn't make sense without the ability to query //! instruction names, register names, etc... //! +//! - \ref ASMJIT_NO_LOGGING - Disables \ref Logger and \ref Formatter. +//! //! - \ref ASMJIT_NO_VALIDATION - Disables validation API. //! //! - \ref ASMJIT_NO_INTROSPECTION - Disables instruction introspection API, must be used together with \ref //! ASMJIT_NO_COMPILER as \ref asmjit_compiler requires introspection for its liveness analysis and register //! allocation. //! +//! - \ref ASMJIT_NO_BUILDER - Disables \ref asmjit_builder functionality completely. This implies \ref +//! ASMJIT_NO_COMPILER as \ref asmjit_compiler cannot be used without \ref asmjit_builder. +//! +//! - \ref ASMJIT_NO_COMPILER - Disables \ref asmjit_compiler functionality completely. +//! //! \note It's not recommended to disable features if you plan to build AsmJit as a shared library that will be //! used by multiple projects that you don't control how AsmJit was built (for example AsmJit in a Linux distribution). //! The possibility to disable certain features exists mainly for customized AsmJit builds. @@ -227,17 +280,164 @@ namespace asmjit { //! removed APIs and should serve as a how-to guide for people that want to port existing code to work with the //! newest AsmJit. //! -//! ### Tips +//! \section tips Tips //! //! Useful tips before you start: //! -//! - Visit our [Public Gitter Channel](https://gitter.im/asmjit/asmjit) if you need a quick help. +//! - Visit our [Public Chat](https://app.gitter.im/#/room/#asmjit:gitter.im) if you need a quick help. //! //! - Build AsmJit with `ASMJIT_NO_DEPRECATED` macro defined to make sure that you are not using deprecated -//! functionality at all. Deprecated functions are decorated with `ASMJIT_DEPRECATED()` macro, but sometimes +//! functionality at all. Deprecated functions are decorated with `[[deprecated]]` attribute, but sometimes //! it's not possible to decorate everything like classes, which are used by deprecated functions as well, //! because some compilers would warn about that. If your project compiles fine with `ASMJIT_NO_DEPRECATED` -//! it's not using anything, which was deprecated. +//! it's not using anything, which will be definitely removed in the future. +//! +//! \section api_changes API Changes +//! +//! ### Changes committed at 2025-06-15 +//! +//! Core changes: +//! +//! - No more architecture specific \ref RegTraits - removed `BaseRegTraits` and kept just \ref RegTraits: +//! +//! - `BaseRegTraits` -> `RegTraits` +//! - `arm::RegTraits` -> `RegTraits` +//! - `x86::RegTraits` -> `RegTraits` +//! +//! - Removed register signature and helper functions from ArchTraits. This functionality is now available +//! via asmjit::RegTraits and asmjit::RegUtils and doesn't require a valid architecture traits instance. +//! +//! - No more architecture specific Gp/Vec/Mask register types in \ref RegType and \ref RegGroup: +//! +//! - `RegGroup::kX86_Rip` -> `RegGroup::kPC` +//! - `RegGroup::kX86_KReg` -> `RegGroup::kMask` +//! - `RegType::kX86_GpbLo` -> `RegType::kGp8Lo` +//! - `RegType::kX86_GpbLo` -> `RegType::kGp8Lo` +//! - `RegType::kX86_GpbHi` -> `RegType::kGp8Hi` +//! - `RegType::kX86_Gpw` -> `RegType::kGp16` +//! - `RegType::kX86_Gpd` -> `RegType::kGp32` +//! - `RegType::kX86_Gpq` -> `RegType::kGp64` +//! - `RegType::kX86_Xmm` -> `RegType::kVec128` +//! - `RegType::kX86_Ymm` -> `RegType::kVec256` +//! - `RegType::kX86_Zmm` -> `RegType::kVec512` +//! - `RegType::kX86_KReg` -> `RegType::kMask` +//! - `RegType::kARM_PC` -> `RegType::kPC` +//! - `RegType::kARM_GpW` -> `RegType::kGp32` +//! - `RegType::kARM_GpX` -> `RegType::kGp64` +//! - `RegType::kARM_VecB` -> `RegType::kVec8` +//! - `RegType::kARM_VecH` -> `RegType::kVec16` +//! - `RegType::kARM_VecS` -> `RegType::kVec32` +//! - `RegType::kARM_VecD` -> `RegType::kVec64` +//! - `RegType::kARM_VecQ` -> `RegType::kVec128` +//! - `RegType::kARM_VecV` -> `RegType::kVec128` +//! +//! - Renamed `asmjit::BaseReg` to asmjit::Reg, added `asmjit::UniGp` and `asmjit::UniVec` which are now base +//! classes for platform specific `[x86|a64]::Gp` and `[x86|a64]::Vec` +//! +//! - Gp and Vec register API is now more platform independent - use `isGp32()` instead of `isGpd()`, similarly, +//! use `isGp64` instead of `isGpq()` (X86_64) or `isGpX()` (AArch64), etc... The same applies to vectors - +//! use `isVec128()` instead of `isXmm()` (X86) or `isVecQ()` (AArch64), `isVec256()` instead of `isYmm()`, +//! etc... +//! +//! - Renamed some member functions in Operand and Reg: +//! +//! - `isType(regId)` -> `isReg(regId)` +//! - `isGroup(regGroup)` -> `isReg(regGroup)` +//! - `regOp.type()` -> `regOp.regType()` +//! - `regOp.group()` -> `regOp.regGroup()` +//! +//! - Removed some static functions from \ref Operand, \reg Reg, etc... in favor of member functions. Most +//! of the operand functionality is now provided by \ref Operand_: +//! +//! - `Operand::isGp(op)` -> op.isGp(); +//! - `x86::Reg::isGp(op, id)` -> op.isGp(id); +//! +//! - Removed sub-registers `x86::Gpb`, `x86::GpbLo`, `x86::GpbHi`, `x86::Gpw`, `x86::Gpd`, `x86::Gpq`, `x86::Xmm`, +//! `x86::Ymm`, `x86::Zmm` - use just `x86::Gp` and `x86::Vec`, which represent all removed X86 sub-registers. +//! From now, use `x86::Gp` to work with x86 general purpose registers and `x86::Vec` to work with XMM, YMM, +//! and ZMM registers. +//! +//! - Removed sub-registers `a64::GpW` and `a64::GpX`, `a64::VecB`, `a64::VecH`, `a64::VecS`, `a64::VecD`, +//! `a64::VecV`, which represent all removed AArch64 sub-registers. From now, use `a64::Gp` to work with +//! general purpose registers and `a64::Vec` to work with NEON registers of any size, element type, and +//! element index. +//! +//! - Since sub-register types are gone it's no longer possible to write `gpb(id)` to get AL register, etc... +//! However, all register operands that can hold multiple register types now offer `Reg::make_xxx(id)` API, +//! which can be used as a convenience or just use platform specific API like `x86::gpd(id)` or `x86::gp32(id)`, +//! similarly for AArch64 use `a64::x(id)` or `a64::gp64(id)`, etc... +//! +//! - Renamed some id getters - `Section::id()` -> `Section::sectionId()`, etc... +//! +//! Builder changes: +//! +//! - Removed BaseBuilder::deletePass() - this function was most likely never used by user code and it was also +//! never used by AsmJit. Passes should be only added and not removed, which simplifies some planned future +//! changes. +//! +//! ### Changes committed at 2025-05-24 +//! +//! Core changes: +//! +//! - AsmJit now requires C++17 to compile. +//! +//! - Deprecated asmjit/asmjit.h header. Use asmjit/core.h to include everything except backend specific stuff, +//! and asmjit/x86.h or asmjit/a64.h to include tools of a specific architecture. At this time the asmjit.h +//! header is just deprecated, so it will still work as it used to for some time. +//! +//! ### Changes committed at 2025-05-10 +//! +//! Core changes: +//! +//! - Removed AVX512 functionality that was never used on x86 hardware as Xeon Phi was never supported by AsmJit: +//! +//! - AVX512_ER +//! - AVX512_PF +//! - AVX512_4FMAPS +//! - AVX512_4VNNIW +//! +//! - Instruction 'vcvtneps2bf16' no longer accepts memory operand without explicit size (to minimize ambiguity) +//! +//! ### Changes committed at 2024-01-01 +//! +//! Core changes: +//! +//! - Renamed equality functions `eq()` to `equals()` - Only related to `String`, `ZoneVector`, and `CpuFeatures`. +//! Old function names were deprecated. +//! +//! - Removed `CallConvId::kNone` in favor of `CallConvId::kCDecl`, which is now the default calling convention. +//! +//! - Deprecated `CallConvId::kHost` in favor of `CallConvId::kCDecl` - host calling convention is now not part +//! of CallConvId, it can be calculated from CallConvId and Environment instead. +//! +//! ### Changes committed at 2023-12-27 +//! +//! Core changes: +//! +//! - Renamed `a64::Vec::ElementType` to `a64::VecElementType` and made it a typed enum. This enum was used mostly +//! internally, but there is a public API using it, so it's a breaking change. +//! +//! - Refactored `FuncSignature`, `FuncSignatureT`, and `FuncSignatureBuilder`. There is only `FuncSignature` now, +//! which acts as a function signature holder and builder. Replace `FuncSignatureBuilder` with `FuncSignature` +//! and use `FuncSignature::build` instead of `FuncSignatureT`. The old API has been deprecated. +//! +//! - The maximum number of function arguments was raised from 16 to 32. +//! +//! ### Changes committed at 2023-12-26 +//! +//! Core changes: +//! +//! - Reworked InstNode and InstExNode to be friendlier to static analysis and to not cause undefined behavior. +//! InstNode has no operands visually embedded within the struct so there is no _opArray (which was internal). +//! This means that sizeof(InstNode) changed, but since it's allocated by AsmJit this should be fine. Moreover, +//! there is no longer InstExNode as that was more a hack, instead there is now InstNodeWithOperands, which is +//! a template and specifies the number of operands embedded (InstNode accesses these). All nodes that inherited +//! InstExNode now just inherit InstNodeWithOperands, which would provide the same +//! number of nodes as InstNode. +//! +//! - Moved GP and Vec registers from asmjit::arm namespace to asmjit::a64 namespace. At this time there was +//! no prior deprecation as having arm::Vec would collide with a64::Vec as arm namespace is used within a64 +//! namespace. Just change `arm::Gp` to `a64::Gp` and `arm::Vec` to `a64::Vec`. //! //! ### Changes committed at 2023-09-10 //! @@ -419,7 +619,7 @@ namespace asmjit { //! // Calling a function (Compiler) changed - use invoke() instead of call(). //! void functionInvocation(x86::Compiler& cc) { //! InvokeNode* invokeNode; -//! cc.invoke(&invokeNode, targetOperand, FuncSignatureT<...>(...)); +//! cc.invoke(&invokeNode, targetOperand, FuncSignature::build<...>(...)); //! } //! ``` @@ -434,12 +634,13 @@ namespace asmjit { //! //! - \ref Section - stores information about a code or data section. //! - \ref CodeBuffer - stores actual code or data, part of \ref Section. -//! - \ref LabelEntry - stores information about a label - its name, offset, section where it belongs to, and -//! other bits. -//! - \ref LabelLink - stores information about yet unbound label, which was already used by the assembler. +//! - \ref LabelEntry - stores information about a \ref Label - its name, offset, section where it belongs to, +//! and other bits. +//! - \ref Fixup - stores information about positions in code that needs to be fixed up later, for example +//! when referencing a \ref Label, which was not bound to a position in code yet. //! - \ref RelocEntry - stores information about a relocation. -//! - \ref AddressTableEntry - stores information about an address, which was used in a jump or call. Such -//! address may need relocation. +//! - \ref AddressTableEntry - stores information about an absolute address, which was used in a jump or call. +//! Code referencing an absolute address may need relocation or a record in address table. //! //! To generate code you would need to instantiate at least the following classes: //! @@ -464,7 +665,7 @@ namespace asmjit { //! //! \note CodeHolder examples use \ref x86::Assembler as abstract interfaces cannot be used to generate code. //! -//! ### CodeHolder & Emitters +//! \section code_holder_and_emitters CodeHolder & Emitters //! //! The example below shows how the mentioned classes interact to generate X86 code: //! @@ -475,7 +676,7 @@ namespace asmjit { //! using namespace asmjit; //! //! // Signature of the generated function. -//! typedef int (*Func)(void); +//! using Func = int (*)(void); //! //! int main() { //! JitRuntime rt; // Runtime specialized for JIT code execution. @@ -513,7 +714,7 @@ namespace asmjit { //! - \ref asmjit_builder - Low-level emitter that emits to a \ref BaseNode list. //! - \ref asmjit_compiler - High-level emitter that provides register allocation. //! -//! ### Targets and JitRuntime +//! \section targets_and_jit_runtime Targets and JitRuntime //! //! AsmJit's \ref Target is an interface that provides basic target abstraction. At the moment AsmJit provides only //! one implementation called \ref JitRuntime, which as the name suggests provides JIT code target and execution @@ -582,7 +783,7 @@ namespace asmjit { //! } //! ``` //! -//! ### Explicit Code Relocation +//! \section explicit_code_relocation Explicit Code Relocation //! //! In addition to \ref Environment, \ref CodeHolder can be configured to specify a base-address (or a virtual base //! address in a linker terminology), which could be static (useful when you know the location where the target's @@ -592,21 +793,21 @@ namespace asmjit { //! required if you call external functions from the generated code that cannot be encoded by using a 32-bit //! displacement (64-bit displacements are not provided by aby supported architecture). //! -//! There is also a concept called \ref LabelLink - label link is a lightweight data structure that doesn't have any -//! identifier and is stored in \ref LabelEntry as a single-linked list. Label link represents either unbound yet used -//! label and cross-sections links (only relevant to code that uses multiple sections). Since crossing sections is -//! something that cannot be resolved immediately these links persist until offsets of these sections are assigned and -//! until \ref CodeHolder::resolveUnresolvedLinks() is called. It's an error if you end up with code that has -//! unresolved label links after flattening. You can verify it by calling \ref CodeHolder::hasUnresolvedLinks(), which -//! inspects the value returned by \ref CodeHolder::unresolvedLinkCount(). +//! There is also a concept called \ref Fixup - it's a lightweight data structure that doesn't have any identifier and +//! is stored in \ref LabelEntry and \ref CodeHolder as a single-linked list. Fixup represents either a reference to an +//! unbound label and cross-sections references (only relevant to code that uses multiple sections). Since crossing +//! sections is something that cannot be resolved immediately these fixups persist until offsets of these sections are +//! assigned and until \ref CodeHolder::resolveCrossSectionFixups() is called. It's an error if you end up with code that +//! still has fixups after flattening. You can verify it by calling \ref CodeHolder::hasUnresolvedFixups(), which inspects +//! the value returned by \ref CodeHolder::unresolvedFixupCount(). //! //! AsmJit can flatten code that uses multiple sections by assigning each section an incrementing offset that respects //! its alignment. Use \ref CodeHolder::flatten() to do that. After the sections are flattened their offsets and //! virtual sizes are adjusted to respect each section's buffer size and alignment. The \ref -//! CodeHolder::resolveUnresolvedLinks() function must be called before relocating the code held by \ref CodeHolder. +//! CodeHolder::resolveCrossSectionFixups() function must be called before relocating the code held by \ref CodeHolder. //! You can also flatten your code manually by iterating over all sections and calculating their offsets (relative to //! base) by your own algorithm. In that case \ref CodeHolder::flatten() should not be called, however, -//! \ref CodeHolder::resolveUnresolvedLinks() should be. +//! \ref CodeHolder::resolveCrossSectionFixups() should be. //! //! The example below shows how to use a built-in virtual memory allocator \ref JitAllocator instead of using \ref //! JitRuntime (just in case you want to use your own memory management) and how to relocate the generated code @@ -622,7 +823,7 @@ namespace asmjit { //! //! using namespace asmjit; //! -//! typedef void (*SumIntsFunc)(int* dst, const int* a, const int* b); +//! using SumIntsFunc = void (*)(int* dst, const int* a, const int* b); //! //! int main() { //! // Create a custom environment that matches the current host environment. @@ -675,7 +876,7 @@ namespace asmjit { //! // such relocations. You can use `CodeHolder::hasAddressTable()` to verify //! // whether the address table section does exist. //! code.flatten(); -//! code.resolveUnresolvedLinks(); +//! code.resolveCrossSectionFixups(); //! //! // After the code was generated it can be relocated manually to any memory //! // location, however, we need to know it's size before we perform memory @@ -690,15 +891,18 @@ namespace asmjit { //! JitAllocator allocator; //! //! // Allocate an executable virtual memory and handle a possible failure. -//! void* p = allocator.alloc(estimatedSize); -//! if (!p) +//! JitAllocator::Span span; +//! Error err = allocator.alloc(span, estimatedSize); +//! +//! if (err != kErrorOk) { // <- NOTE: This must be checked, always! //! return 0; +//! } //! //! // Now relocate the code to the address provided by the memory allocator. -//! // Please note that this DOESN'T COPY anything to `p`. This function will -//! // store the address in CodeHolder and use relocation entries to patch the -//! // existing code in all sections to respect the base address provided. -//! code.relocateToBase((uint64_t)p); +//! // Please note that this DOESN'T COPY anything to it. This function will +//! // store the address in CodeHolder and use relocation entries to patch +//! // the existing code in all sections to respect the base address provided. +//! code.relocateToBase((uint64_t)span.rx()); //! //! // This is purely optional. There are cases in which the relocation can omit //! // unneeded data, which would shrink the size of address table. If that @@ -711,12 +915,17 @@ namespace asmjit { //! // additional options that can be used to also zero pad sections' virtual //! // size, etc. //! // -//! // With some additional features, copyFlattenData() does roughly this: -//! // for (Section* section : code.sections()) -//! // memcpy((uint8_t*)p + section->offset(), -//! // section->data(), -//! // section->bufferSize()); -//! code.copyFlattenedData(p, codeSize, CopySectionFlags::kPadSectionBuffer); +//! // With some additional features, copyFlattenData() does roughly the following: +//! // +//! // allocator.write([&](JitAllocator::Span& span) { +//! // for (Section* section : code.sections()) { +//! // uint8_t* p = (uint8_t*)span.rw() + section->offset(); +//! // memcpy(p, section->data(), section->bufferSize()); +//! // } +//! // } +//! allocator.write([&](JitAllocator::Span& span) { +//! code.copyFlattenedData(span.rw(), codeSize, CopySectionFlags::kPadSectionBuffer); +//! }); //! //! // Execute the generated function. //! int inA[4] = { 4, 3, 2, 1 }; @@ -724,15 +933,16 @@ namespace asmjit { //! int out[4]; //! //! // This code uses AsmJit's ptr_as_func<> to cast between void* and SumIntsFunc. -//! ptr_as_func(p)(out, inA, inB); +//! SumIntsFunc fn = ptr_as_func(span.rx()); +//! fn(out, inA, inB); //! //! // Prints {5 8 4 9} //! printf("{%d %d %d %d}\n", out[0], out[1], out[2], out[3]); //! -//! // Release 'p' is it's no longer needed. It will be destroyed with 'vm' +//! // Release `fn` is it's no longer needed. It will be destroyed with 'vm' //! // instance anyway, but it's a good practice to release it explicitly //! // when you know that the function will not be needed anymore. -//! allocator.release(p); +//! allocator.release(fn); //! //! return 0; //! } @@ -759,11 +969,11 @@ namespace asmjit { //! } //! ``` //! -//! ### Label Offsets and Links +//! \section labels Label Offsets and Links //! -//! When a label that is not yet bound is used by the Assembler, it creates a \ref LabelLink, which is then added to -//! a \ref LabelEntry. These links are also created if a label is used in a different section than in which it was -//! bound. Let's examine some functions that can be used to check whether there are any unresolved links. +//! When a label that is not yet bound is used by the Assembler, it creates a \ref Fixup, which is then referenced +//! by \ref LabelEntry. Fixups are also created if a label is referenced in a different section than in which it was +//! bound. Let's examine some functions that can be used to check whether there are any unresolved fixups. //! //! ``` //! #include @@ -777,17 +987,17 @@ namespace asmjit { //! printf("Label %u is %s\n", label.id(), isBound ? "bound" : "not bound"); //! //! // Returns true if the code contains either referenced, but unbound -//! // labels, or cross-section label links that are not resolved yet. -//! bool hasUnresolved = code.hasUnresolvedLinks(); // Boolean answer. -//! size_t nUnresolved = code.unresolvedLinkCount(); // Count of unresolved links. +//! // labels, or cross-section fixups that are not resolved yet. +//! bool hasUnresolved = code.hasUnresolvedFixups(); // Boolean answer. +//! size_t nUnresolved = code.unresolvedFixupCount(); // Count of unresolved fixups. //! -//! printf("Number of unresolved links: %zu\n", nUnresolved); +//! printf("Number of unresolved fixups: %zu\n", nUnresolved); //! } //! ``` //! //! There is no function that would return the number of unbound labels as this is completely unimportant from //! CodeHolder's perspective. If a label is not used then it doesn't matter whether it's bound or not, only actually -//! used labels matter. After a Label is bound it's possible to query its offset offset relative to the start of the +//! used labels matter. After a Label is bound it's possible to query its offset relative to the start of the //! section where it was bound: //! //! ``` @@ -813,7 +1023,7 @@ namespace asmjit { //! } //! ``` //! -//! ### Sections +//! \section code_sections Code & Data Sections //! //! AsmJit allows to create multiple sections within the same \ref CodeHolder. A test-case //! [asmjit_test_x86_sections.cpp](https://github.com/asmjit/asmjit/blob/master/test/asmjit_test_x86_sections.cpp) @@ -854,17 +1064,16 @@ namespace asmjit { //! a.section(text); // Switches to the end of .text section. //! a.add(x86::ebx, x86::eax); // Emits in .text section. //! -//! // References a label in .text section, which was bound in .data section. -//! // This would create a LabelLink even when the L_Data is already bound, -//! // because the reference crosses sections. See below... +//! // References a label in .text section, which was bound in .data section. This would create a +//! // fixup even when the L_Data is already bound, because the reference crosses sections. See below... //! a.lea(x86::rsi, x86::ptr(L_Data)); //! } //! ``` //! -//! The last line in the example above shows that a LabelLink would be created even for bound labels that cross -//! sections. In this case a referenced label was bound in another section, which means that the link couldn't be -//! resolved at that moment. If your code uses sections, but you wish AsmJit to flatten these sections (you don't -//! plan to flatten them manually) then there is an API for that. +//! The last line in the example above shows that a \ref Fixup would be created even for bound labels that cross +//! sections. In this case a referenced label was bound in another section, which means that the reference couldn't +//! be resolved at that moment. If your code uses sections, but you wish AsmJit to flatten these sections (you don't +//! plan to flatten them manually) then there is a ready API for that. //! //! ``` //! #include @@ -889,18 +1098,18 @@ namespace asmjit { //! // guaranteed that the offset cannot be greater than `2^32 - 1`. //! printf("Data section offset %zu", size_t(data->offset())); //! -//! // The flattening doesn't resolve unresolved label links, this +//! // The flattening doesn't resolve unresolved fixups, this //! // has to be done manually as flattening can be done separately. -//! err = code.resolveUnresolvedLinks(); +//! err = code.resolveCrossSectionFixups(); //! if (err) { //! // This is the kind of error that should always be handled... -//! printf("Failed to resolve label links: %s\n", DebugUtils::errorAsString(err)); +//! printf("Failed to resolve fixups: %s\n", DebugUtils::errorAsString(err)); //! exit(1); //! } //! -//! if (code.hasUnresolvedLinks()) { +//! if (code.hasUnresolvedFixups()) { //! // This would mean either unbound label or some other issue. -//! printf("The code has %zu unbound labels\n", code.unresolvedLinkCount()); +//! printf("The code has %zu unbound labels\n", code.unresolvedFixupCount()); //! exit(1); //! } //! } @@ -916,18 +1125,25 @@ namespace asmjit { //! with assembler requires the knowledge of the following: //! //! - \ref BaseAssembler and architecture-specific assemblers: -//! - \ref x86::Assembler - Assembler specific to X86 architecture +//! - \ref x86::Assembler - Assembler implementation targeting X86 and X86_64 architectures. +//! - \ref a64::Assembler - Assembler implementation targeting AArch64 architecture. //! - \ref Operand and its variations: -//! - \ref BaseReg - Base class for a register operand, inherited by: -//! - \ref x86::Reg - Register operand specific to X86 architecture. +//! - \ref Reg - Base class for a register operand, inherited by: +//! - \ref UniGp - Universal abstraction of a general purpose register, inherited by: +//! - \ref x86::Gp - GP register operand specific to X86 and X86_64 architectures. +//! - \ref a64::Gp - GP Register operand specific to AArch64 architecture. +//! - \ref UniVec - Universal abstraction of a vector register, inherited by: +//! - \ref x86::Vec - Vector register operand specific to X86 and X86_64 architectures. +//! - \ref a64::Vec - Vector register operand specific to AArch64 architecture. //! - \ref BaseMem - Base class for a memory operand, inherited by: -//! - \ref x86::Mem - Memory operand specific to X86 architecture. +//! - \ref x86::Mem - Memory operand specific to X86 and X86_64 architectures. +//! - \ref arm::Mem - Memory operand specific to AArch64 architecture. //! - \ref Imm - Immediate (value) operand. //! - \ref Label - Label operand. //! //! \note Assembler examples use \ref x86::Assembler as abstract interfaces cannot be used to generate code. //! -//! ### Operand Basics +//! \section operand_basics Operand Basics //! //! Let's start with operands. \ref Operand is a data structure that defines a data layout of any operand. It can be //! inherited, but any class inheriting it cannot add any members to it, only the existing layout can be reused. @@ -967,7 +1183,7 @@ namespace asmjit { //! - \ref Label - used to reference a location in code or data. Labels must be created by the \ref BaseEmitter or //! by \ref CodeHolder. Each label has its unique id per \ref CodeHolder instance. //! -//! ### Operand Manipulation +//! \section operand_manipulation Operand Manipulation //! //! AsmJit allows to construct operands dynamically, to store them, and to query a complete information about them at //! run-time. Operands are small (always 16 bytes per `Operand`) and should be always copied (by value) if you intend @@ -996,7 +1212,7 @@ namespace asmjit { //! // Constructs [src + idx] memory address - referencing [rax + r10]. //! x86::Mem m = x86::ptr(src, idx); //! -//! // Examine `m`: Returns `RegType::kX86_Gpq`. +//! // Examine `m`: Returns `RegType::kGp64`. //! m.indexType(); //! // Examine `m`: Returns 10 (`r10`). //! m.indexId(); @@ -1029,7 +1245,7 @@ namespace asmjit { //! //! // Type-unsafe, but possible. //! a.emit(x86::Inst::kIdMov, dst, m); -//! // Also possible, `emit()` is typeless and can be used with raw Operand. +//! // Also possible, `emit()` is type-less and can be used with raw Operand. //! a.emit(x86::Inst::kIdMov, dst, op); //! } //! ``` @@ -1038,7 +1254,7 @@ namespace asmjit { //! BaseEmitter::newLabel(), which creates a label entry and returns a \ref Label operand with the id that refers //! to it. Such label then can be used by emitters. //! -//! ### Memory Operands +//! \section memory_operands Memory Operands //! //! Some architectures like X86 provide a complex memory addressing model that allows to encode addresses having a //! BASE register, INDEX register with a possible scale (left shift), and displacement (called offset in AsmJit). @@ -1100,10 +1316,10 @@ namespace asmjit { //! //! void testX86Mem() { //! // The same as: dword ptr [rax + rbx]. -//! x86::Mem a = x86::dword_ptr(rax, rbx); +//! x86::Mem a = x86::dword_ptr(x86::rax, x86::rbx); //! //! // The same as: qword ptr [rdx + rsi << 0 + 1]. -//! x86::Mem b = x86::qword_ptr(rdx, rsi, 0, 1); +//! x86::Mem b = x86::qword_ptr(x86::rdx, x86::rsi, 0, 1); //! } //! ``` //! @@ -1116,18 +1332,18 @@ namespace asmjit { //! //! void testX86Mem() { //! // The same as: dword ptr [rax + 12]. -//! x86::Mem mem = x86::dword_ptr(rax, 12); +//! x86::Mem mem = x86::dword_ptr(x86::rax, 12); //! //! mem.hasBase(); // true. //! mem.hasIndex(); // false. //! mem.size(); // 4. //! mem.offset(); // 12. //! -//! mem.setSize(0); // Sets the size to 0 (makes it sizeless). +//! mem.setSize(0); // Sets the size to 0 (makes it size-less). //! mem.addOffset(-1); // Adds -1 to the offset and makes it 11. //! mem.setOffset(0); // Sets the offset to 0. -//! mem.setBase(rcx); // Changes BASE to RCX. -//! mem.setIndex(rax); // Changes INDEX to RAX. +//! mem.setBase(x86::rcx); // Changes BASE to RCX. +//! mem.setIndex(x86::rax); // Changes INDEX to RAX. //! mem.hasIndex(); // true. //! } //! // ... @@ -1171,7 +1387,7 @@ namespace asmjit { //! } //! ``` //! -//! ### Assembler Examples +//! \section examples Assembler Examples //! //! - \ref x86::Assembler provides many X86/X64 examples. @@ -1191,7 +1407,7 @@ namespace asmjit { //! compatibility with the existing \ref BaseAssembler emitter so users can move from assembler to builder when needed, //! for example to implement post-processing, which is not possible with Assembler. //! -//! ### Builder Nodes +//! \section builder_nodes Builder Nodes //! //! \ref BaseBuilder doesn't generate machine code directly, it uses an intermediate representation based on nodes, //! however, it allows to serialize to \ref BaseAssembler when the code is ready to be encoded. @@ -1217,9 +1433,10 @@ namespace asmjit { //! //! - Other nodes are provided by \ref asmjit_compiler infrastructure. //! -//! ### Builder Examples +//! \section builder_examples Examples //! -//! - \ref x86::Builder provides many X86/X64 examples. +//! - \ref x86::Builder - Builder implementation targeting X86 and X86_64 architectures. +//! - \ref a64::Builder - Builder implementation targeting AArch64 architecture. //! \defgroup asmjit_compiler Compiler @@ -1243,7 +1460,7 @@ namespace asmjit { //! return value(s) are handled by assigning virtual registers to them. Similarly, function calls are handled the same //! way. //! -//! ### Compiler Nodes +//! \section compiler_nodes Compiler Nodes //! //! \ref BaseCompiler adds some nodes that are required for function generation and invocation: //! @@ -1254,11 +1471,12 @@ namespace asmjit { //! \ref BaseCompiler also makes the use of passes (\ref Pass) and automatically adds an architecture-dependent //! register allocator pass to the list of passes when attached to \ref CodeHolder. //! -//! ### Compiler Examples +//! \section compiler_examples Compiler Examples //! -//! - \ref x86::Compiler provides many X86/X64 examples. +//! - \ref x86::Compiler - Compiler implementation targeting X86 and X86_64 architectures. +//! - \ref a64::Compiler - Compiler implementation targeting AArch64 architecture. //! -//! ### Compiler Tips +//! \section compiler_tips Compiler Tips //! //! Users of AsmJit have done mistakes in the past, this section should provide some useful tips for beginners: //! @@ -1360,7 +1578,7 @@ namespace asmjit { //! - \ref FormatFlags //! - \ref FormatIndentationGroup //! -//! ### Logging +//! \section logging Logging //! //! A \ref Logger is typically attached to a \ref CodeHolder, which propagates it to all attached emitters //! automatically. The example below illustrates how to use \ref FileLogger that outputs to standard output: @@ -1418,7 +1636,7 @@ namespace asmjit { //! } //! ``` //! -//! ### Formatting +//! \section formatting Formatting //! //! AsmJit uses \ref Formatter to format inputs that are then passed to \ref Logger. Formatting is public and can be //! used by AsmJit users as well. The most important thing to know regarding formatting is that \ref Formatter always @@ -1428,7 +1646,7 @@ namespace asmjit { //! The first example illustrates how to format operands: //! //! ``` -//! #include +//! #include //! #include //! //! using namespace asmjit; @@ -1453,17 +1671,17 @@ namespace asmjit { //! // compatible with what AsmJit normally does. //! Arch arch = Arch::kX64; //! -//! log(arch, rax); // Prints 'rax'. -//! log(arch, ptr(rax, rbx, 2)); // Prints '[rax + rbx * 4]`. -//! log(arch, dword_ptr(rax, rbx, 2)); // Prints 'dword [rax + rbx * 4]`. -//! log(arch, imm(42)); // Prints '42'. +//! logOperand(arch, rax); // Prints 'rax'. +//! logOperand(arch, ptr(rax, rbx, 2)); // Prints '[rax + rbx * 4]`. +//! logOperand(arch, dword_ptr(rax, rbx, 2)); // Prints 'dword [rax + rbx * 4]`. +//! logOperand(arch, imm(42)); // Prints '42'. //! } //! ``` //! //! Next example illustrates how to format whole instructions: //! //! ``` -//! #include +//! #include //! #include //! #include //! @@ -1478,7 +1696,7 @@ namespace asmjit { //! FormatFlags formatFlags = FormatFlags::kNone; //! //! // The formatter expects operands in an array. -//! Operand_ operands { std::forward(args)... }; +//! Operand_ operands[] { std::forward(args)... }; //! //! StringTmp<128> sb; //! Formatter::formatInstruction( @@ -1500,13 +1718,13 @@ namespace asmjit { //! // Prints 'vaddpd zmm0, zmm1, [rax] {1to8}'. //! logInstruction(arch, //! BaseInst(Inst::kIdVaddpd), -//! zmm0, zmm1, ptr(rax)._1toN()); +//! zmm0, zmm1, ptr(rax)._1to8()); //! //! // BaseInst abstracts instruction id, instruction options, and extraReg. //! // Prints 'lock add [rax], rcx'. //! logInstruction(arch, //! BaseInst(Inst::kIdAdd, InstOptions::kX86_Lock), -//! x86::ptr(rax), rcx); +//! ptr(rax), rcx); //! //! // Similarly an extra register (like AVX-512 selector) can be used. //! // Prints 'vaddpd zmm0 {k2} {z}, zmm1, [rax]'. @@ -1568,7 +1786,7 @@ namespace asmjit { //! Zone memory and the ownership of memory it allocates always ends with the instance that allocated it. If //! using this approach please never jump outside the life-time of \ref CodeHolder and \ref BaseEmitter. //! -//! ### Using ErrorHandler +//! \section using_error_handler Using ErrorHandler //! //! An example of attaching \ref ErrorHandler to \ref CodeHolder. //! @@ -1624,7 +1842,7 @@ namespace asmjit { //! //! Each instruction can be then queried for the following information: //! -//! - \ref InstRWInfo - Read/write information of instruction and its oprands (includes \ref OpRWInfo). +//! - \ref InstRWInfo - Read/write information of instruction and its operands (includes \ref OpRWInfo). //! //! - \ref CpuFeatures - CPU features required to execute the instruction. //! @@ -1632,7 +1850,7 @@ namespace asmjit { //! valid. This is useful for making sure that what user tries to emit is correct and it can be also used by other //! projects that parse user input, like AsmTK project. //! -//! ### Query API +//! \section instruction_query Instruction Queries //! //! The instruction query API is provided by \ref InstAPI namespace. The following queries are possible: //! @@ -1646,7 +1864,7 @@ namespace asmjit { //! - asmjit_test_instinfo.cpp //! can be also used as a reference about accessing instruction information. //! -//! ### Validation API +//! \section instruction_validation Instruction Validation //! //! The instruction validation API is provided by \ref InstAPI namespace in the similar fashion like the Query API, //! however, validation can also be turned on at \ref BaseEmitter level. The following is possible: @@ -1685,7 +1903,7 @@ namespace asmjit { //! example. \ref JitAllocator then tracks used space of each page it maintains. Internally, \ref JitAllocator uses //! two bit arrays to track occupied regions in each allocated block of pages. //! -//! ### Hardened Environments +//! \section hardened_environments Hardened Environments //! //! In the past, allocating virtual memory with Read+Write+Execute (RWX) access permissions was easy. However, modern //! operating systems and runtime environments often use hardening, which typically prohibits mapping pages with both @@ -1733,7 +1951,7 @@ namespace asmjit { //! its data in destructor or in their reset() member function for a future reuse. For this purpose all containers in //! AsmJit are also zone allocated. //! -//! ### Zone Allocation +//! \section zone_allocation Zone Allocation //! //! - \ref Zone - Incremental zone memory allocator with minimum features. It can only allocate memory without the //! possibility to return it back to the allocator. @@ -1745,7 +1963,7 @@ namespace asmjit { //! - \ref ZoneAllocator - A wrapper of \ref Zone that provides the capability of returning memory to the allocator. //! Such memory is stored in a pool for later reuse. //! -//! ### Zone Allocated Containers +//! \section zone_containers Zone Allocated Containers //! //! - \ref ZoneString - Zone allocated string. //! - \ref ZoneHash - Zone allocated hash table. @@ -1755,7 +1973,7 @@ namespace asmjit { //! - \ref ZoneVector - Zone allocated vector. //! - \ref ZoneBitVector - Zone allocated vector of bits. //! -//! ### Using Zone Allocated Containers +//! \section using_zone_containers Using Zone Allocated Containers //! //! The most common data structure exposed by AsmJit is \ref ZoneVector. It's very similar to `std::vector`, but the //! implementation doesn't use exceptions and uses the mentioned \ref ZoneAllocator for performance reasons. You don't @@ -1768,14 +1986,11 @@ namespace asmjit { //! using namespace asmjit; //! //! void example(CodeHolder& code) { -//! // Contains all emitters attached to CodeHolder. -//! const ZoneVector& emitters = code.emitters(); -//! //! // Contains all section entries managed by CodeHolder. //! const ZoneVector& sections = code.sections(); //! //! // Contains all label entries managed by CodeHolder. -//! const ZoneVector& labelEntries = code.labelEntries(); +//! const ZoneVector& labelEntries = code.labelEntries(); //! //! // Contains all relocation entries managed by CodeHolder. //! const ZoneVector& relocEntries = code.relocEntries(); @@ -1790,16 +2005,16 @@ namespace asmjit { //! using namespace asmjit; //! //! void example(CodeHolder& code) { -//! for (LabelEntry* le : code.labelEntries()) { -//! printf("Label #%u {Bound=%s Offset=%llu}", -//! le->id(), -//! le->isBound() ? "true" : "false", -//! (unsigned long long)le->offset()); +//! for (uint32_t labelId = 0; labelId < code.labelCount(); labelId++) { +//! const LabelEntry& le = code.labelEntry(labelId); +//! if (le.isBound()) { +//! printf("Bound Label #%u at offset=%llu\n", labelId, (unsigned long long)le->offset()); +//! } //! } //! } //! ``` //! -//! ### Design Considerations +//! \section design_considerations Design Considerations //! //! Zone-allocated containers do not store the allocator within the container. This decision was made to reduce the //! footprint of such containers as AsmJit tooling, especially Compiler's register allocation, may use many instances @@ -1860,7 +2075,7 @@ namespace asmjit { //! AsmJit uses and provides utility classes and functions, that can be used with AsmJit. The functionality can be //! divided into the following topics: //! -//! ### String Functionality +//! \section string_utilities String Utilities //! //! - \ref String - AsmJit's string container, which is used internally and which doesn't use exceptions and has //! a stable layout, which is not dependent on C++ standard library. @@ -1870,11 +2085,11 @@ namespace asmjit { //! //! - \ref FixedString - Fixed string container limited up to N characters. //! -//! ### Code Generation Utilities +//! \section codegen_utilities Code Generation Utilities //! //! - \ref ConstPool - Constant pool used by \ref BaseCompiler, but also available to users that may find use of it. //! -//! ### Support Functionality Used by AsmJit +//! \section support_utilities Support Functionality Used by AsmJit //! //! - \ref Support namespace provides many other utility functions and classes that are used by AsmJit, and made //! public. @@ -1891,7 +2106,6 @@ namespace asmjit { //! \defgroup asmjit_a64 AArch64 Backend //! \brief AArch64 backend. - //! \cond INTERNAL //! \defgroup asmjit_ra RA //! \brief Register allocator internals. @@ -1903,6 +2117,7 @@ namespace asmjit { #include "core/archtraits.h" #include "core/assembler.h" #include "core/builder.h" +#include "core/codebuffer.h" #include "core/codeholder.h" #include "core/compiler.h" #include "core/constpool.h" @@ -1910,6 +2125,7 @@ namespace asmjit { #include "core/emitter.h" #include "core/environment.h" #include "core/errorhandler.h" +#include "core/fixup.h" #include "core/formatter.h" #include "core/func.h" #include "core/globals.h" diff --git a/pe-packer/asmjit/core/api-build_p.h b/pe-packer/asmjit/core/api-build_p.h index 6eca971..bb65fc8 100644 --- a/pe-packer/asmjit/core/api-build_p.h +++ b/pe-packer/asmjit/core/api-build_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_API_BUILD_P_H_INCLUDED @@ -27,6 +27,25 @@ #define NOMINMAX #endif #include +#else + // Most production code is compiled with large file support, so do the same. + #if !defined(_WIN32) && !defined(_LARGEFILE64_SOURCE) + #define _LARGEFILE64_SOURCE 1 + #endif + + // These OSes use 64-bit API by default. + #if defined(__APPLE__ ) || \ + defined(__HAIKU__ ) || \ + defined(__bsdi__ ) || \ + defined(__DragonFly__) || \ + defined(__FreeBSD__ ) || \ + defined(__NetBSD__ ) || \ + defined(__OpenBSD__ ) + #define ASMJIT_FILE64_API(NAME) NAME + #else + #define ASMJIT_FILE64_API(NAME) NAME##64 + #endif + #endif #include "./api-config.h" diff --git a/pe-packer/asmjit/core/api-config.h b/pe-packer/asmjit/core/api-config.h index 2b726ca..a97ea17 100644 --- a/pe-packer/asmjit/core/api-config.h +++ b/pe-packer/asmjit/core/api-config.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_API_CONFIG_H_INCLUDED @@ -16,7 +16,7 @@ #define ASMJIT_LIBRARY_MAKE_VERSION(major, minor, patch) ((major << 16) | (minor << 8) | (patch)) //! AsmJit library version, see \ref ASMJIT_LIBRARY_MAKE_VERSION for a version format reference. -#define ASMJIT_LIBRARY_VERSION ASMJIT_LIBRARY_MAKE_VERSION(1, 11, 0) +#define ASMJIT_LIBRARY_VERSION ASMJIT_LIBRARY_MAKE_VERSION(1, 17, 0) //! \def ASMJIT_ABI_NAMESPACE //! @@ -27,7 +27,7 @@ //! AsmJit default, which makes it possible to use multiple AsmJit libraries within a single project, totally //! controlled by users. This is useful especially in cases in which some of such library comes from third party. #if !defined(ASMJIT_ABI_NAMESPACE) - #define ASMJIT_ABI_NAMESPACE _abi_1_11 + #define ASMJIT_ABI_NAMESPACE v1_17 #endif // !ASMJIT_ABI_NAMESPACE //! \} @@ -42,9 +42,8 @@ #include #include -#include +#include #include -#include #include #include @@ -79,23 +78,20 @@ namespace asmjit { //! \note Can be defined explicitly to bypass auto-detection. #define ASMJIT_BUILD_RELEASE +//! Disables deprecated API at compile time (deprecated API won't be available). +#define ASMJIT_NO_DEPRECATED + +//! Disables the use of an inline ABI namespace within asmjit namespace (the inline namespace is used as an ABI tag). +#define ASMJIT_NO_ABI_NAMESPACE + //! Disables X86/X64 backends. #define ASMJIT_NO_X86 //! Disables AArch64 backend. #define ASMJIT_NO_AARCH64 -//! Disables non-host backends entirely (useful for JIT compilers to minimize the library size). -#define ASMJIT_NO_FOREIGN - -//! Disables deprecated API at compile time (deprecated API won't be available). -#define ASMJIT_NO_DEPRECATED - -//! Disables \ref asmjit_builder functionality completely. -#define ASMJIT_NO_BUILDER - -//! Disables \ref asmjit_compiler functionality completely. -#define ASMJIT_NO_COMPILER +//! Disables the use of `shm_open` on all targets even when it's supported. +#define ASMJIT_NO_SHM_OPEN //! Disables JIT memory management and \ref asmjit::JitRuntime. #define ASMJIT_NO_JIT @@ -112,21 +108,36 @@ namespace asmjit { //! Disables instruction introspection API. #define ASMJIT_NO_INTROSPECTION +//! Disables non-host backends entirely (useful for JIT compilers to minimize the library size). +#define ASMJIT_NO_FOREIGN + +//! Disables \ref asmjit_builder functionality completely. +#define ASMJIT_NO_BUILDER + +//! Disables \ref asmjit_compiler functionality completely. +#define ASMJIT_NO_COMPILER + // Avoid doxygen preprocessor using feature-selection definitions. -#undef ASMJIT_BUILD_EMBNED +#undef ASMJIT_BUILD_EMBED #undef ASMJIT_BUILD_STATIC #undef ASMJIT_BUILD_DEBUG #undef ASMJIT_BUILD_RELEASE -#undef ASMJIT_NO_X86 -#undef ASMJIT_NO_FOREIGN + // (keep ASMJIT_NO_DEPRECATED defined, we don't document deprecated APIs). -#undef ASMJIT_NO_BUILDER -#undef ASMJIT_NO_COMPILER +#undef ASMJIT_NO_ABI_NAMESPACE + +#undef ASMJIT_NO_X86 +#undef ASMJIT_NO_AARCH64 +#undef ASMJIT_NO_FOREIGN + #undef ASMJIT_NO_JIT #undef ASMJIT_NO_LOGGING #undef ASMJIT_NO_TEXT #undef ASMJIT_NO_VALIDATION #undef ASMJIT_NO_INTROSPECTION +#undef ASMJIT_NO_BUILDER +#undef ASMJIT_NO_COMPILER +#undef ASMJIT_NO_UJIT //! \} @@ -138,6 +149,11 @@ namespace asmjit { #define ASMJIT_NO_COMPILER #endif +// ASMJIT_NO_COMPILER implies ASMJIT_NO_UJIT. +#if defined(ASMJIT_NO_COMPILER) && !defined(ASMJIT_NO_UJIT) + #define ASMJIT_NO_UJIT +#endif + // Prevent compile-time errors caused by misconfiguration. #if defined(ASMJIT_NO_TEXT) && !defined(ASMJIT_NO_LOGGING) #pragma message("'ASMJIT_NO_TEXT' can only be defined when 'ASMJIT_NO_LOGGING' is defined.") @@ -164,6 +180,48 @@ namespace asmjit { // Target Architecture Detection // ============================= +//! \addtogroup asmjit_core +//! \{ + +//! \def ASMJIT_ARCH_X86 +//! +//! Defined to either 0, 32, or 64 depending on whether the target CPU is X86 (32) or X86_64 (64). + +//! \def ASMJIT_ARCH_ARM +//! +//! Defined to either 0, 32, or 64 depending on whether the target CPU is ARM (32) or AArch64 (64). + +//! \def ASMJIT_ARCH_MIPS +//! +//! Defined to either 0, 32, or 64 depending on whether the target CPU is MIPS (32) or MISP64 (64). + +//! \def ASMJIT_ARCH_RISCV +//! +//! Defined to either 0, 32, or 64 depending on whether the target CPU is RV32 (32) or RV64 (64). + +//! \def ASMJIT_ARCH_BITS +//! +//! Defined to either 32 or 64 depending on the target. + +//! \def ASMJIT_ARCH_LE +//! +//! Defined to 1 if the target architecture is little endian. + +//! \def ASMJIT_ARCH_BE +//! +//! Defined to 1 if the target architecture is big endian. + +//! \def ASMJIT_HAS_HOST_BACKEND +//! +//! Defined when AsmJit is built with the target architecture backend. +//! +//! For example if AsmJit is building for x86 or x86_64 architectures and `ASMJIT_NO_X86` is not defined, +//! it would define `ASMJIT_HAS_HOST_BACKEND` when `` or ``` is included. + +//! \} + +//! \cond NONE + #if defined(_M_X64) || defined(__x86_64__) #define ASMJIT_ARCH_X86 64 #elif defined(_M_IX86) || defined(__X86__) || defined(__i386__) @@ -188,10 +246,17 @@ namespace asmjit { #define ASMJIT_ARCH_MIPS 0 #endif -#define ASMJIT_ARCH_BITS (ASMJIT_ARCH_X86 | ASMJIT_ARCH_ARM | ASMJIT_ARCH_MIPS) +// NOTE `__riscv` is the correct macro in this case as specified by "RISC-V Toolchain Conventions". +#if (defined(__riscv) || defined(__riscv__)) && defined(__riscv_xlen) + #define ASMJIT_ARCH_RISCV __riscv_xlen +#else + #define ASMJIT_ARCH_RISCV 0 +#endif + +#define ASMJIT_ARCH_BITS (ASMJIT_ARCH_X86 | ASMJIT_ARCH_ARM | ASMJIT_ARCH_MIPS | ASMJIT_ARCH_RISCV) #if ASMJIT_ARCH_BITS == 0 #undef ASMJIT_ARCH_BITS - #if defined (__LP64__) || defined(_LP64) + #if defined(__LP64__) || defined(_LP64) #define ASMJIT_ARCH_BITS 64 #else #define ASMJIT_ARCH_BITS 32 @@ -213,62 +278,105 @@ namespace asmjit { #define ASMJIT_NO_X86 #endif - #if !ASMJIT_ARCH_ARM && !defined(ASMJIT_NO_AARCH64) + #if ASMJIT_ARCH_ARM != 64 && !defined(ASMJIT_NO_AARCH64) #define ASMJIT_NO_AARCH64 #endif #endif +#if ASMJIT_ARCH_X86 != 0 && !defined(ASMJIT_NO_X86) + #define ASMJIT_HAS_HOST_BACKEND +#endif + +#if ASMJIT_ARCH_ARM == 64 && !defined(ASMJIT_NO_AARCH64) + #define ASMJIT_HAS_HOST_BACKEND +#endif + +#if !defined(ASMJIT_NO_UJIT) + #if !defined(ASMJIT_NO_X86) && ASMJIT_ARCH_X86 != 0 + #define ASMJIT_UJIT_X86 + #elif !defined(ASMJIT_NO_AARCH64) && ASMJIT_ARCH_ARM == 64 + #define ASMJIT_UJIT_AARCH64 + #else + #define ASMJIT_NO_UJIT + #endif +#endif + +//! \endcond // C++ Compiler and Features Detection // =================================== -#define ASMJIT_CXX_GNU 0 -#define ASMJIT_CXX_MAKE_VER(MAJOR, MINOR) ((MAJOR) * 1000 + (MINOR)) - -// Intel Compiler [pretends to be GNU or MSC, so it must be checked first]: -// - https://software.intel.com/en-us/articles/c0x-features-supported-by-intel-c-compiler -// - https://software.intel.com/en-us/articles/c14-features-supported-by-intel-c-compiler -// - https://software.intel.com/en-us/articles/c17-features-supported-by-intel-c-compiler -#if defined(__INTEL_COMPILER) - -// MSC Compiler: -// - https://msdn.microsoft.com/en-us/library/hh567368.aspx -// -// Version List: -// - 16.00.0 == VS2010 -// - 17.00.0 == VS2012 -// - 18.00.0 == VS2013 -// - 19.00.0 == VS2015 -// - 19.10.0 == VS2017 -#elif defined(_MSC_VER) && defined(_MSC_FULL_VER) - -// Clang Compiler [Pretends to be GNU, so it must be checked before]: -// - https://clang.llvm.org/cxx_status.html -#elif defined(__clang_major__) && defined(__clang_minor__) && defined(__clang_patchlevel__) - -// GNU Compiler: -// - https://gcc.gnu.org/projects/cxx-status.html -#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) - - #undef ASMJIT_CXX_GNU - #define ASMJIT_CXX_GNU ASMJIT_CXX_MAKE_VER(__GNUC__, __GNUC_MINOR__) - -#endif - -// Compiler features detection macros. -#if defined(__clang__) && defined(__has_attribute) +#if defined(__GNUC__) && defined(__has_attribute) #define ASMJIT_CXX_HAS_ATTRIBUTE(NAME, CHECK) (__has_attribute(NAME)) #else #define ASMJIT_CXX_HAS_ATTRIBUTE(NAME, CHECK) (!(!(CHECK))) -#endif +#endif // !ASMJIT_CXX_HAS_ATTRIBUTE // API Decorators & C++ Extensions // =============================== +//! \addtogroup asmjit_core +//! \{ + //! \def ASMJIT_API //! //! A decorator that is used to decorate API that AsmJit exports when built as a shared library. +//! \def ASMJIT_VIRTAPI +//! +//! This is basically a workaround. When using MSVC and marking class as DLL export everything gets exported, which +//! is unwanted in most projects. MSVC automatically exports typeinfo and vtable if at least one symbol of the class +//! is exported. However, GCC has some strange behavior that even if one or more symbol is exported it doesn't export +//! typeinfo unless the class itself is decorated with "visibility(default)" (i.e. ASMJIT_API). + +//! \def ASMJIT_INLINE +//! +//! Decorator to force inlining of functions, uses either `__attribute__((__always_inline__))` or __forceinline, +//! depending on C++ compiler. + +//! \def ASMJIT_INLINE_NODEBUG +//! +//! Like \ref ASMJIT_INLINE, but uses additionally `__nodebug__` or `__artificial__` attribute to make the +//! debugging of some AsmJit functions easier, especially getters and one-line abstractions where usually you don't +//! want to step in. + +//! \def ASMJIT_INLINE_CONSTEXPR +//! +//! Like \ref ASMJIT_INLINE_NODEBUG, but having an additional `constexpr` attribute. + +//! \def ASMJIT_NOINLINE +//! +//! Decorator to avoid inlining of functions, uses either `__attribute__((__noinline__))` or `__declspec(noinline)` +//! depending on C++ compiler. + +//! \def ASMJIT_CDECL +//! +//! CDECL function attribute - either `__attribute__((__cdecl__))` or `__cdecl`. + +//! \def ASMJIT_STDCALL +//! +//! STDCALL function attribute - either `__attribute__((__stdcall__))` or `__stdcall`. +//! +//! \note This expands to nothing on non-x86 targets as STDCALL is X86 specific. + +//! \def ASMJIT_FASTCALL +//! +//! FASTCALL function attribute - either `__attribute__((__fastcall__))` or `__fastcall`. +//! +//! \note Expands to nothing on non-x86 targets as FASTCALL is X86 specific. + +//! \def ASMJIT_REGPARM(N) +//! +//! Expands to `__attribute__((__regparm__(N)))` when compiled by GCC or clang, nothing otherwise. + +//! \def ASMJIT_VECTORCALL +//! +//! VECTORCALL function attribute - either `__attribute__((__vectorcall__))` or `__vectorcall`. +//! +//! \note Expands to nothing on non-x86 targets as VECTORCALL is X86 specific. + +//! \} + // API (Export / Import). #if !defined(ASMJIT_STATIC) #if defined(_WIN32) && (defined(_MSC_VER) || defined(__MINGW32__)) @@ -296,13 +404,7 @@ namespace asmjit { #define ASMJIT_VARAPI extern ASMJIT_API #endif -//! \def ASMJIT_VIRTAPI -//! -//! This is basically a workaround. When using MSVC and marking class as DLL export everything gets exported, which -//! is unwanted in most projects. MSVC automatically exports typeinfo and vtable if at least one symbol of the class -//! is exported. However, GCC has some strange behavior that even if one or more symbol is exported it doesn't export -//! typeinfo unless the class itself is decorated with "visibility(default)" (i.e. ASMJIT_API). -#if !defined(_WIN32) && defined(__GNUC__) +#if defined(__GNUC__) && !defined(_WIN32) #define ASMJIT_VIRTAPI ASMJIT_API #else #define ASMJIT_VIRTAPI @@ -310,11 +412,11 @@ namespace asmjit { // Function attributes. #if !defined(ASMJIT_BUILD_DEBUG) && defined(__GNUC__) - #define ASMJIT_FORCE_INLINE inline __attribute__((__always_inline__)) + #define ASMJIT_INLINE inline __attribute__((__always_inline__)) #elif !defined(ASMJIT_BUILD_DEBUG) && defined(_MSC_VER) - #define ASMJIT_FORCE_INLINE __forceinline + #define ASMJIT_INLINE __forceinline #else - #define ASMJIT_FORCE_INLINE inline + #define ASMJIT_INLINE inline #endif @@ -326,15 +428,14 @@ namespace asmjit { #define ASMJIT_INLINE_NODEBUG inline #endif +#define ASMJIT_INLINE_CONSTEXPR constexpr ASMJIT_INLINE_NODEBUG + #if defined(__GNUC__) #define ASMJIT_NOINLINE __attribute__((__noinline__)) - #define ASMJIT_NORETURN __attribute__((__noreturn__)) #elif defined(_MSC_VER) #define ASMJIT_NOINLINE __declspec(noinline) - #define ASMJIT_NORETURN __declspec(noreturn) #else #define ASMJIT_NOINLINE - #define ASMJIT_NORETURN #endif // Calling conventions. @@ -363,13 +464,13 @@ namespace asmjit { #define ASMJIT_VECTORCALL #endif -// Type alignment (not allowed by C++11 'alignas' keyword). +// Type alignment (not allowed by C++17 'alignas' keyword). #if defined(__GNUC__) - #define ASMJIT_ALIGN_TYPE(TYPE, N) __attribute__((__aligned__(N))) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __attribute__((__aligned__(N))) __VA_ARGS__ #elif defined(_MSC_VER) - #define ASMJIT_ALIGN_TYPE(TYPE, N) __declspec(align(N)) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __declspec(align(N)) __VA_ARGS__ #else - #define ASMJIT_ALIGN_TYPE(TYPE, N) TYPE + #define ASMJIT_ALIGN_TYPE(N, ...) __VA_ARGS__ #endif //! \def ASMJIT_MAY_ALIAS @@ -381,35 +482,15 @@ namespace asmjit { #define ASMJIT_MAY_ALIAS #endif -//! \def ASMJIT_MAYBE_UNUSED -//! -//! Expands to `[[maybe_unused]]` if supported or a compiler attribute instead. -#if __cplusplus >= 201703L - #define ASMJIT_MAYBE_UNUSED [[maybe_unused]] -#elif defined(__GNUC__) - #define ASMJIT_MAYBE_UNUSED __attribute__((unused)) -#else - #define ASMJIT_MAYBE_UNUSED -#endif - -#if defined(__clang_major__) && __clang_major__ >= 4 && !defined(_DOXYGEN) - // NOTE: Clang allows to apply this attribute to function arguments, which is what we want. Once GCC decides to - // support this use, we will enable it for GCC as well. However, until that, it will be clang only, which is - // what we need for static analysis. +#if defined(__clang__) && !defined(_DOXYGEN) + // NOTE: Clang allows to apply this attribute to function arguments, which is what we want. Once GCC decides + // to support this use, we will enable it for GCC as well. However, until that, it will be clang only, which + // is what we need for static analysis. #define ASMJIT_NONNULL(FUNCTION_ARGUMENT) FUNCTION_ARGUMENT __attribute__((__nonnull__)) #else #define ASMJIT_NONNULL(FUNCTION_ARGUMENT) FUNCTION_ARGUMENT #endif -//! \def ASMJIT_NOEXCEPT_TYPE -//! -//! Defined to `noexcept` in C++17 mode or nothing otherwise. Used by function typedefs. -#if __cplusplus >= 201703L - #define ASMJIT_NOEXCEPT_TYPE noexcept -#else - #define ASMJIT_NOEXCEPT_TYPE -#endif - //! \def ASMJIT_ASSUME(...) //! //! Macro that tells the C/C++ compiler that the expression `...` evaluates to true. @@ -443,112 +524,66 @@ namespace asmjit { #define ASMJIT_UNLIKELY(...) (__VA_ARGS__) #endif -//! \def ASMJIT_FALLTHROUGH -//! -//! Portable [[fallthrough]] attribute. -#if defined(__clang__) && __cplusplus >= 201103L - #define ASMJIT_FALLTHROUGH [[clang::fallthrough]] -#elif defined(__GNUC__) && __GNUC__ >= 7 - #define ASMJIT_FALLTHROUGH __attribute__((__fallthrough__)) -#else - #define ASMJIT_FALLTHROUGH ((void)0) /* fallthrough */ -#endif - -//! \def ASMJIT_DEPRECATED -//! -//! Marks function, class, struct, enum, or anything else as deprecated. -#if defined(__GNUC__) - #define ASMJIT_DEPRECATED(MESSAGE) __attribute__((__deprecated__(MESSAGE))) - #if defined(__clang__) - #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) __attribute__((__deprecated__(MESSAGE))) - #else - #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) /* not usable if a deprecated function uses it */ - #endif -#elif defined(_MSC_VER) - #define ASMJIT_DEPRECATED(MESSAGE) __declspec(deprecated(MESSAGE)) - #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) /* not usable if a deprecated function uses it */ -#else - #define ASMJIT_DEPRECATED(MESSAGE) - #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) -#endif - // Utilities. #define ASMJIT_OFFSET_OF(STRUCT, MEMBER) ((int)(intptr_t)((const char*)&((const STRUCT*)0x100)->MEMBER) - 0x100) #define ASMJIT_ARRAY_SIZE(X) uint32_t(sizeof(X) / sizeof(X[0])) #if ASMJIT_CXX_HAS_ATTRIBUTE(no_sanitize, 0) #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF __attribute__((__no_sanitize__("undefined"))) -#elif ASMJIT_CXX_GNU >= ASMJIT_CXX_MAKE_VER(4, 9) +#elif defined(__GNUC__) #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF __attribute__((__no_sanitize_undefined__)) #else #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF #endif +// Diagnostic Macros +// ====================================== + +#if defined(_MSC_VER) && !defined(__clang__) && !defined(_DOXYGEN) + #define ASMJIT_BEGIN_DIAGNOSTIC_SCOPE \ + __pragma(warning(push)) \ + __pragma(warning(disable: 4127)) /* conditional expression is const */ \ + __pragma(warning(disable: 4201)) /* nameless struct/union */ + #define ASMJIT_END_DIAGNOSTIC_SCOPE \ + __pragma(warning(pop)) +#else + #define ASMJIT_BEGIN_DIAGNOSTIC_SCOPE + #define ASMJIT_END_DIAGNOSTIC_SCOPE +#endif + // Begin-Namespace & End-Namespace Macros // ====================================== -#if defined _DOXYGEN - #define ASMJIT_BEGIN_NAMESPACE namespace asmjit { - #define ASMJIT_END_NAMESPACE } -#elif defined(__clang__) - #define ASMJIT_BEGIN_NAMESPACE \ - namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wconstant-logical-operand\"") \ - _Pragma("clang diagnostic ignored \"-Wunnamed-type-template-args\"") - #define ASMJIT_END_NAMESPACE \ - _Pragma("clang diagnostic pop") \ - }} -#elif defined(__GNUC__) && __GNUC__ == 4 - #define ASMJIT_BEGIN_NAMESPACE \ - namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") - #define ASMJIT_END_NAMESPACE \ - _Pragma("GCC diagnostic pop") \ - }} -#elif defined(__GNUC__) && __GNUC__ >= 8 - #define ASMJIT_BEGIN_NAMESPACE \ - namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ - _Pragma("GCC diagnostic push") \ - _Pragma("GCC diagnostic ignored \"-Wclass-memaccess\"") - #define ASMJIT_END_NAMESPACE \ - _Pragma("GCC diagnostic pop") \ - }} -#elif defined(_MSC_VER) && !defined(__INTEL_COMPILER) - #define ASMJIT_BEGIN_NAMESPACE \ - namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ - __pragma(warning(push)) \ - __pragma(warning(disable: 4127)) /* conditional expression is const */ \ - __pragma(warning(disable: 4201)) /* nameless struct/union */ - #define ASMJIT_END_NAMESPACE \ - __pragma(warning(pop)) \ - }} +#if !defined(ASMJIT_NO_ABI_NAMESPACE) && !defined(_DOXYGEN) + #define ASMJIT_BEGIN_NAMESPACE \ + ASMJIT_BEGIN_DIAGNOSTIC_SCOPE \ + namespace asmjit { \ + inline namespace ASMJIT_ABI_NAMESPACE { + #define ASMJIT_END_NAMESPACE \ + }} \ + ASMJIT_END_DIAGNOSTIC_SCOPE +#else + #define ASMJIT_BEGIN_NAMESPACE \ + ASMJIT_BEGIN_DIAGNOSTIC_SCOPE \ + namespace asmjit { + #define ASMJIT_END_NAMESPACE \ + } \ + ASMJIT_END_DIAGNOSTIC_SCOPE #endif -#if !defined(ASMJIT_BEGIN_NAMESPACE) && !defined(ASMJIT_END_NAMESPACE) - #define ASMJIT_BEGIN_NAMESPACE namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { - #define ASMJIT_END_NAMESPACE }} -#endif - -#define ASMJIT_BEGIN_SUB_NAMESPACE(NAMESPACE) \ - ASMJIT_BEGIN_NAMESPACE \ - namespace NAMESPACE { - -#define ASMJIT_END_SUB_NAMESPACE \ - } \ - ASMJIT_END_NAMESPACE +#define ASMJIT_BEGIN_SUB_NAMESPACE(NAMESPACE) ASMJIT_BEGIN_NAMESPACE namespace NAMESPACE { +#define ASMJIT_END_SUB_NAMESPACE } ASMJIT_END_NAMESPACE // C++ Utilities // ============= -#define ASMJIT_NONCOPYABLE(Type) \ - Type(const Type& other) = delete; \ +#define ASMJIT_NONCOPYABLE(Type) \ + Type(const Type& other) = delete; \ Type& operator=(const Type& other) = delete; -#define ASMJIT_NONCONSTRUCTIBLE(Type) \ - Type() = delete; \ - Type(const Type& other) = delete; \ +#define ASMJIT_NONCONSTRUCTIBLE(Type) \ + Type() = delete; \ + Type(const Type& other) = delete; \ Type& operator=(const Type& other) = delete; //! \def ASMJIT_DEFINE_ENUM_FLAGS(T) @@ -557,38 +592,32 @@ namespace asmjit { #ifdef _DOXYGEN #define ASMJIT_DEFINE_ENUM_FLAGS(T) #else - #define ASMJIT_DEFINE_ENUM_FLAGS(T) \ - static ASMJIT_INLINE_NODEBUG constexpr T operator~(T a) noexcept { \ - return T(~(std::underlying_type::type)(a)); \ - } \ - \ - static ASMJIT_INLINE_NODEBUG constexpr T operator|(T a, T b) noexcept { \ - return T((std::underlying_type::type)(a) | \ - (std::underlying_type::type)(b)); \ - } \ - static ASMJIT_INLINE_NODEBUG constexpr T operator&(T a, T b) noexcept { \ - return T((std::underlying_type::type)(a) & \ - (std::underlying_type::type)(b)); \ - } \ - static ASMJIT_INLINE_NODEBUG constexpr T operator^(T a, T b) noexcept { \ - return T((std::underlying_type::type)(a) ^ \ - (std::underlying_type::type)(b)); \ - } \ - \ - static ASMJIT_INLINE_NODEBUG T& operator|=(T& a, T b) noexcept { \ - a = T((std::underlying_type::type)(a) | \ - (std::underlying_type::type)(b)); \ - return a; \ - } \ - static ASMJIT_INLINE_NODEBUG T& operator&=(T& a, T b) noexcept { \ - a = T((std::underlying_type::type)(a) & \ - (std::underlying_type::type)(b)); \ - return a; \ - } \ - static ASMJIT_INLINE_NODEBUG T& operator^=(T& a, T b) noexcept { \ - a = T((std::underlying_type::type)(a) ^ \ - (std::underlying_type::type)(b)); \ - return a; \ + #define ASMJIT_DEFINE_ENUM_FLAGS(T) \ + static ASMJIT_INLINE_CONSTEXPR T operator~(T a) noexcept { \ + return T(~std::underlying_type_t(a)); \ + } \ + \ + static ASMJIT_INLINE_CONSTEXPR T operator|(T a, T b) noexcept { \ + return T(std::underlying_type_t(a) | std::underlying_type_t(b)); \ + } \ + static ASMJIT_INLINE_CONSTEXPR T operator&(T a, T b) noexcept { \ + return T(std::underlying_type_t(a) & std::underlying_type_t(b)); \ + } \ + static ASMJIT_INLINE_CONSTEXPR T operator^(T a, T b) noexcept { \ + return T(std::underlying_type_t(a) ^ std::underlying_type_t(b)); \ + } \ + \ + static ASMJIT_INLINE_CONSTEXPR T& operator|=(T& a, T b) noexcept { \ + a = T(std::underlying_type_t(a) | std::underlying_type_t(b)); \ + return a; \ + } \ + static ASMJIT_INLINE_CONSTEXPR T& operator&=(T& a, T b) noexcept { \ + a = T(std::underlying_type_t(a) & std::underlying_type_t(b)); \ + return a; \ + } \ + static ASMJIT_INLINE_CONSTEXPR T& operator^=(T& a, T b) noexcept { \ + a = T(std::underlying_type_t(a) ^ std::underlying_type_t(b)); \ + return a; \ } #endif @@ -598,25 +627,71 @@ namespace asmjit { #if defined(_DOXYGEN) || (defined(_MSC_VER) && _MSC_VER <= 1900) #define ASMJIT_DEFINE_ENUM_COMPARE(T) #else - #define ASMJIT_DEFINE_ENUM_COMPARE(T) \ - static ASMJIT_INLINE_NODEBUG bool operator<(T a, T b) noexcept { \ - return (std::underlying_type::type)(a) < (std::underlying_type::type)(b); \ - } \ - static ASMJIT_INLINE_NODEBUG bool operator<=(T a, T b) noexcept { \ - return (std::underlying_type::type)(a) <= (std::underlying_type::type)(b); \ - } \ - static ASMJIT_INLINE_NODEBUG bool operator>(T a, T b) noexcept { \ - return (std::underlying_type::type)(a) > (std::underlying_type::type)(b); \ - } \ - static ASMJIT_INLINE_NODEBUG bool operator>=(T a, T b) noexcept { \ - return (std::underlying_type::type)(a) >= (std::underlying_type::type)(b); \ + #define ASMJIT_DEFINE_ENUM_COMPARE(T) \ + static ASMJIT_INLINE_CONSTEXPR bool operator<(T a, T b) noexcept { \ + return (std::underlying_type_t)(a) < (std::underlying_type_t)(b); \ + } \ + static ASMJIT_INLINE_CONSTEXPR bool operator<=(T a, T b) noexcept { \ + return (std::underlying_type_t)(a) <= (std::underlying_type_t)(b); \ + } \ + static ASMJIT_INLINE_CONSTEXPR bool operator>(T a, T b) noexcept { \ + return (std::underlying_type_t)(a) > (std::underlying_type_t)(b); \ + } \ + static ASMJIT_INLINE_CONSTEXPR bool operator>=(T a, T b) noexcept { \ + return (std::underlying_type_t)(a) >= (std::underlying_type_t)(b); \ } #endif -// Cleanup Api-Config Specific Macros -// ================================== - -#undef ASMJIT_CXX_GNU -#undef ASMJIT_CXX_MAKE_VER +//! Defines a strong type `C` that wraps a value of `T`. +#define ASMJIT_DEFINE_STRONG_TYPE(C, T) \ +struct C { \ + T v; \ + \ + ASMJIT_INLINE_NODEBUG C() = default; \ + ASMJIT_INLINE_CONSTEXPR explicit C(T x) noexcept : v(x) {} \ + ASMJIT_INLINE_CONSTEXPR C(const C& other) noexcept = default; \ + \ + ASMJIT_INLINE_CONSTEXPR T value() const noexcept { return v; } \ + \ + ASMJIT_INLINE_CONSTEXPR T* valuePtr() noexcept { return &v; } \ + ASMJIT_INLINE_CONSTEXPR const T* valuePtr() const noexcept { return &v; } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator=(T x) noexcept { v = x; return *this; }; \ + ASMJIT_INLINE_CONSTEXPR C& operator=(const C& x) noexcept { v = x.v; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR C operator+(T x) const noexcept { return C(v + x); } \ + ASMJIT_INLINE_CONSTEXPR C operator-(T x) const noexcept { return C(v - x); } \ + ASMJIT_INLINE_CONSTEXPR C operator*(T x) const noexcept { return C(v * x); } \ + ASMJIT_INLINE_CONSTEXPR C operator/(T x) const noexcept { return C(v / x); } \ + \ + ASMJIT_INLINE_CONSTEXPR C operator+(const C& x) const noexcept { return C(v + x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator-(const C& x) const noexcept { return C(v - x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator*(const C& x) const noexcept { return C(v * x.v); } \ + ASMJIT_INLINE_CONSTEXPR C operator/(const C& x) const noexcept { return C(v / x.v); } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator+=(T x) noexcept { v += x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator-=(T x) noexcept { v -= x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator*=(T x) noexcept { v *= x; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator/=(T x) noexcept { v /= x; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR C& operator+=(const C& x) noexcept { v += x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator-=(const C& x) noexcept { v -= x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator*=(const C& x) noexcept { v *= x.v; return *this; } \ + ASMJIT_INLINE_CONSTEXPR C& operator/=(const C& x) noexcept { v /= x.v; return *this; } \ + \ + ASMJIT_INLINE_CONSTEXPR bool operator==(T x) const noexcept { return v == x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator!=(T x) const noexcept { return v != x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator> (T x) const noexcept { return v > x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator>=(T x) const noexcept { return v >= x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator< (T x) const noexcept { return v < x; } \ + ASMJIT_INLINE_CONSTEXPR bool operator<=(T x) const noexcept { return v <= x; } \ + \ + ASMJIT_INLINE_CONSTEXPR bool operator==(const C& x) const noexcept { return v == x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator!=(const C& x) const noexcept { return v != x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator> (const C& x) const noexcept { return v > x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator>=(const C& x) const noexcept { return v >= x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator< (const C& x) const noexcept { return v < x.v; } \ + ASMJIT_INLINE_CONSTEXPR bool operator<=(const C& x) const noexcept { return v <= x.v; } \ +}; #endif // ASMJIT_CORE_API_CONFIG_H_INCLUDED diff --git a/pe-packer/asmjit/core/archcommons.h b/pe-packer/asmjit/core/archcommons.h index 7321c12..16865b9 100644 --- a/pe-packer/asmjit/core/archcommons.h +++ b/pe-packer/asmjit/core/archcommons.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED @@ -29,8 +29,8 @@ enum class CondCode : uint8_t { kNE = 0x03u, //!< Z==0 (any_sign !=) kCS = 0x04u, //!< C==1 (unsigned >=) kHS = 0x04u, //!< C==1 (unsigned >=) - kCC = 0x05u, //!< C==0 (unsigned < ) kLO = 0x05u, //!< C==0 (unsigned < ) + kCC = 0x05u, //!< C==0 (unsigned < ) kMI = 0x06u, //!< N==1 (is negative) kPL = 0x07u, //!< N==0 (is positive or zero) kVS = 0x08u, //!< V==1 (is overflow) @@ -42,21 +42,24 @@ enum class CondCode : uint8_t { kGT = 0x0Eu, //!< Z==0 & N==V (signed > ) kLE = 0x0Fu, //!< Z==1 | N!=V (signed <=) - kSign = kMI, //!< Sign. - kNotSign = kPL, //!< Not sign. - - kOverflow = kVS, //!< Signed overflow. - kNotOverflow = kVC, //!< Not signed overflow. + kZero = kEQ, //!< Zero flag (alias to equal). + kNotZero = kNE, //!< Not zero (alias to Not Equal). kEqual = kEQ, //!< Equal `a == b`. kNotEqual = kNE, //!< Not Equal `a != b`. - kZero = kEQ, //!< Zero (alias to equal). - kNotZero = kNE, //!< Not Zero (alias to Not Equal). + kCarry = kCS, //!< Carry flag. + kNotCarry = kCC, //!< Not carry. + + kSign = kMI, //!< Sign flag. + kNotSign = kPL, //!< Not sign. kNegative = kMI, //!< Negative. kPositive = kPL, //!< Positive or zero. + kOverflow = kVS, //!< Signed overflow. + kNotOverflow = kVC, //!< Not signed overflow. + kSignedLT = kLT, //!< Signed `a < b`. kSignedLE = kLE, //!< Signed `a <= b`. kSignedGT = kGT, //!< Signed `a > b`. @@ -67,49 +70,54 @@ enum class CondCode : uint8_t { kUnsignedGT = kHI, //!< Unsigned `a > b`. kUnsignedGE = kHS, //!< Unsigned `a >= b`. + kBTZero = kZero, //!< Tested bit is zero. + kBTNotZero = kNotZero, //!< Tested bit is not zero. + kAlways = kAL, //!< No condition code (always). kMaxValue = 0x0Fu //!< Maximum value of `CondCode`. }; + +//! \cond +static constexpr CondCode _reverseCondTable[] = { + CondCode::kAL, // AL <- AL + CondCode::kNA, // NA <- NA + CondCode::kEQ, // EQ <- EQ + CondCode::kNE, // NE <- NE + CondCode::kLS, // LS <- CS + CondCode::kHI, // HI <- LO + CondCode::kMI, // MI <- MI + CondCode::kPL, // PL <- PL + CondCode::kVS, // VS <- VS + CondCode::kVC, // VC <- VC + CondCode::kLO, // LO <- HI + CondCode::kCS, // CS <- LS + CondCode::kLE, // LE <- GE + CondCode::kGT, // GT <- LT + CondCode::kLT, // LT <- GT + CondCode::kGE // GE <- LE +}; +//! \endcond + +//! Reverses a condition code (reverses the corresponding operands of a comparison). +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR CondCode reverseCond(CondCode cond) noexcept { return _reverseCondTable[uint8_t(cond)]; } + //! Negates a condition code. -static ASMJIT_INLINE_NODEBUG constexpr CondCode negateCond(CondCode cond) noexcept { return CondCode(uint8_t(cond) ^ uint8_t(1)); } +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR CondCode negateCond(CondCode cond) noexcept { return CondCode(uint8_t(cond) ^ uint8_t(1)); } -//! Data type that can be encoded with the instruction (AArch32 only). -enum class DataType : uint32_t { - //! No data type specified (default for all general purpose instructions). - kNone = 0, - //! 8-bit signed integer, specified as `.s8` in assembly. - kS8 = 1, - //! 16-bit signed integer, specified as `.s16` in assembly. - kS16 = 2, - //! 32-bit signed integer, specified as `.s32` in assembly. - kS32 = 3, - //! 64-bit signed integer, specified as `.s64` in assembly. - kS64 = 4, - //! 8-bit unsigned integer, specified as `.u8` in assembly. - kU8 = 5, - //! 16-bit unsigned integer, specified as `.u16` in assembly. - kU16 = 6, - //! 32-bit unsigned integer, specified as `.u32` in assembly. - kU32 = 7, - //! 64-bit unsigned integer, specified as `.u64` in assembly. - kU64 = 8, - //! 16-bit floating point (half precision), specified as `.f16` in assembly. - kF16 = 10, - //! 32-bit floating point (single precision), specified as `.f32` in assembly. - kF32 = 11, - //! 64-bit floating point (double precision), specified as `.f64` in assembly. - kF64 = 12, - //! 8-bit polynomial. - kP8 = 13, - //! 16-bit BF16 floating point. - kBF16 = 14, - //! 64-bit polynomial. - kP64 = 15, - - //! Maximum value of `DataType`. - kMaxValue = 15 +//! Memory offset mode. +//! +//! Describes either fixed, pre-index, or post-index offset modes. +enum class OffsetMode : uint32_t { + //! Fixed offset mode (either no index at all or a regular index without a write-back). + kFixed = 0u, + //! Pre-index "[BASE, #Offset {, }]!" with write-back. + kPreIndex = 1u, + //! Post-index "[BASE], #Offset {, }" with write-back. + kPostIndex = 2u }; //! Shift operation predicate (ARM) describes either SHIFT or EXTEND operation. @@ -175,57 +183,86 @@ public: ASMJIT_INLINE_NODEBUG Shift() noexcept = default; //! Copy constructor (default) - ASMJIT_INLINE_NODEBUG constexpr Shift(const Shift& other) noexcept = default; + ASMJIT_INLINE_CONSTEXPR Shift(const Shift& other) noexcept = default; //! Constructs Shift from operation `op` and shift `value`. - ASMJIT_INLINE_NODEBUG constexpr Shift(ShiftOp op, uint32_t value) noexcept + ASMJIT_INLINE_CONSTEXPR Shift(ShiftOp op, uint32_t value) noexcept : _op(op), _value(value) {} //! Returns the shift operation. - ASMJIT_INLINE_NODEBUG constexpr ShiftOp op() const noexcept { return _op; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR ShiftOp op() const noexcept { return _op; } + //! Sets shift operation to `op`. ASMJIT_INLINE_NODEBUG void setOp(ShiftOp op) noexcept { _op = op; } - //! Returns the shift smount. - ASMJIT_INLINE_NODEBUG constexpr uint32_t value() const noexcept { return _value; } + //! Returns the shift amount. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t value() const noexcept { return _value; } + //! Sets shift amount to `value`. ASMJIT_INLINE_NODEBUG void setValue(uint32_t value) noexcept { _value = value; } }; -//! Constructs a `LSL #value` shift (logical shift left). -static ASMJIT_INLINE_NODEBUG constexpr Shift lsl(uint32_t value) noexcept { return Shift(ShiftOp::kLSL, value); } -//! Constructs a `LSR #value` shift (logical shift right). -static ASMJIT_INLINE_NODEBUG constexpr Shift lsr(uint32_t value) noexcept { return Shift(ShiftOp::kLSR, value); } -//! Constructs a `ASR #value` shift (arithmetic shift right). -static ASMJIT_INLINE_NODEBUG constexpr Shift asr(uint32_t value) noexcept { return Shift(ShiftOp::kASR, value); } -//! Constructs a `ROR #value` shift (rotate right). -static ASMJIT_INLINE_NODEBUG constexpr Shift ror(uint32_t value) noexcept { return Shift(ShiftOp::kROR, value); } -//! Constructs a `RRX` shift (rotate with carry by 1). -static ASMJIT_INLINE_NODEBUG constexpr Shift rrx() noexcept { return Shift(ShiftOp::kRRX, 0); } -//! Constructs a `MSL #value` shift (logical shift left filling ones). -static ASMJIT_INLINE_NODEBUG constexpr Shift msl(uint32_t value) noexcept { return Shift(ShiftOp::kMSL, value); } - -//! Constructs a `UXTB #value` extend and shift (unsigned byte extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift uxtb(uint32_t value) noexcept { return Shift(ShiftOp::kUXTB, value); } -//! Constructs a `UXTH #value` extend and shift (unsigned hword extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift uxth(uint32_t value) noexcept { return Shift(ShiftOp::kUXTH, value); } -//! Constructs a `UXTW #value` extend and shift (unsigned word extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift uxtw(uint32_t value) noexcept { return Shift(ShiftOp::kUXTW, value); } -//! Constructs a `UXTX #value` extend and shift (unsigned dword extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift uxtx(uint32_t value) noexcept { return Shift(ShiftOp::kUXTX, value); } - -//! Constructs a `SXTB #value` extend and shift (signed byte extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift sxtb(uint32_t value) noexcept { return Shift(ShiftOp::kSXTB, value); } -//! Constructs a `SXTH #value` extend and shift (signed hword extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift sxth(uint32_t value) noexcept { return Shift(ShiftOp::kSXTH, value); } -//! Constructs a `SXTW #value` extend and shift (signed word extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift sxtw(uint32_t value) noexcept { return Shift(ShiftOp::kSXTW, value); } -//! Constructs a `SXTX #value` extend and shift (signed dword extend). -static ASMJIT_INLINE_NODEBUG constexpr Shift sxtx(uint32_t value) noexcept { return Shift(ShiftOp::kSXTX, value); } - //! \} ASMJIT_END_SUB_NAMESPACE +ASMJIT_BEGIN_SUB_NAMESPACE(a32) + +using namespace arm; + +//! Data type that can be encoded with AArch32 instruction identifier. +//! +//! \note Data types are frequently used with AArch32 SIMD instructions. For example `VMAX` instruction can +//! use almost all datatypes in a form `VMAX.F32`, `VMAX.S16`, `VMAX.U32`, etc... Emitter automatically adds +//! the required data type at emit level. +enum class DataType : uint32_t { + //! No data type specified (default for all general purpose instructions). + kNone = 0, + //! 8-bit signed integer, specified as `.s8` in assembly. + kS8 = 1, + //! 16-bit signed integer, specified as `.s16` in assembly. + kS16 = 2, + //! 32-bit signed integer, specified as `.s32` in assembly. + kS32 = 3, + //! 64-bit signed integer, specified as `.s64` in assembly. + kS64 = 4, + //! 8-bit unsigned integer, specified as `.u8` in assembly. + kU8 = 5, + //! 16-bit unsigned integer, specified as `.u16` in assembly. + kU16 = 6, + //! 32-bit unsigned integer, specified as `.u32` in assembly. + kU32 = 7, + //! 64-bit unsigned integer, specified as `.u64` in assembly. + kU64 = 8, + //! 16-bit floating point (half precision), specified as `.f16` in assembly. + kF16 = 10, + //! 32-bit floating point (single precision), specified as `.f32` in assembly. + kF32 = 11, + //! 64-bit floating point (double precision), specified as `.f64` in assembly. + kF64 = 12, + //! 8-bit polynomial. + kP8 = 13, + //! 16-bit BF16 floating point. + kBF16 = 14, + //! 64-bit polynomial. + kP64 = 15, + + //! Maximum value of `DataType`. + kMaxValue = 15 +}; + +static ASMJIT_INLINE_NODEBUG uint32_t dataTypeSize(DataType dt) noexcept { + static constexpr uint8_t table[] = { 0, 1, 2, 4, 8, 1, 2, 4, 8, 2, 4, 8, 1, 2, 8 }; + return table[size_t(dt)]; +} + +ASMJIT_END_SUB_NAMESPACE + +ASMJIT_BEGIN_SUB_NAMESPACE(a64) +using namespace arm; +ASMJIT_END_SUB_NAMESPACE + #endif // ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED diff --git a/pe-packer/asmjit/core/archtraits.cpp b/pe-packer/asmjit/core/archtraits.cpp index fc825df..b6f56bb 100644 --- a/pe-packer/asmjit/core/archtraits.cpp +++ b/pe-packer/asmjit/core/archtraits.cpp @@ -1,10 +1,11 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #include "../core/archtraits.h" +#include "../core/environment.h" #include "../core/misc_p.h" #if !defined(ASMJIT_NO_X86) @@ -19,18 +20,21 @@ ASMJIT_BEGIN_NAMESPACE static const constexpr ArchTraits noArchTraits = { // SP/FP/LR/PC. - 0xFF, 0xFF, 0xFF, 0xFF, + 0xFFu, 0xFFu, 0xFFu, 0xFFu, // Reserved, - { 0, 0, 0 }, + { 0u, 0u, 0u }, // HW stack alignment. - 0, + 0u, // Min/Max stack offset. 0, 0, - // ISA features [Gp, Vec, Other0, Other1]. + // Supported register types. + 0u, + + // ISA features [Gp, Vec, Mask, Extra]. {{ InstHints::kNoHints, InstHints::kNoHints, @@ -38,16 +42,6 @@ static const constexpr ArchTraits noArchTraits = { InstHints::kNoHints }}, - // RegTypeToSignature. - #define V(index) OperandSignature{0} - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - - // RegTypeToTypeId. - #define V(index) TypeId::kVoid - {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, - #undef V - // TypeIdToRegType. #define V(index) RegType::kNone {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, @@ -105,55 +99,68 @@ ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegSignature(Arch arch, TypeId typeId // TODO: Remove this, should never be used like this. // Passed RegType instead of TypeId? - if (uint32_t(typeId) <= uint32_t(RegType::kMaxValue)) - typeId = archTraits.regTypeToTypeId(RegType(uint32_t(typeId))); + if (uint32_t(typeId) <= uint32_t(RegType::kMaxValue)) { + typeId = RegUtils::typeIdOf(RegType(uint32_t(typeId))); + } - if (ASMJIT_UNLIKELY(!TypeUtils::isValid(typeId))) + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(typeId))) { return DebugUtils::errored(kErrorInvalidTypeId); + } // First normalize architecture dependent types. if (TypeUtils::isAbstract(typeId)) { bool is32Bit = Environment::is32Bit(arch); - if (typeId == TypeId::kIntPtr) + if (typeId == TypeId::kIntPtr) { typeId = is32Bit ? TypeId::kInt32 : TypeId::kInt64; - else + } + else { typeId = is32Bit ? TypeId::kUInt32 : TypeId::kUInt64; + } } // Type size helps to construct all groups of registers. // TypeId is invalid if the size is zero. uint32_t size = TypeUtils::sizeOf(typeId); - if (ASMJIT_UNLIKELY(!size)) + if (ASMJIT_UNLIKELY(!size)) { return DebugUtils::errored(kErrorInvalidTypeId); + } - if (ASMJIT_UNLIKELY(typeId == TypeId::kFloat80)) + if (ASMJIT_UNLIKELY(typeId == TypeId::kFloat80)) { return DebugUtils::errored(kErrorInvalidUseOfF80); + } RegType regType = RegType::kNone; if (TypeUtils::isBetween(typeId, TypeId::_kBaseStart, TypeId::_kVec32Start)) { regType = archTraits._typeIdToRegType[uint32_t(typeId) - uint32_t(TypeId::_kBaseStart)]; if (regType == RegType::kNone) { - if (typeId == TypeId::kInt64 || typeId == TypeId::kUInt64) + if (typeId == TypeId::kInt64 || typeId == TypeId::kUInt64) { return DebugUtils::errored(kErrorInvalidUseOfGpq); - else + } + else { return DebugUtils::errored(kErrorInvalidTypeId); + } } } else { - if (size <= 8 && archTraits._regSignature[RegType::kVec64].isValid()) + if (size <= 8 && archTraits.hasRegType(RegType::kVec64)) { regType = RegType::kVec64; - else if (size <= 16 && archTraits._regSignature[RegType::kVec128].isValid()) + } + else if (size <= 16 && archTraits.hasRegType(RegType::kVec128)) { regType = RegType::kVec128; - else if (size == 32 && archTraits._regSignature[RegType::kVec256].isValid()) + } + else if (size == 32 && archTraits.hasRegType(RegType::kVec256)) { regType = RegType::kVec256; - else if (archTraits._regSignature[RegType::kVec512].isValid()) + } + else if (archTraits.hasRegType(RegType::kVec512)) { regType = RegType::kVec512; - else + } + else { return DebugUtils::errored(kErrorInvalidTypeId); + } } *typeIdOut = typeId; - *regSignatureOut = archTraits.regTypeToSignature(regType); + *regSignatureOut = RegUtils::signatureOf(regType); return kErrorOk; } diff --git a/pe-packer/asmjit/core/archtraits.h b/pe-packer/asmjit/core/archtraits.h index df87412..dacfbeb 100644 --- a/pe-packer/asmjit/core/archtraits.h +++ b/pe-packer/asmjit/core/archtraits.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ARCHTRAITS_H_INCLUDED @@ -74,6 +74,9 @@ enum class Arch : uint8_t { ASMJIT_ARCH_X86 == 32 ? kX86 : ASMJIT_ARCH_X86 == 64 ? kX64 : + ASMJIT_ARCH_RISCV == 32 ? kRISCV32 : + ASMJIT_ARCH_RISCV == 64 ? kRISCV64 : + ASMJIT_ARCH_ARM == 32 && ASMJIT_ARCH_LE ? kARM : ASMJIT_ARCH_ARM == 32 && ASMJIT_ARCH_BE ? kARM_BE : ASMJIT_ARCH_ARM == 64 && ASMJIT_ARCH_LE ? kAArch64 : @@ -107,13 +110,13 @@ enum class SubArch : uint8_t { //! Identifier used to represent names of different data types across architectures. enum class ArchTypeNameId : uint8_t { - //! Describes 'db' (X86/X86_64 convention, always 8-bit quantity). + //! Describes 'db' (X86|X86_64 convention, always 8-bit quantity). kDB = 0, - //! Describes 'dw' (X86/X86_64 convention, always 16-bit word). + //! Describes 'dw' (X86|X86_64 convention, always 16-bit word). kDW, - //! Describes 'dd' (X86/X86_64 convention, always 32-bit word). + //! Describes 'dd' (X86|X86_64 convention, always 32-bit word). kDD, - //! Describes 'dq' (X86/X86_64 convention, always 64-bit word). + //! Describes 'dq' (X86|X86_64 convention, always 64-bit word). kDQ, //! Describes 'byte' (always 8-bit quantity). kByte, @@ -171,7 +174,7 @@ struct ArchTraits { //! Link register id. uint8_t _linkRegId; //! Instruction pointer (or program counter) register id, if accessible. - uint8_t _ipRegId; + uint8_t _pcRegId; // Reserved. uint8_t _reserved[3]; @@ -183,17 +186,16 @@ struct ArchTraits { //! Maximum addressable offset on stack depending on specific instruction. uint32_t _maxStackOffset; + //! Bit-mask indexed by \ref RegType that describes, which register types are supported by the ISA. + uint32_t _supportedRegTypes; + //! Flags for each virtual register group. Support::Array _instHints; - //! Maps register type into a signature, that provides group, size and can be used to construct register operands. - Support::Array _regSignature; - //! Maps a register to type-id, see \ref TypeId. - Support::Array _regTypeToTypeId; //! Maps scalar TypeId values (from TypeId::_kIdBaseStart) to register types, see \ref TypeId. Support::Array _typeIdToRegType; - //! Word name identifiers of 8-bit, 16-bit, 32-biit, and 64-bit quantities that appear in formatted text. + //! Word name identifiers of 8-bit, 16-bit, 32-bit, and 64-bit quantities that appear in formatted text. ArchTypeNameId _typeNameIdTable[4]; //! \} @@ -201,52 +203,66 @@ struct ArchTraits { //! \name Accessors //! \{ - //! Returns stack pointer register id. + //! Returns stack pointer register id (always GP register). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t spRegId() const noexcept { return _spRegId; } - //! Returns stack frame register id. + + //! Returns stack frame register id (always GP register). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t fpRegId() const noexcept { return _fpRegId; } - //! Returns link register id, if the architecture provides it. + + //! Returns link register id, if the architecture provides it (always GP register). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t linkRegId() const noexcept { return _linkRegId; } - //! Returns instruction pointer register id, if the architecture provides it. - ASMJIT_INLINE_NODEBUG uint32_t ipRegId() const noexcept { return _ipRegId; } + + //! Returns program counter register id, if the architecture exposes it (always GP register). + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t pcRegId() const noexcept { return _pcRegId; } //! Returns a hardware stack alignment requirement. //! //! \note This is a hardware constraint. Architectures that don't constrain it would return the lowest alignment //! (1), however, some architectures may constrain the alignment, for example AArch64 requires 16-byte alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t hwStackAlignment() const noexcept { return _hwStackAlignment; } //! Tests whether the architecture provides link register, which is used across function calls. If the link //! register is not provided then a function call pushes the return address on stack (X86/X64). - ASMJIT_INLINE_NODEBUG bool hasLinkReg() const noexcept { return _linkRegId != BaseReg::kIdBad; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasLinkReg() const noexcept { return _linkRegId != Reg::kIdBad; } //! Returns minimum addressable offset on stack guaranteed for all instructions. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t minStackOffset() const noexcept { return _minStackOffset; } + //! Returns maximum addressable offset on stack depending on specific instruction. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t maxStackOffset() const noexcept { return _maxStackOffset; } //! Returns ISA flags of the given register `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstHints instFeatureHints(RegGroup group) const noexcept { return _instHints[group]; } + //! Tests whether the given register `group` has the given `flag` set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasInstHint(RegGroup group, InstHints feature) const noexcept { return Support::test(_instHints[group], feature); } + //! Tests whether the ISA provides register swap instruction for the given register `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasInstRegSwap(RegGroup group) const noexcept { return hasInstHint(group, InstHints::kRegSwap); } + //! Tests whether the ISA provides push/pop instructions for the given register `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasInstPushPop(RegGroup group) const noexcept { return hasInstHint(group, InstHints::kPushPop); } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRegType(RegType type) const noexcept { - return type <= RegType::kMaxValue && _regSignature[type].isValid(); + if (ASMJIT_UNLIKELY(type > RegType::kMaxValue)) { + type = RegType::kNone; + } + return Support::bitTest(_supportedRegTypes, uint32_t(type)); } - //! Returns an operand signature from the given register `type` of this architecture. - ASMJIT_INLINE_NODEBUG OperandSignature regTypeToSignature(RegType type) const noexcept { return _regSignature[type]; } - //! Returns a register from the given register `type` of this architecture. - ASMJIT_INLINE_NODEBUG RegGroup regTypeToGroup(RegType type) const noexcept { return _regSignature[type].regGroup(); } - //! Returns a register size the given register `type` of this architecture. - ASMJIT_INLINE_NODEBUG uint32_t regTypeToSize(RegType type) const noexcept { return _regSignature[type].size(); } - //! Returns a corresponding `TypeId` from the given register `type` of this architecture. - ASMJIT_INLINE_NODEBUG TypeId regTypeToTypeId(RegType type) const noexcept { return _regTypeToTypeId[type]; } - //! Returns a table of ISA word names that appear in formatted text. Word names are ISA dependent. //! //! The index of this table is log2 of the size: @@ -254,9 +270,11 @@ struct ArchTraits { //! - [1] 16-bits //! - [2] 32-bits //! - [3] 64-bits + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ArchTypeNameId* typeNameIdTable() const noexcept { return _typeNameIdTable; } //! Returns an ISA word name identifier of the given `index`, see \ref typeNameIdTable() for more details. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ArchTypeNameId typeNameIdByIndex(uint32_t index) const noexcept { return _typeNameIdTable[index]; } //! \} @@ -265,6 +283,7 @@ struct ArchTraits { //! \{ //! Returns a const reference to `ArchTraits` for the given architecture `arch`. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG const ArchTraits& byArch(Arch arch) noexcept; //! \} diff --git a/pe-packer/asmjit/core/assembler.cpp b/pe-packer/asmjit/core/assembler.cpp index d6c8762..a9f681d 100644 --- a/pe-packer/asmjit/core/assembler.cpp +++ b/pe-packer/asmjit/core/assembler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -26,12 +26,14 @@ BaseAssembler::~BaseAssembler() noexcept {} // ================================= Error BaseAssembler::setOffset(size_t offset) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } size_t size = Support::max(_section->bufferSize(), this->offset()); - if (ASMJIT_UNLIKELY(offset > size)) + if (ASMJIT_UNLIKELY(offset > size)) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } _bufferPtr = _bufferData + offset; return kErrorOk; @@ -40,74 +42,85 @@ Error BaseAssembler::setOffset(size_t offset) { // BaseAssembler - Section Management // ================================== -static void BaseAssembler_initSection(BaseAssembler* self, Section* section) noexcept { +static ASMJIT_INLINE Error BaseAssembler_initSection(BaseAssembler* self, Section* section) noexcept { uint8_t* p = section->_buffer._data; self->_section = section; self->_bufferData = p; self->_bufferPtr = p + section->_buffer._size; self->_bufferEnd = p + section->_buffer._capacity; + + return kErrorOk; } Error BaseAssembler::section(Section* section) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - if (!_code->isSectionValid(section->id()) || _code->_sections[section->id()] != section) + if (!_code->isSectionValid(section->sectionId()) || _code->_sections[section->sectionId()] != section) { return reportError(DebugUtils::errored(kErrorInvalidSection)); + } #ifndef ASMJIT_NO_LOGGING - if (_logger) - _logger->logf(".section %s {#%u}\n", section->name(), section->id()); + if (_logger) { + _logger->logf(".section %s {#%u}\n", section->name(), section->sectionId()); + } #endif - BaseAssembler_initSection(this, section); - return kErrorOk; + return BaseAssembler_initSection(this, section); } // BaseAssembler - Label Management // ================================ Label BaseAssembler::newLabel() { - uint32_t labelId = Globals::kInvalidId; + Label label; + if (ASMJIT_LIKELY(_code)) { - LabelEntry* le; - Error err = _code->newLabelEntry(&le); - if (ASMJIT_UNLIKELY(err)) + Error err = _code->newLabelId(&label._baseId); + if (ASMJIT_UNLIKELY(err)) { reportError(err); - else - labelId = le->id(); + } } - return Label(labelId); + + return label; } Label BaseAssembler::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { - uint32_t labelId = Globals::kInvalidId; + Label label; + if (ASMJIT_LIKELY(_code)) { - LabelEntry* le; - Error err = _code->newNamedLabelEntry(&le, name, nameSize, type, parentId); - if (ASMJIT_UNLIKELY(err)) + uint32_t labelId; + Error err = _code->newNamedLabelId(&labelId, name, nameSize, type, parentId); + if (ASMJIT_UNLIKELY(err)) { reportError(err); - else - labelId = le->id(); + } + else { + label.setId(labelId); + } } - return Label(labelId); + + return label; } Error BaseAssembler::bind(const Label& label) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - Error err = _code->bindLabel(label, _section->id(), offset()); + Error err = _code->bindLabel(label, _section->sectionId(), offset()); #ifndef ASMJIT_NO_LOGGING - if (_logger) + if (_logger) { EmitterUtils::logLabelBound(this, label); + } #endif resetInlineComment(); - if (err) + if (err) { return reportError(err); + } return kErrorOk; } @@ -116,11 +129,13 @@ Error BaseAssembler::bind(const Label& label) { // ===================== Error BaseAssembler::embed(const void* data, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - if (dataSize == 0) + if (dataSize == 0) { return kErrorOk; + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); @@ -144,11 +159,13 @@ Error BaseAssembler::embedDataArray(TypeId typeId, const void* data, size_t item uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize()); TypeId finalTypeId = TypeUtils::deabstract(typeId, deabstractDelta); - if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } - if (itemCount == 0 || repeatCount == 0) + if (itemCount == 0 || repeatCount == 0) { return kErrorOk; + } uint32_t typeSize = TypeUtils::sizeOf(finalTypeId); Support::FastUInt8 of = 0; @@ -156,15 +173,16 @@ Error BaseAssembler::embedDataArray(TypeId typeId, const void* data, size_t item size_t dataSize = Support::mulOverflow(itemCount, size_t(typeSize), &of); size_t totalSize = Support::mulOverflow(dataSize, repeatCount, &of); - if (ASMJIT_UNLIKELY(of)) + if (ASMJIT_UNLIKELY(of)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, totalSize)); - for (size_t i = 0; i < repeatCount; i++) + for (size_t i = 0; i < repeatCount; i++) { writer.emitData(data, dataSize); - + } writer.done(this); #ifndef ASMJIT_NO_LOGGING @@ -194,18 +212,21 @@ static const TypeId dataTypeIdBySize[9] = { #endif Error BaseAssembler::embedConstPool(const Label& label, const ConstPool& pool) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - if (ASMJIT_UNLIKELY(!isLabelValid(label))) + if (ASMJIT_UNLIKELY(!isLabelValid(label))) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); + } ASMJIT_PROPAGATE(align(AlignMode::kData, uint32_t(pool.alignment()))); ASMJIT_PROPAGATE(bind(label)); size_t size = pool.size(); - if (!size) + if (!size) { return kErrorOk; + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, size)); @@ -234,21 +255,24 @@ Error BaseAssembler::embedConstPool(const Label& label, const ConstPool& pool) { } Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - ASMJIT_ASSERT(_code != nullptr); - RelocEntry* re; - LabelEntry* le = _code->labelEntry(label); - - if (ASMJIT_UNLIKELY(!le)) + if (ASMJIT_UNLIKELY(isLabelValid(label))) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); + } - if (dataSize == 0) + RelocEntry* re; + LabelEntry& le = _code->labelEntry(label); + + if (dataSize == 0) { dataSize = registerSize(); + } - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(dataSize, 8u))) { return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); @@ -266,26 +290,28 @@ Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { #endif Error err = _code->newRelocEntry(&re, RelocType::kRelToAbs); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); - if (le->isBound()) { - re->_targetSectionId = le->section()->id(); - re->_payload = le->offset(); + if (le.isBound()) { + re->_targetSectionId = le.sectionId(); + re->_payload = le.offset(); } else { OffsetFormat of; of.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); - LabelLink* link = _code->newLabelLink(le, _section->id(), offset(), 0, of); - if (ASMJIT_UNLIKELY(!link)) + Fixup* fixup = _code->newFixup(le, _section->sectionId(), offset(), 0, of); + if (ASMJIT_UNLIKELY(!fixup)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } - link->relocId = re->id(); + fixup->labelOrRelocId = re->id(); } // Emit dummy DWORD/QWORD depending on the data size. @@ -296,20 +322,24 @@ Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { } Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } - LabelEntry* labelEntry = _code->labelEntry(label); - LabelEntry* baseEntry = _code->labelEntry(base); - - if (ASMJIT_UNLIKELY(!labelEntry || !baseEntry)) + if (ASMJIT_UNLIKELY(!Support::bool_and(_code->isLabelValid(label), _code->isLabelValid(base)))) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); + } - if (dataSize == 0) + LabelEntry& labelEntry = _code->labelEntry(label); + LabelEntry& baseEntry = _code->labelEntry(base); + + if (dataSize == 0) { dataSize = registerSize(); + } - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) + if (ASMJIT_UNLIKELY(!Support::isPowerOf2UpTo(dataSize, 8u))) { return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); + } CodeWriter writer(this); ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); @@ -329,27 +359,29 @@ Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size #endif // If both labels are bound within the same section it means the delta can be calculated now. - if (labelEntry->isBound() && baseEntry->isBound() && labelEntry->section() == baseEntry->section()) { - uint64_t delta = labelEntry->offset() - baseEntry->offset(); + if (labelEntry.isBound() && baseEntry.isBound() && labelEntry.sectionId() == baseEntry.sectionId()) { + uint64_t delta = labelEntry.offset() - baseEntry.offset(); writer.emitValueLE(delta, dataSize); } else { RelocEntry* re; Error err = _code->newRelocEntry(&re, RelocType::kExpression); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } Expression* exp = _code->_zone.newT(); - if (ASMJIT_UNLIKELY(!exp)) + if (ASMJIT_UNLIKELY(!exp)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } exp->reset(); exp->opType = ExpressionOpType::kSub; - exp->setValueAsLabel(0, labelEntry); - exp->setValueAsLabel(1, baseEntry); + exp->setValueAsLabelId(0, label.id()); + exp->setValueAsLabelId(1, base.id()); re->_format.resetToSimpleValue(OffsetType::kSignedOffset, dataSize); - re->_sourceSectionId = _section->id(); + re->_sourceSectionId = _section->sectionId(); re->_sourceOffset = offset(); re->_payload = (uint64_t)(uintptr_t)exp; @@ -365,8 +397,9 @@ Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size Error BaseAssembler::comment(const char* data, size_t size) { if (!hasEmitterFlag(EmitterFlags::kLogComments)) { - if (!hasEmitterFlag(EmitterFlags::kAttached)) + if (!hasEmitterFlag(EmitterFlags::kAttached)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } return kErrorOk; } @@ -386,16 +419,14 @@ Error BaseAssembler::comment(const char* data, size_t size) { // BaseAssembler - Events // ====================== -Error BaseAssembler::onAttach(CodeHolder* code) noexcept { +Error BaseAssembler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); // Attach to the end of the .text section. - BaseAssembler_initSection(this, code->_sections[0]); - - return kErrorOk; + return BaseAssembler_initSection(this, code._sections[0]); } -Error BaseAssembler::onDetach(CodeHolder* code) noexcept { +Error BaseAssembler::onDetach(CodeHolder& code) noexcept { _section = nullptr; _bufferData = nullptr; _bufferEnd = nullptr; @@ -403,4 +434,11 @@ Error BaseAssembler::onDetach(CodeHolder* code) noexcept { return Base::onDetach(code); } +Error BaseAssembler::onReinit(CodeHolder& code) noexcept { + // BaseEmitter::onReinit() never fails. + (void)Base::onReinit(code); + + return BaseAssembler_initSection(this, code._sections[0]); +} + ASMJIT_END_NAMESPACE diff --git a/pe-packer/asmjit/core/assembler.h b/pe-packer/asmjit/core/assembler.h index fefd97e..2ca7c6c 100644 --- a/pe-packer/asmjit/core/assembler.h +++ b/pe-packer/asmjit/core/assembler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ASSEMBLER_H_INCLUDED @@ -17,18 +17,18 @@ ASMJIT_BEGIN_NAMESPACE //! Base assembler. //! -//! This is a base class that provides interface used by architecture specific -//! assembler implementations. Assembler doesn't hold any data, instead it's -//! attached to \ref CodeHolder, which provides all the data that Assembler -//! needs and which can be altered by it. +//! This is a base class that provides interface used by architecture specific assembler implementations. Assembler +//! doesn't hold any data, instead it's attached to \ref CodeHolder, which provides all the data that Assembler needs +//! and which can be altered by it. //! //! Check out architecture specific assemblers for more details and examples: //! //! - \ref x86::Assembler - X86/X64 assembler implementation. +//! - \ref a64::Assembler - AArch64 assembler implementation. class ASMJIT_VIRTAPI BaseAssembler : public BaseEmitter { public: ASMJIT_NONCOPYABLE(BaseAssembler) - typedef BaseEmitter Base; + using Base = BaseEmitter; //! Current section where the assembling happens. Section* _section = nullptr; @@ -45,7 +45,7 @@ public: //! Creates a new `BaseAssembler` instance. ASMJIT_API BaseAssembler() noexcept; //! Destroys the `BaseAssembler` instance. - ASMJIT_API virtual ~BaseAssembler() noexcept; + ASMJIT_API ~BaseAssembler() noexcept override; //! \} @@ -53,24 +53,32 @@ public: //! \{ //! Returns the capacity of the current CodeBuffer. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t bufferCapacity() const noexcept { return (size_t)(_bufferEnd - _bufferData); } + //! Returns the number of remaining bytes in the current CodeBuffer. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t remainingSpace() const noexcept { return (size_t)(_bufferEnd - _bufferPtr); } //! Returns the current position in the CodeBuffer. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t offset() const noexcept { return (size_t)(_bufferPtr - _bufferData); } //! Sets the current position in the CodeBuffer to `offset`. //! - //! \note The `offset` cannot be greater than buffer size even if it's - //! within the buffer's capacity. + //! \note The `offset` cannot be greater than buffer size even if it's within the buffer's capacity. ASMJIT_API Error setOffset(size_t offset); //! Returns the start of the CodeBuffer in the current section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* bufferData() const noexcept { return _bufferData; } + //! Returns the end (first invalid byte) in the current section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* bufferEnd() const noexcept { return _bufferEnd; } + //! Returns the current pointer in the CodeBuffer in the current section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* bufferPtr() const noexcept { return _bufferPtr; } //! \} @@ -79,6 +87,7 @@ public: //! \{ //! Returns the current section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Section* currentSection() const noexcept { return _section; } ASMJIT_API Error section(Section* section) override; @@ -116,8 +125,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; diff --git a/pe-packer/asmjit/core/builder.cpp b/pe-packer/asmjit/core/builder.cpp index 9ec29b9..891ea91 100644 --- a/pe-packer/asmjit/core/builder.cpp +++ b/pe-packer/asmjit/core/builder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -35,8 +35,9 @@ public: // ======================= static void BaseBuilder_deletePasses(BaseBuilder* self) noexcept { - for (Pass* pass : self->_passes) + for (Pass* pass : self->_passes) { pass->~Pass(); + } self->_passes.reset(); } @@ -45,9 +46,8 @@ static void BaseBuilder_deletePasses(BaseBuilder* self) noexcept { BaseBuilder::BaseBuilder() noexcept : BaseEmitter(EmitterType::kBuilder), - _codeZone(32768 - Zone::kBlockOverhead), - _dataZone(16384 - Zone::kBlockOverhead), - _passZone(65536 - Zone::kBlockOverhead), + _codeZone(64u * 1024u), + _passZone(64u * 1024u), _allocator(&_codeZone) {} BaseBuilder::~BaseBuilder() noexcept { @@ -61,15 +61,15 @@ Error BaseBuilder::newInstNode(InstNode** out, InstId instId, InstOptions instOp uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); - InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); - if (ASMJIT_UNLIKELY(!node)) + void* ptr = _codeZone.alloc(InstNode::nodeSizeOfOpCapacity(opCapacity)); + if (ASMJIT_UNLIKELY(!ptr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } - *out = new(node) InstNode(this, instId, instOptions, opCount, opCapacity); + *out = new(Support::PlacementNew{ptr}) InstNode(instId, instOptions, opCount, opCapacity); return kErrorOk; } - Error BaseBuilder::newLabelNode(LabelNode** out) { *out = nullptr; @@ -88,34 +88,27 @@ Error BaseBuilder::newEmbedDataNode(EmbedDataNode** out, TypeId typeId, const vo uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize()); TypeId finalTypeId = TypeUtils::deabstract(typeId, deabstractDelta); - if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } uint32_t typeSize = TypeUtils::sizeOf(finalTypeId); Support::FastUInt8 of = 0; - size_t dataSize = Support::mulOverflow(itemCount, size_t(typeSize), &of); - if (ASMJIT_UNLIKELY(of)) + size_t nodeSize = Support::maddOverflow(itemCount, size_t(typeSize), sizeof(EmbedDataNode), &of); + if (ASMJIT_UNLIKELY(of)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); - - EmbedDataNode* node; - ASMJIT_PROPAGATE(_newNodeT(&node)); - - node->_embed._typeId = typeId; - node->_embed._typeSize = uint8_t(typeSize); - node->_itemCount = itemCount; - node->_repeatCount = repeatCount; - - uint8_t* dstData = node->_inlineData; - if (dataSize > EmbedDataNode::kInlineBufferSize) { - dstData = static_cast(_dataZone.alloc(dataSize, 8)); - if (ASMJIT_UNLIKELY(!dstData)) - return reportError(DebugUtils::errored(kErrorOutOfMemory)); - node->_externalData = dstData; } - if (data) - memcpy(dstData, data, dataSize); + EmbedDataNode* node = nullptr; + ASMJIT_PROPAGATE(_newNodeTWithSize( + &node, Support::alignUp(nodeSize, Globals::kZoneAlignment), + typeId, uint8_t(typeSize), itemCount, repeatCount + )); + + if (data) { + memcpy(node->data(), data, node->dataSize()); + } *out = node; return kErrorOk; @@ -124,7 +117,7 @@ Error BaseBuilder::newEmbedDataNode(EmbedDataNode** out, TypeId typeId, const vo Error BaseBuilder::newConstPoolNode(ConstPoolNode** out) { *out = nullptr; - ASMJIT_PROPAGATE(_newNodeT(out)); + ASMJIT_PROPAGATE(_newNodeT(out, &_codeZone)); return registerLabelNode(*out); } @@ -132,13 +125,15 @@ Error BaseBuilder::newCommentNode(CommentNode** out, const char* data, size_t si *out = nullptr; if (data) { - if (size == SIZE_MAX) + if (size == SIZE_MAX) { size = strlen(data); + } if (size > 0) { - data = static_cast(_dataZone.dup(data, size, true)); - if (ASMJIT_UNLIKELY(!data)) + data = static_cast(_codeZone.dup(data, size, true)); + if (ASMJIT_UNLIKELY(!data)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } } } @@ -168,15 +163,18 @@ BaseNode* BaseBuilder::addNode(BaseNode* node) noexcept { node->_next = next; prev->_next = node; - if (next) + if (next) { next->_prev = node; - else + } + else { _nodeList._last = node; + } } - node->addFlags(NodeFlags::kIsActive); - if (node->isSection()) + node->_addFlags(NodeFlags::kIsActive); + if (node->isSection()) { _dirtySectionLinks = true; + } _cursor = node; return node; @@ -192,15 +190,18 @@ BaseNode* BaseBuilder::addAfter(BaseNode* node, BaseNode* ref) noexcept { node->_prev = prev; node->_next = next; - node->addFlags(NodeFlags::kIsActive); - if (node->isSection()) + node->_addFlags(NodeFlags::kIsActive); + if (node->isSection()) { _dirtySectionLinks = true; + } prev->_next = node; - if (next) + if (next) { next->_prev = node; - else + } + else { _nodeList._last = node; + } return node; } @@ -217,44 +218,55 @@ BaseNode* BaseBuilder::addBefore(BaseNode* node, BaseNode* ref) noexcept { node->_prev = prev; node->_next = next; - node->addFlags(NodeFlags::kIsActive); - if (node->isSection()) + node->_addFlags(NodeFlags::kIsActive); + if (node->isSection()) { _dirtySectionLinks = true; + } next->_prev = node; - if (prev) + if (prev) { prev->_next = node; - else + } + else { _nodeList._first = node; + } return node; } BaseNode* BaseBuilder::removeNode(BaseNode* node) noexcept { - if (!node->isActive()) + if (!node->isActive()) { return node; + } BaseNode* prev = node->prev(); BaseNode* next = node->next(); - if (_nodeList._first == node) + if (_nodeList._first == node) { _nodeList._first = next; - else + } + else { prev->_next = next; + } - if (_nodeList._last == node) + if (_nodeList._last == node) { _nodeList._last = prev; - else + } + else { next->_prev = prev; + } node->_prev = nullptr; node->_next = nullptr; - node->clearFlags(NodeFlags::kIsActive); - if (node->isSection()) - _dirtySectionLinks = true; + node->_clearFlags(NodeFlags::kIsActive); - if (_cursor == node) + if (node->isSection()) { + _dirtySectionLinks = true; + } + + if (_cursor == node) { _cursor = prev; + } return node; } @@ -265,21 +277,26 @@ void BaseBuilder::removeNodes(BaseNode* first, BaseNode* last) noexcept { return; } - if (!first->isActive()) + if (!first->isActive()) { return; + } BaseNode* prev = first->prev(); BaseNode* next = last->next(); - if (_nodeList._first == first) + if (_nodeList._first == first) { _nodeList._first = next; - else + } + else { prev->_next = next; + } - if (_nodeList._last == last) + if (_nodeList._last == last) { _nodeList._last = prev; - else + } + else { next->_prev = prev; + } BaseNode* node = first; uint32_t didRemoveSection = false; @@ -290,19 +307,22 @@ void BaseBuilder::removeNodes(BaseNode* first, BaseNode* last) noexcept { node->_prev = nullptr; node->_next = nullptr; - node->clearFlags(NodeFlags::kIsActive); + node->_clearFlags(NodeFlags::kIsActive); didRemoveSection |= uint32_t(node->isSection()); - if (_cursor == node) + if (_cursor == node) { _cursor = prev; + } - if (node == last) + if (node == last) { break; + } node = next; } - if (didRemoveSection) + if (didRemoveSection) { _dirtySectionLinks = true; + } } BaseNode* BaseBuilder::setCursor(BaseNode* node) noexcept { @@ -317,28 +337,34 @@ BaseNode* BaseBuilder::setCursor(BaseNode* node) noexcept { Error BaseBuilder::sectionNodeOf(SectionNode** out, uint32_t sectionId) { *out = nullptr; - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } - if (ASMJIT_UNLIKELY(!_code->isSectionValid(sectionId))) + if (ASMJIT_UNLIKELY(!_code->isSectionValid(sectionId))) { return reportError(DebugUtils::errored(kErrorInvalidSection)); + } if (sectionId >= _sectionNodes.size()) { Error err = _sectionNodes.reserve(&_allocator, sectionId + 1); - if (ASMJIT_UNLIKELY(err != kErrorOk)) + if (ASMJIT_UNLIKELY(err != kErrorOk)) { return reportError(err); + } } SectionNode* node = nullptr; - if (sectionId < _sectionNodes.size()) + if (sectionId < _sectionNodes.size()) { node = _sectionNodes[sectionId]; + } if (!node) { ASMJIT_PROPAGATE(_newNodeT(&node, sectionId)); // We have already reserved enough space, this cannot fail now. - if (sectionId >= _sectionNodes.size()) - _sectionNodes.resize(&_allocator, sectionId + 1); + if (sectionId >= _sectionNodes.size()) { + // SAFETY: No need to check for error condition as we have already reserved enough space. + (void)_sectionNodes.resize(&_allocator, sectionId + 1); + } _sectionNodes[sectionId] = node; } @@ -349,7 +375,7 @@ Error BaseBuilder::sectionNodeOf(SectionNode** out, uint32_t sectionId) { Error BaseBuilder::section(Section* section) { SectionNode* node; - ASMJIT_PROPAGATE(sectionNodeOf(&node, section->id())); + ASMJIT_PROPAGATE(sectionNodeOf(&node, section->sectionId())); ASMJIT_ASSUME(node != nullptr); if (!node->isActive()) { @@ -358,39 +384,44 @@ Error BaseBuilder::section(Section* section) { _cursor = node; } else { - // This is a bit tricky. We cache section links to make sure that - // switching sections doesn't involve traversal in linked-list unless - // the position of the section has changed. - if (hasDirtySectionLinks()) + // This is a bit tricky. We cache section links to make sure that switching sections doesn't involve + // traversal in linked-list unless the position of the section has changed. + if (hasDirtySectionLinks()) { updateSectionLinks(); + } - if (node->_nextSection) + if (node->_nextSection) { _cursor = node->_nextSection->_prev; - else + } + else { _cursor = _nodeList.last(); + } } return kErrorOk; } void BaseBuilder::updateSectionLinks() noexcept { - if (!_dirtySectionLinks) + if (!_dirtySectionLinks) { return; + } BaseNode* node_ = _nodeList.first(); SectionNode* currentSection = nullptr; while (node_) { if (node_->isSection()) { - if (currentSection) + if (currentSection) { currentSection->_nextSection = node_->as(); + } currentSection = node_->as(); } node_ = node_->next(); } - if (currentSection) + if (currentSection) { currentSection->_nextSection = nullptr; + } _dirtySectionLinks = false; } @@ -401,15 +432,18 @@ void BaseBuilder::updateSectionLinks() noexcept { Error BaseBuilder::labelNodeOf(LabelNode** out, uint32_t labelId) { *out = nullptr; - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } uint32_t index = labelId; - if (ASMJIT_UNLIKELY(index >= _code->labelCount())) + if (ASMJIT_UNLIKELY(index >= _code->labelCount())) { return DebugUtils::errored(kErrorInvalidLabel); + } - if (index >= _labelNodes.size()) + if (index >= _labelNodes.size()) { ASMJIT_PROPAGATE(_labelNodes.resize(&_allocator, index + 1)); + } LabelNode* node = _labelNodes[index]; if (!node) { @@ -422,12 +456,12 @@ Error BaseBuilder::labelNodeOf(LabelNode** out, uint32_t labelId) { } Error BaseBuilder::registerLabelNode(LabelNode* node) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } - LabelEntry* le; - ASMJIT_PROPAGATE(_code->newLabelEntry(&le)); - uint32_t labelId = le->id(); + uint32_t labelId; + ASMJIT_PROPAGATE(_code->newLabelId(&labelId)); // We just added one label so it must be true. ASMJIT_ASSERT(_labelNodes.size() < labelId + 1); @@ -445,42 +479,58 @@ static Error BaseBuilder_newLabelInternal(BaseBuilder* self, uint32_t labelId) { uint32_t growBy = labelId - self->_labelNodes.size(); Error err = self->_labelNodes.willGrow(&self->_allocator, growBy); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return self->reportError(err); + } - LabelNode* node; + LabelNode* node = nullptr; ASMJIT_PROPAGATE(self->_newNodeT(&node, labelId)); - self->_labelNodes.resize(&self->_allocator, labelId + 1); + // SAFETY: No need to check for error condition as we have already reserved enough space. + (void)self->_labelNodes.resize(&self->_allocator, labelId + 1); self->_labelNodes[labelId] = node; node->_labelId = labelId; return kErrorOk; } Label BaseBuilder::newLabel() { - uint32_t labelId = Globals::kInvalidId; - LabelEntry* le; + Label label; - if (_code && - _code->newLabelEntry(&le) == kErrorOk && - BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { - labelId = le->id(); + if (ASMJIT_LIKELY(_code)) { + uint32_t labelId; + Error err = _code->newLabelId(&labelId); + + if (ASMJIT_UNLIKELY(err)) { + reportError(err); + } + else { + if (ASMJIT_LIKELY(BaseBuilder_newLabelInternal(this, labelId)) == kErrorOk) { + label.setId(labelId); + } + } } - return Label(labelId); + return label; } Label BaseBuilder::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { - uint32_t labelId = Globals::kInvalidId; - LabelEntry* le; + Label label; - if (_code && - _code->newNamedLabelEntry(&le, name, nameSize, type, parentId) == kErrorOk && - BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { - labelId = le->id(); + if (ASMJIT_LIKELY(_code)) { + uint32_t labelId; + Error err = _code->newNamedLabelId(&labelId, name, nameSize, type, parentId); + + if (ASMJIT_UNLIKELY(err)) { + reportError(err); + } + else { + if (ASMJIT_LIKELY(BaseBuilder_newLabelInternal(this, labelId) == kErrorOk)) { + label.setId(labelId); + } + } } - return Label(labelId); + return label; } Error BaseBuilder::bind(const Label& label) { @@ -495,15 +545,18 @@ Error BaseBuilder::bind(const Label& label) { // ==================== ASMJIT_FAVOR_SIZE Pass* BaseBuilder::passByName(const char* name) const noexcept { - for (Pass* pass : _passes) - if (strcmp(pass->name(), name) == 0) + for (Pass* pass : _passes) { + if (strcmp(pass->name(), name) == 0) { return pass; + } + } return nullptr; } ASMJIT_FAVOR_SIZE Error BaseBuilder::addPass(Pass* pass) noexcept { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } if (ASMJIT_UNLIKELY(pass == nullptr)) { // Since this is directly called by `addPassT()` we treat `null` argument @@ -512,8 +565,9 @@ ASMJIT_FAVOR_SIZE Error BaseBuilder::addPass(Pass* pass) noexcept { } else if (ASMJIT_UNLIKELY(pass->_cb)) { // Kinda weird, but okay... - if (pass->_cb == this) + if (pass->_cb == this) { return kErrorOk; + } return DebugUtils::errored(kErrorInvalidState); } @@ -522,34 +576,14 @@ ASMJIT_FAVOR_SIZE Error BaseBuilder::addPass(Pass* pass) noexcept { return kErrorOk; } -ASMJIT_FAVOR_SIZE Error BaseBuilder::deletePass(Pass* pass) noexcept { - if (ASMJIT_UNLIKELY(!_code)) +Error BaseBuilder::runPasses() { + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); - - if (ASMJIT_UNLIKELY(pass == nullptr)) - return DebugUtils::errored(kErrorInvalidArgument); - - if (pass->_cb != nullptr) { - if (pass->_cb != this) - return DebugUtils::errored(kErrorInvalidState); - - uint32_t index = _passes.indexOf(pass); - ASMJIT_ASSERT(index != Globals::kNotFound); - - pass->_cb = nullptr; - _passes.removeAt(index); } - pass->~Pass(); - return kErrorOk; -} - -Error BaseBuilder::runPasses() { - if (ASMJIT_UNLIKELY(!_code)) - return DebugUtils::errored(kErrorNotInitialized); - - if (_passes.empty()) + if (_passes.empty()) { return kErrorOk; + } ErrorHandler* prev = errorHandler(); PostponedErrorHandler postponed; @@ -560,14 +594,16 @@ Error BaseBuilder::runPasses() { for (Pass* pass : _passes) { _passZone.reset(); err = pass->run(&_passZone, _logger); - if (err) + if (err) { break; + } } _passZone.reset(); setErrorHandler(prev); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err, !postponed._message.empty() ? postponed._message.data() : nullptr); + } return kErrorOk; } @@ -580,8 +616,9 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, InstOptions options = instOptions() | forcedInstOptions(); if (Support::test(options, InstOptions::kReserved)) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } #ifndef ASMJIT_NO_VALIDATION // Strict validation. @@ -590,7 +627,7 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, EmitterUtils::opArrayFromEmitArgs(opArray, o0, o1, o2, opExt); ValidationFlags validationFlags = isCompiler() ? ValidationFlags::kEnableVirtRegs : ValidationFlags::kNone; - Error err = _funcs.validate(arch(), BaseInst(instId, options, _extraReg), opArray, opCount, validationFlags); + Error err = _funcs.validate(BaseInst(instId, options, _extraReg), opArray, opCount, validationFlags); if (ASMJIT_UNLIKELY(err)) { #ifndef ASMJIT_NO_LOGGING @@ -610,28 +647,30 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); - InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); + void* ptr = _codeZone.alloc(InstNode::nodeSizeOfOpCapacity(opCapacity)); const char* comment = inlineComment(); resetInstOptions(); resetInlineComment(); - if (ASMJIT_UNLIKELY(!node)) { + if (ASMJIT_UNLIKELY(!ptr)) { resetExtraReg(); return reportError(DebugUtils::errored(kErrorOutOfMemory)); } - node = new(node) InstNode(this, instId, options, opCount, opCapacity); + InstNode* node = new(Support::PlacementNew{ptr}) InstNode(instId, options, opCount, opCapacity); node->setExtraReg(extraReg()); node->setOp(0, o0); node->setOp(1, o1); node->setOp(2, o2); - for (uint32_t i = 3; i < opCount; i++) + for (uint32_t i = 3; i < opCount; i++) { node->setOp(i, opExt[i - 3]); + } node->resetOpRange(opCount, opCapacity); - if (comment) - node->setInlineComment(static_cast(_dataZone.dup(comment, strlen(comment), true))); + if (comment) { + node->setInlineComment(static_cast(_codeZone.dup(comment, strlen(comment), true))); + } addNode(node); resetExtraReg(); @@ -642,8 +681,9 @@ Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, // =================== Error BaseBuilder::align(AlignMode alignMode, uint32_t alignment) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } AlignNode* node; ASMJIT_PROPAGATE(newAlignNode(&node, alignMode, alignment)); @@ -657,8 +697,9 @@ Error BaseBuilder::align(AlignMode alignMode, uint32_t alignment) { // =================== Error BaseBuilder::embed(const void* data, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } EmbedDataNode* node; ASMJIT_PROPAGATE(newEmbedDataNode(&node, TypeId::kUInt8, data, dataSize)); @@ -669,8 +710,9 @@ Error BaseBuilder::embed(const void* data, size_t dataSize) { } Error BaseBuilder::embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t itemRepeat) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } EmbedDataNode* node; ASMJIT_PROPAGATE(newEmbedDataNode(&node, typeId, data, itemCount, itemRepeat)); @@ -681,11 +723,13 @@ Error BaseBuilder::embedDataArray(TypeId typeId, const void* data, size_t itemCo } Error BaseBuilder::embedConstPool(const Label& label, const ConstPool& pool) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } - if (!isLabelValid(label)) + if (!isLabelValid(label)) { return reportError(DebugUtils::errored(kErrorInvalidLabel)); + } ASMJIT_PROPAGATE(align(AlignMode::kData, uint32_t(pool.alignment()))); ASMJIT_PROPAGATE(bind(label)); @@ -701,22 +745,13 @@ Error BaseBuilder::embedConstPool(const Label& label, const ConstPool& pool) { // BaseBuilder - EmbedLabel & EmbedLabelDelta // ========================================== -// -// If dataSize is zero it means that the size is the same as target register width, however, -// if it's provided we really want to validate whether it's within the possible range. - -static inline bool BaseBuilder_checkDataSize(size_t dataSize) noexcept { - return !dataSize || (Support::isPowerOf2(dataSize) && dataSize <= 8); -} Error BaseBuilder::embedLabel(const Label& label, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) - return DebugUtils::errored(kErrorNotInitialized); + if (ASMJIT_UNLIKELY(!Support::bool_and(_code, Support::isZeroOrPowerOf2UpTo(dataSize, 8u)))) { + return reportError(DebugUtils::errored(!_code ? kErrorNotInitialized : kErrorInvalidArgument)); + } - if (!BaseBuilder_checkDataSize(dataSize)) - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - - EmbedLabelNode* node; + EmbedLabelNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), uint32_t(dataSize))); addNode(node); @@ -724,13 +759,11 @@ Error BaseBuilder::embedLabel(const Label& label, size_t dataSize) { } Error BaseBuilder::embedLabelDelta(const Label& label, const Label& base, size_t dataSize) { - if (ASMJIT_UNLIKELY(!_code)) - return DebugUtils::errored(kErrorNotInitialized); + if (ASMJIT_UNLIKELY(!Support::bool_and(_code, Support::isZeroOrPowerOf2UpTo(dataSize, 8u)))) { + return reportError(DebugUtils::errored(!_code ? kErrorNotInitialized : kErrorInvalidArgument)); + } - if (!BaseBuilder_checkDataSize(dataSize)) - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - - EmbedLabelDeltaNode* node; + EmbedLabelDeltaNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), base.id(), uint32_t(dataSize))); addNode(node); @@ -741,8 +774,9 @@ Error BaseBuilder::embedLabelDelta(const Label& label, const Label& base, size_t // ===================== Error BaseBuilder::comment(const char* data, size_t size) { - if (ASMJIT_UNLIKELY(!_code)) + if (ASMJIT_UNLIKELY(!_code)) { return DebugUtils::errored(kErrorNotInitialized); + } CommentNode* node; ASMJIT_PROPAGATE(newCommentNode(&node, data, size)); @@ -820,14 +854,16 @@ Error BaseBuilder::serializeTo(BaseEmitter* dst) { } else if (node_->isSection()) { SectionNode* node = node_->as(); - err = dst->section(_code->sectionById(node->id())); + err = dst->section(_code->sectionById(node->sectionId())); } else if (node_->isComment()) { CommentNode* node = node_->as(); err = dst->comment(node->inlineComment()); } - if (err) break; + if (err) { + break; + } node_ = node_->next(); } while (node_); @@ -837,45 +873,58 @@ Error BaseBuilder::serializeTo(BaseEmitter* dst) { // BaseBuilder - Events // ==================== -Error BaseBuilder::onAttach(CodeHolder* code) noexcept { - ASMJIT_PROPAGATE(Base::onAttach(code)); +static ASMJIT_INLINE void BaseBuilder_clearAll(BaseBuilder* self) noexcept { + self->_sectionNodes.reset(); + self->_labelNodes.reset(); + self->_allocator.reset(&self->_codeZone); + self->_codeZone.reset(); + self->_passZone.reset(); + + self->_cursor = nullptr; + self->_nodeList.reset(); +} + +static ASMJIT_INLINE Error BaseBuilder_initSection(BaseBuilder* self) noexcept { SectionNode* initialSection; - Error err = sectionNodeOf(&initialSection, 0); - if (!err) - err = _passes.willGrow(&_allocator, 8); - - if (ASMJIT_UNLIKELY(err)) { - onDetach(code); - return err; - } + ASMJIT_PROPAGATE(self->sectionNodeOf(&initialSection, 0)); + ASMJIT_PROPAGATE(self->_passes.willGrow(&self->_allocator, 4)); ASMJIT_ASSUME(initialSection != nullptr); - _cursor = initialSection; - _nodeList.reset(initialSection, initialSection); - initialSection->setFlags(NodeFlags::kIsActive); + self->_cursor = initialSection; + self->_nodeList.reset(initialSection, initialSection); + initialSection->_setFlags(NodeFlags::kIsActive); return kErrorOk; } -Error BaseBuilder::onDetach(CodeHolder* code) noexcept { +Error BaseBuilder::onAttach(CodeHolder& code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + Error err = BaseBuilder_initSection(this); + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + } + return err; +} + +Error BaseBuilder::onDetach(CodeHolder& code) noexcept { BaseBuilder_deletePasses(this); - _sectionNodes.reset(); - _labelNodes.reset(); - - _allocator.reset(&_codeZone); - _codeZone.reset(); - _dataZone.reset(); - _passZone.reset(); - - _nodeFlags = NodeFlags::kNone; - _cursor = nullptr; - _nodeList.reset(); + BaseBuilder_clearAll(this); return Base::onDetach(code); } +Error BaseBuilder::onReinit(CodeHolder& code) noexcept { + // BaseEmitter::onReinit() never fails. + (void)Base::onReinit(code); + + BaseBuilder_deletePasses(this); + BaseBuilder_clearAll(this); + return BaseBuilder_initSection(this); +} + // Pass - Construction & Destruction // ================================= diff --git a/pe-packer/asmjit/core/builder.h b/pe-packer/asmjit/core/builder.h index 2ff035b..35e661b 100644 --- a/pe-packer/asmjit/core/builder.h +++ b/pe-packer/asmjit/core/builder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_BUILDER_H_INCLUDED @@ -48,7 +48,7 @@ enum class NodeType : uint8_t { // [BaseBuilder] - //! Node is \ref InstNode or \ref InstExNode. + //! Node is \ref InstNode. kInst = 1, //! Node is \ref SectionNode. kSection = 2, @@ -163,9 +163,13 @@ public: //! \name Accessors //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return _first == nullptr; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* first() const noexcept { return _first; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* last() const noexcept { return _last; } //! \} @@ -181,18 +185,17 @@ public: //! Check out architecture specific builders for more details and examples: //! //! - \ref x86::Builder - X86/X64 builder implementation. +//! - \ref a64::Builder - AArch64 builder implementation. class ASMJIT_VIRTAPI BaseBuilder : public BaseEmitter { public: ASMJIT_NONCOPYABLE(BaseBuilder) - typedef BaseEmitter Base; + using Base = BaseEmitter; //! \name Members //! \{ //! Base zone used to allocate nodes and passes. Zone _codeZone; - //! Data zone used to allocate data and names. - Zone _dataZone; //! Pass zone, passed to `Pass::run()`. Zone _passZone; //! Allocator that uses `_codeZone`. @@ -210,8 +213,6 @@ public: //! First and last nodes. NodeList _nodeList; - //! Flags assigned to each new node. - NodeFlags _nodeFlags = NodeFlags::kNone; //! The sections links are dirty (used internally). bool _dirtySectionLinks = false; @@ -223,20 +224,38 @@ public: //! Creates a new `BaseBuilder` instance. ASMJIT_API BaseBuilder() noexcept; //! Destroys the `BaseBuilder` instance. - ASMJIT_API virtual ~BaseBuilder() noexcept; + ASMJIT_API ~BaseBuilder() noexcept override; //! \} //! \name Node Management //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG NodeList nodeList() const noexcept { return _nodeList; } //! Returns the first node. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* firstNode() const noexcept { return _nodeList.first(); } + //! Returns the last node. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* lastNode() const noexcept { return _nodeList.last(); } + //! Allocates data required for a node. + template + ASMJIT_INLINE Error _newNodeTWithSize(T** ASMJIT_NONNULL(out), size_t size, Args&&... args) { + ASMJIT_ASSERT(Support::isAligned(size, Globals::kZoneAlignment)); + + void* ptr =_codeZone.alloc(size); + if (ASMJIT_UNLIKELY(!ptr)) { + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + + *out = new(Support::PlacementNew{ptr}) T(std::forward(args)...); + return kErrorOk; + } + //! Allocates and instantiates a new node of type `T` and returns its instance. If the allocation fails `nullptr` //! is returned. //! @@ -245,10 +264,14 @@ public: //! \remarks The pointer returned (if non-null) is owned by the Builder or Compiler. When the Builder/Compiler //! is destroyed it destroys all nodes it created so no manual memory management is required. template - inline Error _newNodeT(T** ASMJIT_NONNULL(out), Args&&... args) { - *out = _allocator.newT(this, std::forward(args)...); - if (ASMJIT_UNLIKELY(!*out)) + ASMJIT_INLINE Error _newNodeT(T** ASMJIT_NONNULL(out), Args&&... args) { + void* ptr = _codeZone.alloc(Zone::alignedSizeOf()); + + if (ASMJIT_UNLIKELY(!ptr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + + *out = new(Support::PlacementNew{ptr}) T(std::forward(args)...); return kErrorOk; } @@ -281,6 +304,7 @@ public: //! When the Builder/Compiler is created it automatically creates a '.text' \ref SectionNode, which will be the //! initial one. When instructions are added they are always added after the cursor and the cursor is changed //! to be that newly added node. Use `setCursor()` to change where new nodes are inserted. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* cursor() const noexcept { return _cursor; } //! Sets the current node to `node` and return the previous one. @@ -301,11 +325,13 @@ public: //! //! \note If a section of some id is not associated with the Builder/Compiler it would be null, so always check //! for nulls if you iterate over the vector. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& sectionNodes() const noexcept { return _sectionNodes; } //! Tests whether the `SectionNode` of the given `sectionId` was registered. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRegisteredSectionNode(uint32_t sectionId) const noexcept { return sectionId < _sectionNodes.size() && _sectionNodes[sectionId] != nullptr; } @@ -320,6 +346,7 @@ public: //! Returns whether the section links of active section nodes are dirty. You can update these links by calling //! `updateSectionLinks()` in such case. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasDirtySectionLinks() const noexcept { return _dirtySectionLinks; } //! Updates links of all active section nodes. @@ -334,14 +361,17 @@ public: //! //! \note If a label of some id is not associated with the Builder/Compiler it would be null, so always check for //! nulls if you iterate over the vector. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& labelNodes() const noexcept { return _labelNodes; } //! Tests whether the `LabelNode` of the given `labelId` was registered. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRegisteredLabelNode(uint32_t labelId) const noexcept { return labelId < _labelNodes.size() && _labelNodes[labelId] != nullptr; } //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRegisteredLabelNode(const Label& label) const noexcept { return hasRegisteredLabelNode(label.id()); } @@ -361,10 +391,15 @@ public: //! //! This function is used internally to register a newly created `LabelNode` with this instance of Builder/Compiler. //! Use \ref labelNodeOf() functions to get back \ref LabelNode from a label or its identifier. + [[nodiscard]] ASMJIT_API Error registerLabelNode(LabelNode* ASMJIT_NONNULL(node)); + [[nodiscard]] ASMJIT_API Label newLabel() override; + + [[nodiscard]] ASMJIT_API Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId) override; + ASMJIT_API Error bind(const Label& label) override; //! \} @@ -373,6 +408,7 @@ public: //! \{ //! Returns a vector of `Pass` instances that will be executed by `runPasses()`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& passes() const noexcept { return _passes; } //! Allocates and instantiates a new pass of type `T` and returns its instance. If the allocation fails `nullptr` is @@ -383,10 +419,12 @@ public: //! \remarks The pointer returned (if non-null) is owned by the Builder or Compiler. When the Builder/Compiler is //! destroyed it destroys all passes it created so no manual memory management is required. template + [[nodiscard]] inline T* newPassT() noexcept { return _codeZone.newT(); } //! \overload template + [[nodiscard]] inline T* newPassT(Args&&... args) noexcept { return _codeZone.newT(std::forward(args)...); } template @@ -398,11 +436,11 @@ public: //! Returns `Pass` by name. //! //! If the pass having the given `name` doesn't exist `nullptr` is returned. + [[nodiscard]] ASMJIT_API Pass* passByName(const char* name) const noexcept; + //! Adds `pass` to the list of passes. ASMJIT_API Error addPass(Pass* pass) noexcept; - //! Removes `pass` from the list of passes and delete it. - ASMJIT_API Error deletePass(Pass* pass) noexcept; //! Runs all passes in order. ASMJIT_API Error runPasses(); @@ -456,8 +494,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; @@ -482,16 +521,17 @@ public: //! Next node. BaseNode* _next; }; - //! Links (an alternative view to previous and next nodes). + //! Links (an alternative view of previous and next nodes). BaseNode* _links[2]; }; + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Data shared between all types of nodes. struct AnyData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Not used by BaseNode. uint8_t _reserved0; //! Not used by BaseNode. @@ -500,10 +540,6 @@ public: //! Data used by \ref AlignNode. struct AlignData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Align mode. AlignMode _alignMode; //! Not used by AlignNode. @@ -512,10 +548,6 @@ public: //! Data used by \ref InstNode. struct InstData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Instruction operands count (used). uint8_t _opCount; //! Instruction operands capacity (allocated). @@ -524,10 +556,6 @@ public: //! Data used by \ref EmbedDataNode. struct EmbedData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Type id. TypeId _typeId; //! Size of `_typeId`. @@ -536,17 +564,13 @@ public: //! Data used by \ref SentinelNode. struct SentinelData { - //! Node type. - NodeType _nodeType; - //! Node flags. - NodeFlags _nodeFlags; //! Sentinel type. SentinelType _sentinelType; //! Not used by BaseNode. uint8_t _reserved1; }; - //! Data that can have different meaning dependning on \ref NodeType. + //! Data that can have different meaning depending on \ref NodeType. union { //! Data useful by any node type. AnyData _any; @@ -583,11 +607,11 @@ public: //! \{ //! Creates a new `BaseNode` - always use `BaseBuilder` to allocate nodes. - ASMJIT_INLINE_NODEBUG BaseNode(BaseBuilder* cb, NodeType nodeType, NodeFlags nodeFlags = NodeFlags::kNone) noexcept { + ASMJIT_INLINE_NODEBUG explicit BaseNode(NodeType nodeType, NodeFlags nodeFlags = NodeFlags::kNone) noexcept { _prev = nullptr; _next = nullptr; - _any._nodeType = nodeType; - _any._nodeFlags = nodeFlags | cb->_nodeFlags; + _nodeType = nodeType; + _nodeFlags = nodeFlags; _any._reserved0 = 0; _any._reserved1 = 0; _position = 0; @@ -603,85 +627,135 @@ public: //! Casts this node to `T*`. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG T* as() noexcept { return static_cast(this); } + //! Casts this node to `const T*`. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG const T* as() const noexcept { return static_cast(this); } - //! Returns previous node or `nullptr` if this node is either first or not - //! part of Builder/Compiler node-list. + //! Returns previous node or `nullptr` if this node is either first or not part of Builder/Compiler node-list. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* prev() const noexcept { return _prev; } - //! Returns next node or `nullptr` if this node is either last or not part - //! of Builder/Compiler node-list. + + //! Returns next node or `nullptr` if this node is either last or not part of Builder/Compiler node-list. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseNode* next() const noexcept { return _next; } - //! Returns the type of the node, see `NodeType`. - ASMJIT_INLINE_NODEBUG NodeType type() const noexcept { return _any._nodeType; } + //! Returns the type of the node, see \ref NodeType. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG NodeType type() const noexcept { return _nodeType; } //! Sets the type of the node, see `NodeType` (internal). //! //! \remarks You should never set a type of a node to anything else than the initial value. This function is only //! provided for users that use custom nodes and need to change the type either during construction or later. - ASMJIT_INLINE_NODEBUG void setType(NodeType type) noexcept { _any._nodeType = type; } + ASMJIT_INLINE_NODEBUG void _setType(NodeType type) noexcept { _nodeType = type; } //! Tests whether this node is either `InstNode` or extends it. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInst() const noexcept { return hasFlag(NodeFlags::kActsAsInst); } + //! Tests whether this node is `SectionNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isSection() const noexcept { return type() == NodeType::kSection; } + //! Tests whether this node is either `LabelNode` or extends it. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabel() const noexcept { return hasFlag(NodeFlags::kActsAsLabel); } + //! Tests whether this node is `AlignNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAlign() const noexcept { return type() == NodeType::kAlign; } + //! Tests whether this node is `EmbedDataNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isEmbedData() const noexcept { return type() == NodeType::kEmbedData; } + //! Tests whether this node is `EmbedLabelNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isEmbedLabel() const noexcept { return type() == NodeType::kEmbedLabel; } + //! Tests whether this node is `EmbedLabelDeltaNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isEmbedLabelDelta() const noexcept { return type() == NodeType::kEmbedLabelDelta; } + //! Tests whether this node is `ConstPoolNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isConstPool() const noexcept { return type() == NodeType::kConstPool; } + //! Tests whether this node is `CommentNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isComment() const noexcept { return type() == NodeType::kComment; } + //! Tests whether this node is `SentinelNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isSentinel() const noexcept { return type() == NodeType::kSentinel; } //! Tests whether this node is `FuncNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFunc() const noexcept { return type() == NodeType::kFunc; } + //! Tests whether this node is `FuncRetNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFuncRet() const noexcept { return type() == NodeType::kFuncRet; } + //! Tests whether this node is `InvokeNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInvoke() const noexcept { return type() == NodeType::kInvoke; } //! Returns the node flags. - ASMJIT_INLINE_NODEBUG NodeFlags flags() const noexcept { return _any._nodeFlags; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG NodeFlags flags() const noexcept { return _nodeFlags; } + //! Tests whether the node has the given `flag` set. - ASMJIT_INLINE_NODEBUG bool hasFlag(NodeFlags flag) const noexcept { return Support::test(_any._nodeFlags, flag); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasFlag(NodeFlags flag) const noexcept { return Support::test(_nodeFlags, flag); } + //! Replaces node flags with `flags`. - ASMJIT_INLINE_NODEBUG void setFlags(NodeFlags flags) noexcept { _any._nodeFlags = flags; } + ASMJIT_INLINE_NODEBUG void _setFlags(NodeFlags flags) noexcept { _nodeFlags = flags; } + //! Adds the given `flags` to node flags. - ASMJIT_INLINE_NODEBUG void addFlags(NodeFlags flags) noexcept { _any._nodeFlags |= flags; } + ASMJIT_INLINE_NODEBUG void _addFlags(NodeFlags flags) noexcept { _nodeFlags |= flags; } + //! Clears the given `flags` from node flags. - ASMJIT_INLINE_NODEBUG void clearFlags(NodeFlags flags) noexcept { _any._nodeFlags &= ~flags; } + ASMJIT_INLINE_NODEBUG void _clearFlags(NodeFlags flags) noexcept { _nodeFlags &= ~flags; } //! Tests whether the node is code that can be executed. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isCode() const noexcept { return hasFlag(NodeFlags::kIsCode); } + //! Tests whether the node is data that cannot be executed. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isData() const noexcept { return hasFlag(NodeFlags::kIsData); } + //! Tests whether the node is informative only (is never encoded like comment, etc...). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInformative() const noexcept { return hasFlag(NodeFlags::kIsInformative); } + //! Tests whether the node is removable if it's in an unreachable code block. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isRemovable() const noexcept { return hasFlag(NodeFlags::kIsRemovable); } + //! Tests whether the node has no effect when executed (label, .align, nop, ...). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasNoEffect() const noexcept { return hasFlag(NodeFlags::kHasNoEffect); } + //! Tests whether the node is part of the code. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isActive() const noexcept { return hasFlag(NodeFlags::kIsActive); } //! Tests whether the node has a position assigned. //! //! \remarks Returns `true` if node position is non-zero. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasPosition() const noexcept { return _position != 0; } + //! Returns node position. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t position() const noexcept { return _position; } + //! Sets node position. //! //! Node position is a 32-bit unsigned integer that is used by Compiler to track where the node is relatively to @@ -694,13 +768,19 @@ public: //! Returns user data casted to `T*`. //! - //! User data is decicated to be used only by AsmJit users and not touched by the library. The data has a pointer - //! size so you can either store a pointer or `intptr_t` value through `setUserDataAsIntPtr()`. + //! User data is dedicated to be used only by AsmJit users and not touched by the library. The data is of a pointer + //! size so you can either store a pointer or `int64_t` value through `setUserDataAsPtr()`, `setUserDataAsInt64()` + //! and `setUserDataAsUInt64()`. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG T* userDataAsPtr() const noexcept { return static_cast(_userDataPtr); } + //! Returns user data casted to `int64_t`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG int64_t userDataAsInt64() const noexcept { return int64_t(_userDataU64); } + //! Returns user data casted to `uint64_t`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t userDataAsUInt64() const noexcept { return _userDataU64; } //! Sets user data to `data`. @@ -715,10 +795,14 @@ public: ASMJIT_INLINE_NODEBUG void resetUserData() noexcept { _userDataU64 = 0; } //! Tests whether the node has an associated pass data. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasPassData() const noexcept { return _passData != nullptr; } + //! Returns the node pass data - data used during processing & transformations. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG T* passData() const noexcept { return (T*)_passData; } + //! Sets the node pass data to `data`. template ASMJIT_INLINE_NODEBUG void setPassData(T* data) noexcept { _passData = (void*)data; } @@ -726,9 +810,13 @@ public: ASMJIT_INLINE_NODEBUG void resetPassData() noexcept { _passData = nullptr; } //! Tests whether the node has an inline comment/annotation. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasInlineComment() const noexcept { return _inlineComment != nullptr; } + //! Returns an inline comment/annotation string. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* inlineComment() const noexcept { return _inlineComment; } + //! Sets an inline comment/annotation string to `s`. ASMJIT_INLINE_NODEBUG void setInlineComment(const char* s) noexcept { _inlineComment = s; } //! Resets an inline comment/annotation string to nullptr. @@ -747,22 +835,48 @@ public: //! \name Constants //! \{ - enum : uint32_t { - //! Count of embedded operands per `InstNode` that are always allocated as a part of the instruction. Minimum - //! embedded operands is 4, but in 32-bit more pointers are smaller and we can embed 5. The rest (up to 6 operands) - //! is always stored in `InstExNode`. - kBaseOpCapacity = uint32_t((128 - sizeof(BaseNode) - sizeof(BaseInst)) / sizeof(Operand_)) - }; + //! The number of embedded operands for a default \ref InstNode instance that are always allocated as a part of + //! the instruction itself. Minimum embedded operands is 4, but in 32-bit more pointers are smaller and we can + //! embed 5. The rest (up to 6 operands) is considered extended. + //! + //! The number of operands InstNode holds is decided when \ref InstNode is created. + static inline constexpr uint32_t kBaseOpCapacity = uint32_t((128u - sizeof(BaseNode) - sizeof(BaseInst)) / sizeof(Operand_)); + + //! Count of maximum number of operands \ref InstNode can hold. + static inline constexpr uint32_t kFullOpCapacity = Globals::kMaxOpCount; //! \} + //! \cond INTERNAL + //! \name Operand Capacity Utilities + //! \{ + + //! Returns the capacity required for the given operands count `opCount`. + //! + //! There are only two capacities used - \ref kBaseOpCapacity and \ref kFullOpCapacity, so this function + //! is used to decide between these two. The general rule is that instructions that can be represented with + //! \ref kBaseOpCapacity would use this value, and all others would take \ref kFullOpCapacity. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR uint32_t capacityOfOpCount(uint32_t opCount) noexcept { + return opCount <= kBaseOpCapacity ? kBaseOpCapacity : kFullOpCapacity; + } + + //! Calculates the size of \ref InstNode required to hold at most `opCapacity` operands. + //! + //! This function is used internally to allocate \ref InstNode. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR size_t nodeSizeOfOpCapacity(uint32_t opCapacity) noexcept { + return Support::alignUp(sizeof(InstNode) + opCapacity * sizeof(Operand), Globals::kZoneAlignment); + } + + //! \} + //! \endcond + //! \name Members //! \{ //! Base instruction data. BaseInst _baseInst; - //! First 4 or 5 operands (indexed from 0). - Operand_ _opArray[kBaseOpCapacity]; //! \} @@ -770,8 +884,8 @@ public: //! \{ //! Creates a new `InstNode` instance. - ASMJIT_INLINE_NODEBUG InstNode(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCount, uint32_t opCapacity = kBaseOpCapacity) noexcept - : BaseNode(cb, NodeType::kInst, NodeFlags::kIsCode | NodeFlags::kIsRemovable | NodeFlags::kActsAsInst), + ASMJIT_INLINE_NODEBUG InstNode(InstId instId, InstOptions options, uint32_t opCount, uint32_t opCapacity = kBaseOpCapacity) noexcept + : BaseNode(NodeType::kInst, NodeFlags::kIsCode | NodeFlags::kIsRemovable | NodeFlags::kActsAsInst), _baseInst(instId, options) { _inst._opCapacity = uint8_t(opCapacity); _inst._opCount = uint8_t(opCount); @@ -790,7 +904,10 @@ public: //! \name Instruction Object //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseInst& baseInst() noexcept { return _baseInst; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const BaseInst& baseInst() const noexcept { return _baseInst; } //! \} @@ -799,8 +916,11 @@ public: //! \{ //! Returns the instruction id, see `BaseInst::Id`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstId id() const noexcept { return _baseInst.id(); } + //! Returns the instruction real id, see `BaseInst::Id`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstId realId() const noexcept { return _baseInst.realId(); } //! Sets the instruction id to `id`, see `BaseInst::Id`. @@ -811,11 +931,21 @@ public: //! \name Instruction Options //! \{ + //! Returns instruction options, see \ref InstOptions for more details. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstOptions options() const noexcept { return _baseInst.options(); } + + //! Tests whether instruction has the given \option` set/enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOption(InstOptions option) const noexcept { return _baseInst.hasOption(option); } + + //! Sets instruction `options` to the provided value, resetting all others. ASMJIT_INLINE_NODEBUG void setOptions(InstOptions options) noexcept { _baseInst.setOptions(options); } + //! Adds instruction `options` to the instruction. ASMJIT_INLINE_NODEBUG void addOptions(InstOptions options) noexcept { _baseInst.addOptions(options); } + //! Clears instruction `options` of the instruction (disables the given options). ASMJIT_INLINE_NODEBUG void clearOptions(InstOptions options) noexcept { _baseInst.clearOptions(options); } + //! Resets instruction options to none - disabling all instruction options. ASMJIT_INLINE_NODEBUG void resetOptions() noexcept { _baseInst.resetOptions(); } //! \} @@ -824,13 +954,19 @@ public: //! \{ //! Tests whether the node has an extra register operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasExtraReg() const noexcept { return _baseInst.hasExtraReg(); } + //! Returns extra register operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegOnly& extraReg() noexcept { return _baseInst.extraReg(); } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _baseInst.extraReg(); } + //! Sets extra register operand to `reg`. - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _baseInst.setExtraReg(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _baseInst.setExtraReg(reg); } //! Sets extra register operand to `reg`. ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _baseInst.setExtraReg(reg); } //! Resets extra register operand. @@ -842,46 +978,63 @@ public: //! \{ //! Returns operand count. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t opCount() const noexcept { return _inst._opCount; } + //! Returns operand capacity. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t opCapacity() const noexcept { return _inst._opCapacity; } //! Sets operand count. ASMJIT_INLINE_NODEBUG void setOpCount(uint32_t opCount) noexcept { _inst._opCount = uint8_t(opCount); } //! Returns operands array. - ASMJIT_INLINE_NODEBUG Operand* operands() noexcept { return (Operand*)_opArray; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG Operand* operands() noexcept { return Support::offsetPtr(this, sizeof(InstNode)); } + //! Returns operands array (const). - ASMJIT_INLINE_NODEBUG const Operand* operands() const noexcept { return (const Operand*)_opArray; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const Operand* operands() const noexcept { return Support::offsetPtr(this, sizeof(InstNode)); } //! Returns operand at the given `index`. + [[nodiscard]] inline Operand& op(uint32_t index) noexcept { ASMJIT_ASSERT(index < opCapacity()); - return _opArray[index].as(); + + Operand* ops = operands(); + return ops[index].as(); } //! Returns operand at the given `index` (const). + [[nodiscard]] inline const Operand& op(uint32_t index) const noexcept { ASMJIT_ASSERT(index < opCapacity()); - return _opArray[index].as(); + + const Operand* ops = operands(); + return ops[index].as(); } //! Sets operand at the given `index` to `op`. inline void setOp(uint32_t index, const Operand_& op) noexcept { ASMJIT_ASSERT(index < opCapacity()); - _opArray[index].copyFrom(op); + + Operand* ops = operands(); + ops[index].copyFrom(op); } //! Resets operand at the given `index` to none. inline void resetOp(uint32_t index) noexcept { ASMJIT_ASSERT(index < opCapacity()); - _opArray[index].reset(); + + Operand* ops = operands(); + ops[index].reset(); } //! Resets operands at `[start, end)` range. inline void resetOpRange(uint32_t start, uint32_t end) noexcept { + Operand* ops = operands(); for (uint32_t i = start; i < end; i++) - _opArray[i].reset(); + ops[i].reset(); } //! \} @@ -889,103 +1042,129 @@ public: //! \name Utilities //! \{ + //! Tests whether the given operand type `opType` is used by the instruction. + [[nodiscard]] inline bool hasOpType(OperandType opType) const noexcept { - for (uint32_t i = 0, count = opCount(); i < count; i++) - if (_opArray[i].opType() == opType) + const Operand* ops = operands(); + for (uint32_t i = 0, count = opCount(); i < count; i++) { + if (ops[i].opType() == opType) { return true; + } + } return false; } + //! Tests whether the instruction uses at least one register operand. + [[nodiscard]] inline bool hasRegOp() const noexcept { return hasOpType(OperandType::kReg); } + + //! Tests whether the instruction uses at least one memory operand. + [[nodiscard]] inline bool hasMemOp() const noexcept { return hasOpType(OperandType::kMem); } + + //! Tests whether the instruction uses at least one immediate operand. + [[nodiscard]] inline bool hasImmOp() const noexcept { return hasOpType(OperandType::kImm); } + + //! Tests whether the instruction uses at least one label operand. + [[nodiscard]] inline bool hasLabelOp() const noexcept { return hasOpType(OperandType::kLabel); } + //! Returns the index of the given operand type `opType`. + //! + //! \note If the operand type wa found, the value returned represents its index in \ref operands() + //! array, otherwise \ref Globals::kNotFound is returned to signalize that the operand was not found. + [[nodiscard]] inline uint32_t indexOfOpType(OperandType opType) const noexcept { uint32_t i = 0; uint32_t count = opCount(); + const Operand* ops = operands(); while (i < count) { - if (_opArray[i].opType() == opType) - break; + if (ops[i].opType() == opType) + return i; i++; } - return i; + return Globals::kNotFound; } + //! A shortcut that calls `indexOfOpType(OperandType::kMem)`. + [[nodiscard]] inline uint32_t indexOfMemOp() const noexcept { return indexOfOpType(OperandType::kMem); } + + //! A shortcut that calls `indexOfOpType(OperandType::kImm)`. + [[nodiscard]] inline uint32_t indexOfImmOp() const noexcept { return indexOfOpType(OperandType::kImm); } + + //! A shortcut that calls `indexOfOpType(OperandType::kLabel)`. + [[nodiscard]] inline uint32_t indexOfLabelOp() const noexcept { return indexOfOpType(OperandType::kLabel); } //! \} + //! \cond INTERNAL //! \name Rewriting //! \{ - //! \cond INTERNAL + //! Returns uint32_t[] view that represents BaseInst::RegOnly and instruction operands. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t* _getRewriteArray() noexcept { return &_baseInst._extraReg._id; } + + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const uint32_t* _getRewriteArray() const noexcept { return &_baseInst._extraReg._id; } - inline uint32_t getRewriteIndex(const uint32_t* id) const noexcept { + //! Maximum value of rewrite id - 6 operands each having 4 slots is 24, one RegOnly having 2 slots => 26. + static inline constexpr uint32_t kMaxRewriteId = 26 - 1; + + //! Returns a rewrite index of the given pointer to `id`. + //! + //! This function returns a value that can be then passed to `\ref _rewriteIdAtIndex() function. It can address + //! any id from any operand that is used by the instruction in addition to \ref BaseInst::regOnly field, which + //! can also be used by the register allocator. + [[nodiscard]] + inline uint32_t _getRewriteIndex(const uint32_t* id) const noexcept { const uint32_t* array = _getRewriteArray(); ASMJIT_ASSERT(array <= id); size_t index = (size_t)(id - array); - ASMJIT_ASSERT(index < 32); + ASMJIT_ASSERT(index <= kMaxRewriteId); return uint32_t(index); } - inline void rewriteIdAtIndex(uint32_t index, uint32_t id) noexcept { + //! Rewrites the given `index` to the provided identifier `id`. + //! + //! \note This is an internal function that is used by a \ref BaseCompiler implementation to rewrite virtual + //! registers to physical registers. The rewriter in this case sees all operands as array of uint32 values + //! and the given `index` describes a position in this array. For example a single \ref Operand would be + //! decomposed to 4 uint32_t values, where the first at index 0 would be operand signature, next would be + //! base id, etc... This is a comfortable way of patching operands without having to check for their types. + inline void _rewriteIdAtIndex(uint32_t index, uint32_t id) noexcept { + ASMJIT_ASSERT(index <= kMaxRewriteId); + uint32_t* array = _getRewriteArray(); array[index] = id; } - //! \endcond //! \} - - //! \name Static Functions - //! \{ - - //! \cond INTERNAL - static ASMJIT_INLINE_NODEBUG uint32_t capacityOfOpCount(uint32_t opCount) noexcept { - return opCount <= kBaseOpCapacity ? kBaseOpCapacity : Globals::kMaxOpCount; - } - - static ASMJIT_INLINE_NODEBUG size_t nodeSizeOfOpCapacity(uint32_t opCapacity) noexcept { - size_t base = sizeof(InstNode) - kBaseOpCapacity * sizeof(Operand); - return base + opCapacity * sizeof(Operand); - } //! \endcond - - //! \} }; -//! Instruction node with maximum number of operands. +//! Instruction node with embedded operands following \ref InstNode layout. //! -//! This node is created automatically by Builder/Compiler in case that the required number of operands exceeds -//! the default capacity of `InstNode`. -class InstExNode : public InstNode { +//! \note This is used to make tools such as static analysis and compilers happy about the layout. There were two +//! instruction nodes in the past, having the second extend the operand array of the first, but that has caused +//! undefined behavior and made recent tools unhappy about that. +template +class InstNodeWithOperands : public InstNode { public: - ASMJIT_NONCOPYABLE(InstExNode) + Operand_ _operands[kN]; - //! \name Members - //! \{ - - //! Continued `_opArray[]` to hold up to `kMaxOpCount` operands. - Operand_ _opArrayEx[Globals::kMaxOpCount - kBaseOpCapacity]; - - //! \} - - //! \name Construction & Destruction - //! \{ - - //! Creates a new `InstExNode` instance. - ASMJIT_INLINE_NODEBUG InstExNode(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCapacity = Globals::kMaxOpCount) noexcept - : InstNode(cb, instId, options, opCapacity) {} - - //! \} + //! Creates a new `InstNodeWithOperands` instance. + ASMJIT_INLINE_NODEBUG InstNodeWithOperands(InstId instId, InstOptions options, uint32_t opCount) noexcept + : InstNode(instId, options, opCount, kN) {} }; //! Section node. @@ -997,7 +1176,7 @@ public: //! \{ //! Section id. - uint32_t _id; + uint32_t _sectionId; //! Next section node that follows this section. //! @@ -1012,9 +1191,9 @@ public: //! \{ //! Creates a new `SectionNode` instance. - ASMJIT_INLINE_NODEBUG SectionNode(BaseBuilder* cb, uint32_t secionId = 0) noexcept - : BaseNode(cb, NodeType::kSection, NodeFlags::kHasNoEffect), - _id(secionId), + ASMJIT_INLINE_NODEBUG explicit SectionNode(uint32_t sectionId = 0) noexcept + : BaseNode(NodeType::kSection, NodeFlags::kHasNoEffect), + _sectionId(sectionId), _nextSection(nullptr) {} //! \} @@ -1023,7 +1202,7 @@ public: //! \{ //! Returns the section id. - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _sectionId; } //! \} }; @@ -1045,8 +1224,8 @@ public: //! \{ //! Creates a new `LabelNode` instance. - ASMJIT_INLINE_NODEBUG LabelNode(BaseBuilder* cb, uint32_t labelId = 0) noexcept - : BaseNode(cb, NodeType::kLabel, NodeFlags::kHasNoEffect | NodeFlags::kActsAsLabel), + ASMJIT_INLINE_NODEBUG explicit LabelNode(uint32_t labelId = Globals::kInvalidId) noexcept + : BaseNode(NodeType::kLabel, NodeFlags::kHasNoEffect | NodeFlags::kActsAsLabel), _labelId(labelId) {} //! \} @@ -1055,8 +1234,11 @@ public: //! \{ //! Returns \ref Label representation of the \ref LabelNode. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label label() const noexcept { return Label(_labelId); } + //! Returns the id of the label. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _labelId; } //! \} @@ -1081,8 +1263,8 @@ public: //! \{ //! Creates a new `AlignNode` instance. - ASMJIT_INLINE_NODEBUG AlignNode(BaseBuilder* cb, AlignMode alignMode, uint32_t alignment) noexcept - : BaseNode(cb, NodeType::kAlign, NodeFlags::kIsCode | NodeFlags::kHasNoEffect) { + ASMJIT_INLINE_NODEBUG AlignNode(AlignMode alignMode, uint32_t alignment) noexcept + : BaseNode(NodeType::kAlign, NodeFlags::kIsCode | NodeFlags::kHasNoEffect) { _alignData._alignMode = alignMode; _alignment = alignment; @@ -1094,12 +1276,16 @@ public: //! \{ //! Returns align mode. + [[nodiscard]] ASMJIT_INLINE_NODEBUG AlignMode alignMode() const noexcept { return _alignData._alignMode; } + //! Sets align mode to `alignMode`. ASMJIT_INLINE_NODEBUG void setAlignMode(AlignMode alignMode) noexcept { _alignData._alignMode = alignMode; } //! Returns align offset in bytes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t alignment() const noexcept { return _alignment; } + //! Sets align offset in bytes to `offset`. ASMJIT_INLINE_NODEBUG void setAlignment(uint32_t alignment) noexcept { _alignment = alignment; } @@ -1114,36 +1300,24 @@ class EmbedDataNode : public BaseNode { public: ASMJIT_NONCOPYABLE(EmbedDataNode) - //! \cond INTERNAL - enum : uint32_t { - kInlineBufferSize = 128 - (sizeof(BaseNode) + sizeof(size_t) * 2) - }; - //! \endcond - //! \name Members //! \{ size_t _itemCount; size_t _repeatCount; - union { - uint8_t* _externalData; - uint8_t _inlineData[kInlineBufferSize]; - }; - //! \} //! \name Construction & Destruction //! \{ //! Creates a new `EmbedDataNode` instance. - ASMJIT_INLINE_NODEBUG EmbedDataNode(BaseBuilder* cb) noexcept - : BaseNode(cb, NodeType::kEmbedData, NodeFlags::kIsData), - _itemCount(0), - _repeatCount(0) { - _embed._typeId = TypeId::kUInt8; - _embed._typeSize = uint8_t(1); - memset(_inlineData, 0, kInlineBufferSize); + ASMJIT_INLINE_NODEBUG EmbedDataNode(TypeId typeId, uint8_t typeSize, size_t itemCount, size_t repeatCount) noexcept + : BaseNode(NodeType::kEmbedData, NodeFlags::kIsData), + _itemCount(itemCount), + _repeatCount(repeatCount) { + _embed._typeId = typeId; + _embed._typeSize = typeSize; } //! \} @@ -1152,31 +1326,38 @@ public: //! \{ //! Returns data type as \ref TypeId. + [[nodiscard]] ASMJIT_INLINE_NODEBUG TypeId typeId() const noexcept { return _embed._typeId; } + //! Returns the size of a single data element. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t typeSize() const noexcept { return _embed._typeSize; } - //! Returns a pointer to the data casted to `uint8_t`. - ASMJIT_INLINE_NODEBUG uint8_t* data() const noexcept { - return dataSize() <= kInlineBufferSize ? const_cast(_inlineData) : _externalData; - } + //! Returns a pointer to the data casted to `T*` - `uint8_t*` by default. + template + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint8_t* data() noexcept { return Support::offsetPtr(this, sizeof(EmbedDataNode)); } - //! Returns a pointer to the data casted to `T`. - template - ASMJIT_INLINE_NODEBUG T* dataAs() const noexcept { return reinterpret_cast(data()); } + //! Returns a pointer to the data casted to `T*` - `const uint8_t*` by default (const). + template + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const uint8_t* data() const noexcept { return Support::offsetPtr(this, sizeof(EmbedDataNode)); } //! Returns the number of (typed) items in the array. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t itemCount() const noexcept { return _itemCount; } //! Returns how many times the data is repeated (default 1). //! //! Repeated data is useful when defining constants for SIMD, for example. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t repeatCount() const noexcept { return _repeatCount; } //! Returns the size of the data, not considering the number of times it repeats. //! //! \note The returned value is the same as `typeSize() * itemCount()`. - ASMJIT_INLINE_NODEBUG size_t dataSize() const noexcept { return typeSize() * _itemCount; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG size_t dataSize() const noexcept { return size_t(typeSize()) * _itemCount; } //! \} }; @@ -1198,8 +1379,8 @@ public: //! \{ //! Creates a new `EmbedLabelNode` instance. - ASMJIT_INLINE_NODEBUG EmbedLabelNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t dataSize = 0) noexcept - : BaseNode(cb, NodeType::kEmbedLabel, NodeFlags::kIsData), + ASMJIT_INLINE_NODEBUG EmbedLabelNode(uint32_t labelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(NodeType::kEmbedLabel, NodeFlags::kIsData), _labelId(labelId), _dataSize(dataSize) {} @@ -1209,17 +1390,23 @@ public: //! \{ //! Returns the label to embed as \ref Label operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label label() const noexcept { return Label(_labelId); } - //! Returns the id of the label. - ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _labelId; } //! Sets the label id from `label` operand. ASMJIT_INLINE_NODEBUG void setLabel(const Label& label) noexcept { setLabelId(label.id()); } + + //! Returns the id of the label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _labelId; } + //! Sets the label id (use with caution, improper use can break a lot of things). ASMJIT_INLINE_NODEBUG void setLabelId(uint32_t labelId) noexcept { _labelId = labelId; } //! Returns the data size. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t dataSize() const noexcept { return _dataSize; } + //! Sets the data size. ASMJIT_INLINE_NODEBUG void setDataSize(uint32_t dataSize) noexcept { _dataSize = dataSize; } @@ -1244,8 +1431,8 @@ public: //! \{ //! Creates a new `EmbedLabelDeltaNode` instance. - ASMJIT_INLINE_NODEBUG EmbedLabelDeltaNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t baseLabelId = 0, uint32_t dataSize = 0) noexcept - : BaseNode(cb, NodeType::kEmbedLabelDelta, NodeFlags::kIsData), + ASMJIT_INLINE_NODEBUG EmbedLabelDeltaNode(uint32_t labelId = 0, uint32_t baseLabelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(NodeType::kEmbedLabelDelta, NodeFlags::kIsData), _labelId(labelId), _baseLabelId(baseLabelId), _dataSize(dataSize) {} @@ -1256,18 +1443,25 @@ public: //! \{ //! Returns the label as `Label` operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label label() const noexcept { return Label(_labelId); } - //! Returns the id of the label. - ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _labelId; } //! Sets the label id from `label` operand. ASMJIT_INLINE_NODEBUG void setLabel(const Label& label) noexcept { setLabelId(label.id()); } + + //! Returns the id of the label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _labelId; } + //! Sets the label id. ASMJIT_INLINE_NODEBUG void setLabelId(uint32_t labelId) noexcept { _labelId = labelId; } //! Returns the base label as `Label` operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label baseLabel() const noexcept { return Label(_baseLabelId); } + //! Returns the id of the base label. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t baseLabelId() const noexcept { return _baseLabelId; } //! Sets the base label id from `label` operand. @@ -1276,7 +1470,9 @@ public: ASMJIT_INLINE_NODEBUG void setBaseLabelId(uint32_t baseLabelId) noexcept { _baseLabelId = baseLabelId; } //! Returns the size of the embedded label address. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t dataSize() const noexcept { return _dataSize; } + //! Sets the size of the embedded label address. ASMJIT_INLINE_NODEBUG void setDataSize(uint32_t dataSize) noexcept { _dataSize = dataSize; } @@ -1299,13 +1495,13 @@ public: //! \{ //! Creates a new `ConstPoolNode` instance. - ASMJIT_INLINE_NODEBUG ConstPoolNode(BaseBuilder* cb, uint32_t id = 0) noexcept - : LabelNode(cb, id), - _constPool(&cb->_codeZone) { + ASMJIT_INLINE_NODEBUG ConstPoolNode(Zone* zone, uint32_t id = 0) noexcept + : LabelNode(id), + _constPool(zone) { - setType(NodeType::kConstPool); - addFlags(NodeFlags::kIsData); - clearFlags(NodeFlags::kIsCode | NodeFlags::kHasNoEffect); + _setType(NodeType::kConstPool); + _addFlags(NodeFlags::kIsData); + _clearFlags(NodeFlags::kIsCode | NodeFlags::kHasNoEffect); } //! \} @@ -1314,15 +1510,23 @@ public: //! \{ //! Tests whether the constant-pool is empty. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return _constPool.empty(); } + //! Returns the size of the constant-pool in bytes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t size() const noexcept { return _constPool.size(); } + //! Returns minimum alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t alignment() const noexcept { return _constPool.alignment(); } //! Returns the wrapped `ConstPool` instance. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ConstPool& constPool() noexcept { return _constPool; } + //! Returns the wrapped `ConstPool` instance (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ConstPool& constPool() const noexcept { return _constPool; } //! \} @@ -1347,8 +1551,8 @@ public: //! \{ //! Creates a new `CommentNode` instance. - ASMJIT_INLINE_NODEBUG CommentNode(BaseBuilder* cb, const char* comment) noexcept - : BaseNode(cb, NodeType::kComment, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect | NodeFlags::kIsRemovable) { + ASMJIT_INLINE_NODEBUG CommentNode(const char* comment) noexcept + : BaseNode(NodeType::kComment, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect | NodeFlags::kIsRemovable) { _inlineComment = comment; } @@ -1367,8 +1571,8 @@ public: //! \{ //! Creates a new `SentinelNode` instance. - ASMJIT_INLINE_NODEBUG SentinelNode(BaseBuilder* cb, SentinelType sentinelType = SentinelType::kUnknown) noexcept - : BaseNode(cb, NodeType::kSentinel, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect) { + ASMJIT_INLINE_NODEBUG SentinelNode(SentinelType sentinelType = SentinelType::kUnknown) noexcept + : BaseNode(NodeType::kSentinel, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect) { _sentinel._sentinelType = sentinelType; } @@ -1379,6 +1583,7 @@ public: //! \{ //! Returns the type of the sentinel. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SentinelType sentinelType() const noexcept { return _sentinel._sentinelType; } @@ -1419,8 +1624,11 @@ public: //! \{ //! Returns \ref BaseBuilder associated with the pass. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const BaseBuilder* cb() const noexcept { return _cb; } + //! Returns the name of the pass. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return _name; } //! \} @@ -1432,7 +1640,7 @@ public: //! //! This is the only function that is called by the `BaseBuilder` to process the code. It passes `zone`, //! which will be reset after the `run()` finishes. - ASMJIT_API virtual Error run(Zone* zone, Logger* logger) = 0; + ASMJIT_API virtual Error run(Zone* zone, Logger* logger); //! \} }; diff --git a/pe-packer/asmjit/core/builder_p.h b/pe-packer/asmjit/core/builder_p.h index 303358f..887e05b 100644 --- a/pe-packer/asmjit/core/builder_p.h +++ b/pe-packer/asmjit/core/builder_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_BUILDER_P_H_INCLUDED @@ -13,12 +13,14 @@ ASMJIT_BEGIN_NAMESPACE +//! \cond INTERNAL //! \addtogroup asmjit_builder //! \{ static inline void BaseBuilder_assignInlineComment(BaseBuilder* self, BaseNode* node, const char* comment) noexcept { - if (comment) - node->setInlineComment(static_cast(self->_dataZone.dup(comment, strlen(comment), true))); + if (comment) { + node->setInlineComment(static_cast(self->_codeZone.dup(comment, strlen(comment), true))); + } } static inline void BaseBuilder_assignInstState(BaseBuilder* self, InstNode* node, const BaseEmitter::State& state) noexcept { @@ -28,6 +30,7 @@ static inline void BaseBuilder_assignInstState(BaseBuilder* self, InstNode* node } //! \} +//! \endcond ASMJIT_END_NAMESPACE diff --git a/pe-packer/asmjit/core/codebuffer.h b/pe-packer/asmjit/core/codebuffer.h index d4b7ceb..ed77cdd 100644 --- a/pe-packer/asmjit/core/codebuffer.h +++ b/pe-packer/asmjit/core/codebuffer.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEBUFFER_H_INCLUDED @@ -45,11 +45,14 @@ struct CodeBuffer { //! \{ //! Returns a reference to the byte at the given `index`. + [[nodiscard]] inline uint8_t& operator[](size_t index) noexcept { ASMJIT_ASSERT(index < _size); return _data[index]; } + //! \overload + [[nodiscard]] inline const uint8_t& operator[](size_t index) const noexcept { ASMJIT_ASSERT(index < _size); return _data[index]; @@ -61,34 +64,47 @@ struct CodeBuffer { //! \{ //! Returns code buffer flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CodeBufferFlags flags() const noexcept { return _flags; } + //! Tests whether the code buffer has the given `flag` set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(CodeBufferFlags flag) const noexcept { return Support::test(_flags, flag); } //! Tests whether this code buffer has a fixed size. //! //! Fixed size means that the code buffer is fixed and cannot grow. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFixed() const noexcept { return hasFlag(CodeBufferFlags::kIsFixed); } //! Tests whether the data in this code buffer is external. //! //! External data can only be provided by users, it's never used by AsmJit. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isExternal() const noexcept { return hasFlag(CodeBufferFlags::kIsExternal); } //! Tests whether the data in this code buffer is allocated (non-null). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAllocated() const noexcept { return _data != nullptr; } //! Tests whether the code buffer is empty. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return !_size; } //! Returns the size of the data. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t size() const noexcept { return _size; } + //! Returns the capacity of the data. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t capacity() const noexcept { return _capacity; } //! Returns the pointer to the data the buffer references. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* data() noexcept { return _data; } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const uint8_t* data() const noexcept { return _data; } //! \} @@ -96,10 +112,16 @@ struct CodeBuffer { //! \name Iterators //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* begin() noexcept { return _data; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const uint8_t* begin() const noexcept { return _data; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* end() noexcept { return _data + _size; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const uint8_t* end() const noexcept { return _data + _size; } //! \} diff --git a/pe-packer/asmjit/core/codeholder.cpp b/pe-packer/asmjit/core/codeholder.cpp index 6c2b762..46b5709 100644 --- a/pe-packer/asmjit/core/codeholder.cpp +++ b/pe-packer/asmjit/core/codeholder.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -14,92 +14,181 @@ ASMJIT_BEGIN_NAMESPACE -// Globals -// ======= +// CodeHolder - X86 Utilities +// ========================== -static const char CodeHolder_addrTabName[] = ".addrtab"; - -//! Encode MOD byte. +//! Encodes a MOD byte. static inline uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept { return (m << 6) | (o << 3) | rm; } -// LabelLinkIterator -// ================= +// CodeHolder - LabelEntry Globals & Utilities +// =========================================== -class LabelLinkIterator { +static constexpr LabelEntry::ExtraData _makeSharedLabelExtraData() noexcept { + LabelEntry::ExtraData extraData {}; + extraData._sectionId = Globals::kInvalidId; + extraData._parentId = Globals::kInvalidId; + return extraData; +} + +static constexpr LabelEntry::ExtraData CodeHolder_sharedLabelExtraData = _makeSharedLabelExtraData(); + +class ResolveFixupIterator { public: - inline LabelLinkIterator(LabelEntry* le) noexcept { reset(le); } + Fixup* _fixup {}; + Fixup** _pPrev {}; + size_t _resolvedCount {}; + size_t _unresolvedCount {}; - inline explicit operator bool() const noexcept { return isValid(); } - inline bool isValid() const noexcept { return _link != nullptr; } + ASMJIT_INLINE_NODEBUG explicit ResolveFixupIterator(Fixup** ppFixup) noexcept { reset(ppFixup); } + ASMJIT_INLINE_NODEBUG bool isValid() const noexcept { return _fixup != nullptr; } + ASMJIT_INLINE_NODEBUG Fixup* fixup() const noexcept { return _fixup; } - inline LabelLink* link() const noexcept { return _link; } - inline LabelLink* operator->() const noexcept { return _link; } - - inline void reset(LabelEntry* le) noexcept { - _pPrev = &le->_links; - _link = *_pPrev; + ASMJIT_INLINE void reset(Fixup** ppFixup) noexcept { + _pPrev = ppFixup; + _fixup = *_pPrev; } - inline void next() noexcept { - _pPrev = &_link->next; - _link = *_pPrev; + ASMJIT_INLINE void next() noexcept { + _pPrev = &_fixup->next; + _fixup = *_pPrev; + _unresolvedCount++; } - inline void resolveAndNext(CodeHolder* code) noexcept { - LabelLink* linkToDelete = _link; + ASMJIT_INLINE void resolveAndNext(CodeHolder* code) noexcept { + Fixup* fixupToDelete = _fixup; - _link = _link->next; - *_pPrev = _link; + _fixup = _fixup->next; + *_pPrev = _fixup; - code->_unresolvedLinkCount--; - code->_allocator.release(linkToDelete, sizeof(LabelLink)); + _resolvedCount++; + code->_fixupDataPool.release(fixupToDelete); } - LabelLink** _pPrev; - LabelLink* _link; + ASMJIT_INLINE_NODEBUG size_t resolvedCount() const noexcept { return _resolvedCount; } + ASMJIT_INLINE_NODEBUG size_t unresolvedCount() const noexcept { return _unresolvedCount; } }; +// CodeHolder - Section Globals & Utilities +// ======================================== + +static const char CodeHolder_addrTabName[] = ".addrtab"; + +static ASMJIT_INLINE void Section_initName( + Section* section, + char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0, + char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept { + + section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3)); + section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7)); + section->_name.u32[2] = 0u; + section->_name.u32[3] = 0u; +} + +static ASMJIT_INLINE void Section_initData(Section* section, uint32_t sectionId, SectionFlags flags, uint32_t alignment, int order) noexcept { + section->_sectionId = sectionId; + + // These two fields are not used by sections (see \ref LabelEntry for more details about why). + section->_internalLabelType = LabelType::kAnonymous; + section->_internalLabelFlags = LabelFlags::kNone; + + section->assignFlags(flags); + section->_alignment = alignment; + section->_order = order; + section->_offset = 0; + section->_virtualSize = 0; +} + +static ASMJIT_INLINE void Section_initBuffer(Section* section) noexcept { + section->_buffer = CodeBuffer{}; +} + +static ASMJIT_INLINE void Section_releaseBuffer(Section* section) noexcept { + if (Support::bool_and(section->_buffer.data() != nullptr, !section->_buffer.isExternal())) { + ::free(section->_buffer._data); + } +} + // CodeHolder - Utilities // ====================== -static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) noexcept { - uint32_t i; - const ZoneVector& emitters = self->emitters(); +static ASMJIT_INLINE Error CodeHolder_initSectionStorage(CodeHolder* self) noexcept { + Error err1 = self->_sections.willGrow(&self->_allocator); + Error err2 = self->_sectionsByOrder.willGrow(&self->_allocator); - i = emitters.size(); - while (i) - self->detach(emitters[--i]); + return err1 | err2; +} - // Reset everything into its construction state. +static ASMJIT_INLINE void CodeHolder_addTextSection(CodeHolder* self) noexcept { + Section* textSection = &self->_textSection; + + Section_initData(textSection, 0u, SectionFlags::kExecutable | SectionFlags::kReadOnly | SectionFlags::kBuiltIn, 0u, 0); + Section_initName(textSection, '.', 't', 'e', 'x', 't'); + + self->_sections.appendUnsafe(textSection); + self->_sectionsByOrder.appendUnsafe(textSection); +} + +static ASMJIT_NOINLINE void CodeHolder_detachEmitters(CodeHolder* self) noexcept { + BaseEmitter* emitter = self->_attachedFirst; + + while (emitter) { + BaseEmitter* next = emitter->_attachedNext; + + emitter->_attachedPrev = nullptr; + (void)emitter->onDetach(*self); + emitter->_attachedNext = nullptr; + emitter->_code = nullptr; + + emitter = next; + self->_attachedFirst = next; + } + + self->_attachedLast = nullptr; +} + +static ASMJIT_INLINE void CodeHolder_resetEnvAndAttachedLogAndEH(CodeHolder* self) noexcept { self->_environment.reset(); self->_cpuFeatures.reset(); self->_baseAddress = Globals::kNoBaseAddress; self->_logger = nullptr; self->_errorHandler = nullptr; +} - // Reset all sections. - uint32_t numSections = self->_sections.size(); - for (i = 0; i < numSections; i++) { +// Reset zone allocator and all containers using it. +static ASMJIT_INLINE void CodeHolder_resetSections(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + // Reset all sections except the first one (.text section). + uint32_t fromSection = resetPolicy == ResetPolicy::kHard ? 0u : 1u; + uint32_t sectionCount = self->_sections.size(); + + for (uint32_t i = fromSection; i < sectionCount; i++) { Section* section = self->_sections[i]; - if (section->_buffer.data() && !section->_buffer.isExternal()) - ::free(section->_buffer._data); + + Section_releaseBuffer(section); section->_buffer._data = nullptr; section->_buffer._capacity = 0; } +} + +// Reset zone allocator and all containers using it. +static ASMJIT_INLINE void CodeHolder_resetContainers(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + // Soft reset won't wipe out the .text section, so set its size to 0 for future reuse. + self->_textSection._buffer._size = 0; - // Reset zone allocator and all containers using it. ZoneAllocator* allocator = self->allocator(); - self->_emitters.reset(); self->_namedLabels.reset(); self->_relocations.reset(); self->_labelEntries.reset(); + + self->_fixups = nullptr; + self->_fixupDataPool.reset(); + self->_unresolvedFixupCount = 0; + self->_sections.reset(); self->_sectionsByOrder.reset(); - self->_unresolvedLinkCount = 0; self->_addressTableSection = nullptr; self->_addressTableEntries.reset(); @@ -107,10 +196,18 @@ static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) self->_zone.reset(resetPolicy); } -static void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept { +// Reset zone allocator and all containers using it. +static ASMJIT_NOINLINE void CodeHolder_resetSectionsAndContainers(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + CodeHolder_resetSections(self, resetPolicy); + CodeHolder_resetContainers(self, resetPolicy); +} + +static ASMJIT_INLINE void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept { // Notify all attached emitters about a settings update. - for (BaseEmitter* emitter : self->emitters()) { + BaseEmitter* emitter = self->_attachedFirst; + while (emitter) { emitter->onSettingsUpdated(); + emitter = emitter->_attachedNext; } } @@ -123,69 +220,85 @@ CodeHolder::CodeHolder(const Support::Temporary* temporary) noexcept _baseAddress(Globals::kNoBaseAddress), _logger(nullptr), _errorHandler(nullptr), - _zone(16384 - Zone::kBlockOverhead, 1, temporary), + _zone(16u * 1024u, temporary), _allocator(&_zone), - _unresolvedLinkCount(0), + _attachedFirst(nullptr), + _attachedLast(nullptr), + _fixups(nullptr), + _unresolvedFixupCount(0), + _textSection{}, _addressTableSection(nullptr) {} CodeHolder::~CodeHolder() noexcept { - CodeHolder_resetInternal(this, ResetPolicy::kHard); + if (isInitialized()) { + CodeHolder_detachEmitters(this); + CodeHolder_resetSections(this, ResetPolicy::kHard); + } + else { + Section_releaseBuffer(&_textSection); + } } -// CodeHolder - Init & Reset -// ========================= - -inline void CodeHolder_setSectionDefaultName( - Section* section, - char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0, - char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept { - - section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3)); - section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7)); -} +// CodeHolder - Initialization & Reset +// =================================== Error CodeHolder::init(const Environment& environment, uint64_t baseAddress) noexcept { return init(environment, CpuFeatures{}, baseAddress); } Error CodeHolder::init(const Environment& environment, const CpuFeatures& cpuFeatures, uint64_t baseAddress) noexcept { - // Cannot reinitialize if it's locked or there is one or more emitter attached. - if (isInitialized()) - return DebugUtils::errored(kErrorAlreadyInitialized); + // Cannot initialize if it's already initialized or the environment passed is invalid. + if (ASMJIT_UNLIKELY(Support::bool_or(isInitialized(), !environment.isInitialized()))) { + Error err = isInitialized() ? kErrorAlreadyInitialized : kErrorInvalidArgument; + return DebugUtils::errored(err); + } // If we are just initializing there should be no emitters attached. - ASMJIT_ASSERT(_emitters.empty()); + ASMJIT_ASSERT(_attachedFirst == nullptr); + ASMJIT_ASSERT(_attachedLast == nullptr); // Create a default section and insert it to the `_sections` array. - Error err = _sections.willGrow(&_allocator) | - _sectionsByOrder.willGrow(&_allocator); - if (err == kErrorOk) { - Section* section = _allocator.allocZeroedT
(); - if (ASMJIT_LIKELY(section)) { - section->_flags = SectionFlags::kExecutable | SectionFlags::kReadOnly; - CodeHolder_setSectionDefaultName(section, '.', 't', 'e', 'x', 't'); - _sections.appendUnsafe(section); - _sectionsByOrder.appendUnsafe(section); - } - else { - err = DebugUtils::errored(kErrorOutOfMemory); - } - } - + Error err = CodeHolder_initSectionStorage(this); if (ASMJIT_UNLIKELY(err)) { _zone.reset(); - return err; + return DebugUtils::errored(kErrorOutOfMemory); } - else { - _environment = environment; - _cpuFeatures = cpuFeatures; - _baseAddress = baseAddress; - return kErrorOk; + + _environment = environment; + _cpuFeatures = cpuFeatures; + _baseAddress = baseAddress; + + CodeHolder_addTextSection(this); + return kErrorOk; +} + +Error CodeHolder::reinit() noexcept { + // Cannot reinitialize if it's not initialized. + if (ASMJIT_UNLIKELY(!isInitialized())) { + return DebugUtils::errored(kErrorNotInitialized); } + + CodeHolder_resetSectionsAndContainers(this, ResetPolicy::kSoft); + + // Create a default section and insert it to the `_sections` array. + (void)CodeHolder_initSectionStorage(this); + CodeHolder_addTextSection(this); + + BaseEmitter* emitter = _attachedFirst; + while (emitter) { + emitter->onReinit(*this); + emitter = emitter->_attachedNext; + } + + return kErrorOk; } void CodeHolder::reset(ResetPolicy resetPolicy) noexcept { - CodeHolder_resetInternal(this, resetPolicy); + if (isInitialized()) { + CodeHolder_detachEmitters(this); + CodeHolder_resetEnvAndAttachedLogAndEH(this); + CodeHolder_resetSectionsAndContainers(this, resetPolicy); + } } // CodeHolder - Attach / Detach @@ -193,57 +306,83 @@ void CodeHolder::reset(ResetPolicy resetPolicy) noexcept { Error CodeHolder::attach(BaseEmitter* emitter) noexcept { // Catch a possible misuse of the API. - if (ASMJIT_UNLIKELY(!emitter)) + if (ASMJIT_UNLIKELY(!emitter)) { return DebugUtils::errored(kErrorInvalidArgument); + } // Invalid emitter, this should not be possible. EmitterType type = emitter->emitterType(); - if (ASMJIT_UNLIKELY(type == EmitterType::kNone || uint32_t(type) > uint32_t(EmitterType::kMaxValue))) + if (ASMJIT_UNLIKELY(type == EmitterType::kNone || uint32_t(type) > uint32_t(EmitterType::kMaxValue))) { return DebugUtils::errored(kErrorInvalidState); + } uint64_t archMask = emitter->_archMask; - if (ASMJIT_UNLIKELY(!(archMask & (uint64_t(1) << uint32_t(arch()))))) + if (ASMJIT_UNLIKELY(!(archMask & (uint64_t(1) << uint32_t(arch()))))) { return DebugUtils::errored(kErrorInvalidArch); + } // This is suspicious, but don't fail if `emitter` is already attached // to this code holder. This is not error, but it's not recommended. if (emitter->_code != nullptr) { - if (emitter->_code == this) + if (emitter->_code == this) { return kErrorOk; + } return DebugUtils::errored(kErrorInvalidState); } // Reserve the space now as we cannot fail after `onAttach()` succeeded. - ASMJIT_PROPAGATE(_emitters.willGrow(&_allocator, 1)); - ASMJIT_PROPAGATE(emitter->onAttach(this)); + ASMJIT_PROPAGATE(emitter->onAttach(*this)); - // Connect CodeHolder <-> BaseEmitter. + // Make sure CodeHolder <-> BaseEmitter are connected. ASMJIT_ASSERT(emitter->_code == this); - _emitters.appendUnsafe(emitter); + + // Add `emitter` to a double linked-list. + { + BaseEmitter* last = _attachedLast; + + emitter->_attachedPrev = last; + _attachedLast = emitter; + + if (last) { + last->_attachedNext = emitter; + } + else { + _attachedFirst = emitter; + } + } return kErrorOk; } Error CodeHolder::detach(BaseEmitter* emitter) noexcept { - if (ASMJIT_UNLIKELY(!emitter)) + if (ASMJIT_UNLIKELY(!emitter)) { return DebugUtils::errored(kErrorInvalidArgument); + } - if (ASMJIT_UNLIKELY(emitter->_code != this)) + if (ASMJIT_UNLIKELY(emitter->_code != this)) { return DebugUtils::errored(kErrorInvalidState); + } // NOTE: We always detach if we were asked to, if error happens during // `emitter->onDetach()` we just propagate it, but the BaseEmitter will // be detached. Error err = kErrorOk; - if (!emitter->isDestroyed()) - err = emitter->onDetach(this); + if (!emitter->isDestroyed()) { + err = emitter->onDetach(*this); + } - // Disconnect CodeHolder <-> BaseEmitter. - uint32_t index = _emitters.indexOf(emitter); - ASMJIT_ASSERT(index != Globals::kNotFound); + // Remove `emitter` from a double linked-list. + { + BaseEmitter* prev = emitter->_attachedPrev; + BaseEmitter* next = emitter->_attachedNext; - _emitters.removeAt(index); - emitter->_code = nullptr; + if (prev) { prev->_attachedNext = next; } else { _attachedFirst = next; } + if (next) { next->_attachedPrev = prev; } else { _attachedLast = prev; } + + emitter->_code = nullptr; + emitter->_attachedPrev = nullptr; + emitter->_attachedNext = nullptr; + } return err; } @@ -275,19 +414,23 @@ static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t uint8_t* oldData = cb->_data; uint8_t* newData; - if (oldData && !cb->isExternal()) + if (oldData && !cb->isExternal()) { newData = static_cast(::realloc(oldData, n)); - else + } + else { newData = static_cast(::malloc(n)); + } - if (ASMJIT_UNLIKELY(!newData)) + if (ASMJIT_UNLIKELY(!newData)) { return DebugUtils::errored(kErrorOutOfMemory); + } cb->_data = newData; cb->_capacity = n; // Update pointers used by assemblers, if attached. - for (BaseEmitter* emitter : self->emitters()) { + BaseEmitter* emitter = self->_attachedFirst; + while (emitter) { if (emitter->isAssembler()) { BaseAssembler* a = static_cast(emitter); if (&a->_section->_buffer == cb) { @@ -298,6 +441,7 @@ static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t a->_bufferPtr = newData + offset; } } + emitter = emitter->_attachedNext; } return kErrorOk; @@ -306,35 +450,41 @@ static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept { // The size of the section must be valid. size_t size = cb->size(); - if (ASMJIT_UNLIKELY(n > std::numeric_limits::max() - size)) + if (ASMJIT_UNLIKELY(n > std::numeric_limits::max() - size)) { return DebugUtils::errored(kErrorOutOfMemory); + } // We can now check if growing the buffer is really necessary. It's unlikely // that this function is called while there is still room for `n` bytes. size_t capacity = cb->capacity(); size_t required = cb->size() + n; - if (ASMJIT_UNLIKELY(required <= capacity)) + + if (ASMJIT_UNLIKELY(required <= capacity)) { return kErrorOk; + } - if (cb->isFixed()) + if (cb->isFixed()) { return DebugUtils::errored(kErrorTooLarge); + } - size_t kInitialCapacity = 8096; - if (capacity < kInitialCapacity) + size_t kInitialCapacity = 8192u - Globals::kAllocOverhead; + if (capacity < kInitialCapacity) { capacity = kInitialCapacity; - else + } + else { capacity += Globals::kAllocOverhead; + } do { size_t old = capacity; - if (capacity < Globals::kGrowThreshold) - capacity *= 2; - else - capacity += Globals::kGrowThreshold; + size_t capacityIncrease = capacity < Globals::kGrowThreshold ? capacity : Globals::kGrowThreshold; + + capacity += capacityIncrease; // Overflow. - if (ASMJIT_UNLIKELY(old > capacity)) + if (ASMJIT_UNLIKELY(old > capacity)) { return DebugUtils::errored(kErrorOutOfMemory); + } } while (capacity - Globals::kAllocOverhead < required); return CodeHolder_reserveInternal(this, cb, capacity - Globals::kAllocOverhead); @@ -343,11 +493,13 @@ Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept { Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept { size_t capacity = cb->capacity(); - if (n <= capacity) + if (n <= capacity) { return kErrorOk; + } - if (cb->isFixed()) + if (cb->isFixed()) { return DebugUtils::errored(kErrorTooLarge); + } return CodeHolder_reserveInternal(this, cb, n); } @@ -358,37 +510,42 @@ Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept { Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t nameSize, SectionFlags flags, uint32_t alignment, int32_t order) noexcept { *sectionOut = nullptr; - if (nameSize == SIZE_MAX) - nameSize = strlen(name); - if (alignment == 0) - alignment = 1; - - if (ASMJIT_UNLIKELY(!Support::isPowerOf2(alignment))) + if (ASMJIT_UNLIKELY(!Support::isZeroOrPowerOf2(alignment))) { return DebugUtils::errored(kErrorInvalidArgument); + } - if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize)) + if (nameSize == SIZE_MAX) { + nameSize = strlen(name); + } + + if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize)) { return DebugUtils::errored(kErrorInvalidSectionName); + } uint32_t sectionId = _sections.size(); - if (ASMJIT_UNLIKELY(sectionId == Globals::kInvalidId)) + if (ASMJIT_UNLIKELY(sectionId == Globals::kInvalidId)) { return DebugUtils::errored(kErrorTooManySections); + } ASMJIT_PROPAGATE(_sections.willGrow(&_allocator)); ASMJIT_PROPAGATE(_sectionsByOrder.willGrow(&_allocator)); - Section* section = _allocator.allocZeroedT
(); - if (ASMJIT_UNLIKELY(!section)) + Section* section = _zone.alloc
(); + if (ASMJIT_UNLIKELY(!section)) { return DebugUtils::errored(kErrorOutOfMemory); + } - section->_id = sectionId; - section->_flags = flags; - section->_alignment = alignment; - section->_order = order; + if (alignment == 0u) { + alignment = 1u; + } + + Section_initData(section, sectionId, flags, alignment, order); + Section_initBuffer(section); memcpy(section->_name.str, name, nameSize); Section** insertPosition = std::lower_bound(_sectionsByOrder.begin(), _sectionsByOrder.end(), section, [](const Section* a, const Section* b) { - return std::make_tuple(a->order(), a->id()) < std::make_tuple(b->order(), b->id()); + return std::make_tuple(a->order(), a->sectionId()) < std::make_tuple(b->order(), b->sectionId()); }); _sections.appendUnsafe(section); @@ -399,24 +556,27 @@ Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t name } Section* CodeHolder::sectionByName(const char* name, size_t nameSize) const noexcept { - if (nameSize == SIZE_MAX) + if (nameSize == SIZE_MAX) { nameSize = strlen(name); + } - // This could be also put in a hash-table similarly like we do with labels, - // however it's questionable as the number of sections should be pretty low - // in general. Create an issue if this becomes a problem. + // This could be also put in a hash-table similarly like we do with labels, however it's questionable as + // the number of sections should be pretty low in general. Create an issue if this becomes a problem. if (nameSize <= Globals::kMaxSectionNameSize) { - for (Section* section : _sections) - if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0') + for (Section* section : _sections) { + if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0') { return section; + } + } } return nullptr; } Section* CodeHolder::ensureAddressTableSection() noexcept { - if (_addressTableSection) + if (_addressTableSection) { return _addressTableSection; + } newSection(&_addressTableSection, CodeHolder_addrTabName, @@ -429,16 +589,19 @@ Section* CodeHolder::ensureAddressTableSection() noexcept { Error CodeHolder::addAddressToAddressTable(uint64_t address) noexcept { AddressTableEntry* entry = _addressTableEntries.get(address); - if (entry) + if (entry) { return kErrorOk; + } Section* section = ensureAddressTableSection(); - if (ASMJIT_UNLIKELY(!section)) + if (ASMJIT_UNLIKELY(!section)) { return DebugUtils::errored(kErrorOutOfMemory); + } entry = _zone.newT(address); - if (ASMJIT_UNLIKELY(!entry)) + if (ASMJIT_UNLIKELY(!entry)) { return DebugUtils::errored(kErrorOutOfMemory); + } _addressTableEntries.insert(entry); section->_virtualSize += _environment.registerSize(); @@ -452,24 +615,26 @@ Error CodeHolder::addAddressToAddressTable(uint64_t address) noexcept { //! Only used to lookup a label from `_namedLabels`. class LabelByName { public: + const char* _key {}; + uint32_t _keySize {}; + uint32_t _hashCode {}; + uint32_t _parentId {}; + inline LabelByName(const char* key, size_t keySize, uint32_t hashCode, uint32_t parentId) noexcept : _key(key), _keySize(uint32_t(keySize)), _hashCode(hashCode), _parentId(parentId) {} + [[nodiscard]] inline uint32_t hashCode() const noexcept { return _hashCode; } - inline bool matches(const LabelEntry* entry) const noexcept { - return entry->nameSize() == _keySize && - entry->parentId() == _parentId && - ::memcmp(entry->name(), _key, _keySize) == 0; + [[nodiscard]] + inline bool matches(const CodeHolder::NamedLabelExtraData* node) const noexcept { + return Support::bool_and(node->extraData._nameSize == _keySize, + node->extraData._parentId == _parentId) && + ::memcmp(node->extraData.name(), _key, _keySize) == 0; } - - const char* _key; - uint32_t _keySize; - uint32_t _hashCode; - uint32_t _parentId; }; // Returns a hash of `name` and fixes `nameSize` if it's `SIZE_MAX`. @@ -479,7 +644,9 @@ static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize size_t i = 0; for (;;) { uint8_t c = uint8_t(name[i]); - if (!c) break; + if (!c) { + break; + } hashCode = Support::hashRound(hashCode, c); i++; } @@ -488,97 +655,106 @@ static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize else { for (size_t i = 0; i < nameSize; i++) { uint8_t c = uint8_t(name[i]); - if (ASMJIT_UNLIKELY(!c)) return DebugUtils::errored(kErrorInvalidLabelName); + if (ASMJIT_UNLIKELY(!c)) { + return DebugUtils::errored(kErrorInvalidLabelName); + } hashCode = Support::hashRound(hashCode, c); } } return hashCode; } -LabelLink* CodeHolder::newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept { - LabelLink* link = _allocator.allocT(); - if (ASMJIT_UNLIKELY(!link)) return nullptr; +Fixup* CodeHolder::newFixup(LabelEntry& le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept { + // Cannot be bound if we are creating a link. + ASMJIT_ASSERT(!le.isBound()); - link->next = le->_links; - le->_links = link; + Fixup* link = _fixupDataPool.alloc(_zone); + if (ASMJIT_UNLIKELY(!link)) { + return nullptr; + } + link->next = le._getFixups(); link->sectionId = sectionId; - link->relocId = Globals::kInvalidId; + link->labelOrRelocId = Globals::kInvalidId; link->offset = offset; link->rel = rel; link->format = format; - _unresolvedLinkCount++; + le._setFixups(link); + _unresolvedFixupCount++; + return link; } -Error CodeHolder::newLabelEntry(LabelEntry** entryOut) noexcept { - *entryOut = nullptr; - +Error CodeHolder::newLabelId(uint32_t* labelIdOut) noexcept { uint32_t labelId = _labelEntries.size(); - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) - return DebugUtils::errored(kErrorTooManyLabels); + Error err = _labelEntries.willGrow(&_allocator); - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - LabelEntry* le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) - return DebugUtils::errored(kErrorOutOfMemory); - - le->_setId(labelId); - le->_parentId = Globals::kInvalidId; - le->_offset = 0; - _labelEntries.appendUnsafe(le); - - *entryOut = le; - return kErrorOk; + if (ASMJIT_UNLIKELY(err != kErrorOk)) { + *labelIdOut = Globals::kInvalidId; + return err; + } + else { + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{const_cast(&CodeHolder_sharedLabelExtraData), uint64_t(0)}); + return kErrorOk; + } } -Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept { - *entryOut = nullptr; +Error CodeHolder::newNamedLabelId(uint32_t* labelIdOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept { + uint32_t labelId = _labelEntries.size(); uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize); - if (ASMJIT_UNLIKELY(nameSize == 0)) { - if (type == LabelType::kAnonymous) - return newLabelEntry(entryOut); - else + *labelIdOut = Globals::kInvalidId; + ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); + + if (nameSize == 0) { + if (type != LabelType::kAnonymous) { return DebugUtils::errored(kErrorInvalidLabelName); + } + + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{const_cast(&CodeHolder_sharedLabelExtraData), uint64_t(0)}); + return kErrorOk; } - if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize)) + if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize)) { return DebugUtils::errored(kErrorLabelNameTooLong); + } + + size_t extraDataSize = sizeof(LabelEntry::ExtraData) + nameSize + 1u; switch (type) { case LabelType::kAnonymous: { // Anonymous labels cannot have a parent (or more specifically, parent is useless here). - if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) + if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) { return DebugUtils::errored(kErrorInvalidParentLabel); + } - uint32_t labelId = _labelEntries.size(); - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) - return DebugUtils::errored(kErrorTooManyLabels); - - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - LabelEntry* le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) + LabelEntry::ExtraData* extraData = _zone.alloc(Support::alignUp(extraDataSize, Globals::kZoneAlignment)); + if (ASMJIT_UNLIKELY(!extraData)) { return DebugUtils::errored(kErrorOutOfMemory); + } - // NOTE: This LabelEntry has a name, but we leave its hashCode as zero as it's anonymous. - le->_setId(labelId); - le->_parentId = Globals::kInvalidId; - le->_offset = 0; - ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + char* namePtr = reinterpret_cast(extraData) + sizeof(LabelEntry::ExtraData); + extraData->_sectionId = Globals::kInvalidId; + extraData->_internalLabelType = type; + extraData->_internalLabelFlags = LabelFlags::kHasOwnExtraData | LabelFlags::kHasName; + extraData->_internalUInt16Data = 0; + extraData->_parentId = Globals::kInvalidId; + extraData->_nameSize = uint32_t(nameSize); + memcpy(namePtr, name, nameSize); + namePtr[nameSize] = '\0'; - _labelEntries.appendUnsafe(le); - - *entryOut = le; + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{extraData, uint64_t(0)}); return kErrorOk; } case LabelType::kLocal: { - if (ASMJIT_UNLIKELY(parentId >= _labelEntries.size())) + if (ASMJIT_UNLIKELY(parentId >= _labelEntries.size())) { return DebugUtils::errored(kErrorInvalidParentLabel); + } hashCode ^= parentId; break; @@ -586,8 +762,9 @@ Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, si case LabelType::kGlobal: case LabelType::kExternal: { - if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) + if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) { return DebugUtils::errored(kErrorInvalidParentLabel); + } break; } @@ -596,161 +773,198 @@ Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, si } } - // Don't allow to insert duplicates. Local labels allow duplicates that have - // different id, this is already accomplished by having a different hashes - // between the same label names having different parent labels. - LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); - if (ASMJIT_UNLIKELY(le)) + extraDataSize += sizeof(ZoneHashNode); + + // Don't allow to insert duplicates. Local labels allow duplicates that have different ids, however, this is + // already accomplished by having a different hashes between the same label names having different parent labels. + NamedLabelExtraData* namedNode = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + if (ASMJIT_UNLIKELY(namedNode)) { return DebugUtils::errored(kErrorLabelAlreadyDefined); + } - Error err = kErrorOk; - uint32_t labelId = _labelEntries.size(); - - if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) - return DebugUtils::errored(kErrorTooManyLabels); - - ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); - le = _allocator.allocZeroedT(); - - if (ASMJIT_UNLIKELY(!le)) + namedNode = _zone.alloc(Support::alignUp(extraDataSize, Globals::kZoneAlignment)); + if (ASMJIT_UNLIKELY(!namedNode)) { return DebugUtils::errored(kErrorOutOfMemory); + } - le->_hashCode = hashCode; - le->_setId(labelId); - le->_type = type; - le->_parentId = parentId; - le->_offset = 0; - ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + LabelFlags labelFlags = + (parentId == Globals::kInvalidId) + ? LabelFlags::kHasOwnExtraData | LabelFlags::kHasName + : LabelFlags::kHasOwnExtraData | LabelFlags::kHasName | LabelFlags::kHasParent; - _labelEntries.appendUnsafe(le); - _namedLabels.insert(allocator(), le); + namedNode->_hashNext = nullptr; + namedNode->_hashCode = hashCode; + namedNode->_customData = labelId; + namedNode->extraData._sectionId = Globals::kInvalidId; + namedNode->extraData._internalLabelType = type; + namedNode->extraData._internalLabelFlags = labelFlags; + namedNode->extraData._internalUInt16Data = 0; + namedNode->extraData._parentId = parentId; + namedNode->extraData._nameSize = uint32_t(nameSize); - *entryOut = le; - return err; + char* namePtr = reinterpret_cast(&namedNode->extraData) + sizeof(LabelEntry::ExtraData); + memcpy(namePtr, name, nameSize); + namePtr[nameSize] = '\0'; + + *labelIdOut = labelId; + _labelEntries.appendUnsafe(LabelEntry{&namedNode->extraData, uint64_t(0)}); + _namedLabels.insert(allocator(), namedNode); + + return kErrorOk; } uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t parentId) noexcept { uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize); - if (ASMJIT_UNLIKELY(!nameSize)) + if (ASMJIT_UNLIKELY(!nameSize)) { return 0; - - if (parentId != Globals::kInvalidId) - hashCode ^= parentId; - - LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); - return le ? le->id() : uint32_t(Globals::kInvalidId); -} - -ASMJIT_API Error CodeHolder::resolveUnresolvedLinks() noexcept { - if (!hasUnresolvedLinks()) - return kErrorOk; - - Error err = kErrorOk; - for (LabelEntry* le : labelEntries()) { - if (!le->isBound()) - continue; - - LabelLinkIterator link(le); - if (link) { - Support::FastUInt8 of = 0; - Section* toSection = le->section(); - uint64_t toOffset = Support::addOverflow(toSection->offset(), le->offset(), &of); - - do { - uint32_t linkSectionId = link->sectionId; - if (link->relocId == Globals::kInvalidId) { - Section* fromSection = sectionById(linkSectionId); - size_t linkOffset = link->offset; - - CodeBuffer& buf = _sections[linkSectionId]->buffer(); - ASMJIT_ASSERT(linkOffset < buf.size()); - - // Calculate the offset relative to the start of the virtual base. - Support::FastUInt8 localOF = of; - uint64_t fromOffset = Support::addOverflow(fromSection->offset(), linkOffset, &localOF); - int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(link->rel))); - - if (!localOF) { - ASMJIT_ASSERT(size_t(linkOffset) < buf.size()); - ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.valueSize()); - - // Overwrite a real displacement in the CodeBuffer. - if (CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { - link.resolveAndNext(this); - continue; - } - } - - err = DebugUtils::errored(kErrorInvalidDisplacement); - // Falls through to `link.next()`. - } - - link.next(); - } while (link); - } } + if (parentId != Globals::kInvalidId) { + hashCode ^= parentId; + } + + NamedLabelExtraData* namedNode = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + return namedNode ? namedNode->labelId() : uint32_t(Globals::kInvalidId); +} + + +ASMJIT_API Error CodeHolder::resolveCrossSectionFixups() noexcept { + if (!hasUnresolvedFixups()) { + return kErrorOk; + } + + Error err = kErrorOk; + ResolveFixupIterator it(&_fixups); + + while (it.isValid()) { + Fixup* fixup = it.fixup(); + LabelEntry& le = labelEntry(fixup->labelOrRelocId); + + Support::FastUInt8 of{}; + Section* toSection = _sections[le.sectionId()]; + uint64_t toOffset = Support::addOverflow(toSection->offset(), le.offset(), &of); + + Section* fromSection = sectionById(fixup->sectionId); + size_t fixupOffset = fixup->offset; + + CodeBuffer& buf = fromSection->buffer(); + ASMJIT_ASSERT(fixupOffset < buf.size()); + + // Calculate the offset relative to the start of the virtual base. + uint64_t fromOffset = Support::addOverflow(fromSection->offset(), fixupOffset, &of); + int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(fixup->rel))); + + if (ASMJIT_UNLIKELY(of)) { + err = DebugUtils::errored(kErrorInvalidDisplacement); + } + else { + ASMJIT_ASSERT(size_t(fixupOffset) < buf.size()); + ASMJIT_ASSERT(buf.size() - size_t(fixupOffset) >= fixup->format.valueSize()); + + // Overwrite a real displacement in the CodeBuffer. + if (CodeWriterUtils::writeOffset(buf._data + fixupOffset, displacement, fixup->format)) { + it.resolveAndNext(this); + continue; + } + } + + it.next(); + } + + _unresolvedFixupCount -= it.resolvedCount(); return err; } ASMJIT_API Error CodeHolder::bindLabel(const Label& label, uint32_t toSectionId, uint64_t toOffset) noexcept { - LabelEntry* le = labelEntry(label); - if (ASMJIT_UNLIKELY(!le)) - return DebugUtils::errored(kErrorInvalidLabel); + uint32_t labelId = label.id(); - if (ASMJIT_UNLIKELY(toSectionId > _sections.size())) + if (ASMJIT_UNLIKELY(labelId >= _labelEntries.size())) { + return DebugUtils::errored(kErrorInvalidLabel); + } + + if (ASMJIT_UNLIKELY(toSectionId >= _sections.size())) { return DebugUtils::errored(kErrorInvalidSection); + } + + LabelEntry& le = _labelEntries[labelId]; // Label can be bound only once. - if (ASMJIT_UNLIKELY(le->isBound())) + if (ASMJIT_UNLIKELY(le.isBound())) { return DebugUtils::errored(kErrorLabelAlreadyBound); + } - // Bind the label. Section* section = _sections[toSectionId]; - le->_section = section; - le->_offset = toOffset; - - Error err = kErrorOk; CodeBuffer& buf = section->buffer(); - // Fix all links to this label we have collected so far if they are within - // the same section. We ignore any inter-section links as these have to be - // fixed later. - LabelLinkIterator link(le); - while (link) { - uint32_t linkSectionId = link->sectionId; - size_t linkOffset = link->offset; + // Bind the label - this either assigns a section to LabelEntry's `_objectData` or `_sectionId` in own `ExtraData`. + // This is basically how this works - when the ExtraData is shared, we replace it by section as the section header + // is compatible with ExtraData header, and when the LabelEntry has its own ExtraData, the section identifier must + // be assigned. + if (le._hasOwnExtraData()) { + le._ownExtraData()->_sectionId = toSectionId; + } + else { + le._objectData = section; + } + + // It must be in this order as _offsetsOrFixups as basically a union. + Fixup* labelFixups = le._getFixups(); + le._offsetOrFixups = toOffset; + + if (!labelFixups) { + return kErrorOk; + } + + // Fix all fixups of this label we have collected so far if they are within the same + // section. We ignore any cross-section fixups as these have to be fixed later. + Error err = kErrorOk; + + ResolveFixupIterator it(&labelFixups); + ASMJIT_ASSERT(it.isValid()); + + do { + Fixup* fixup = it.fixup(); + + uint32_t relocId = fixup->labelOrRelocId; + uint32_t fromSectionId = fixup->sectionId; + size_t fromOffset = fixup->offset; - uint32_t relocId = link->relocId; if (relocId != Globals::kInvalidId) { - // Adjust relocation data only. + // Adjust the relocation payload. RelocEntry* re = _relocations[relocId]; re->_payload += toOffset; re->_targetSectionId = toSectionId; } + else if (fromSectionId != toSectionId) { + fixup->labelOrRelocId = labelId; + it.next(); + continue; + } else { - if (linkSectionId != toSectionId) { - link.next(); - continue; - } + ASMJIT_ASSERT(fromOffset < buf.size()); + int64_t displacement = int64_t(toOffset - uint64_t(fromOffset) + uint64_t(int64_t(fixup->rel))); - ASMJIT_ASSERT(linkOffset < buf.size()); - int64_t displacement = int64_t(toOffset - uint64_t(linkOffset) + uint64_t(int64_t(link->rel))); - - // Size of the value we are going to patch. Only BYTE/DWORD is allowed. - ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.regionSize()); + // Size of the value we are going to patch. + ASMJIT_ASSERT(buf.size() - size_t(fromOffset) >= fixup->format.regionSize()); // Overwrite a real displacement in the CodeBuffer. - if (!CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { + if (!CodeWriterUtils::writeOffset(buf._data + fromOffset, displacement, fixup->format)) { err = DebugUtils::errored(kErrorInvalidDisplacement); - link.next(); + fixup->labelOrRelocId = labelId; + it.next(); continue; } } - link.resolveAndNext(this); + it.resolveAndNext(this); + } while (it.isValid()); + + if (it.unresolvedCount()) { + *it._pPrev = _fixups; + _fixups = labelFixups; } + _unresolvedFixupCount -= it.resolvedCount(); return err; } @@ -761,17 +975,22 @@ Error CodeHolder::newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept ASMJIT_PROPAGATE(_relocations.willGrow(&_allocator)); uint32_t relocId = _relocations.size(); - if (ASMJIT_UNLIKELY(relocId == Globals::kInvalidId)) + if (ASMJIT_UNLIKELY(relocId == Globals::kInvalidId)) { return DebugUtils::errored(kErrorTooManyRelocations); + } - RelocEntry* re = _allocator.allocZeroedT(); - if (ASMJIT_UNLIKELY(!re)) + RelocEntry* re = _zone.alloc(); + if (ASMJIT_UNLIKELY(!re)) { return DebugUtils::errored(kErrorOutOfMemory); + } re->_id = relocId; re->_relocType = relocType; + re->_format = OffsetFormat{}; re->_sourceSectionId = Globals::kInvalidId; re->_targetSectionId = Globals::kInvalidId; + re->_sourceOffset = 0; + re->_payload = 0; _relocations.appendUnsafe(re); *dst = re; @@ -797,10 +1016,17 @@ static Error CodeHolder_evaluateExpression(CodeHolder* self, Expression* exp, ui } case ExpressionValueType::kLabel: { - LabelEntry* le = exp->value[i].label; - if (!le->isBound()) + uint32_t labelId = exp->value[i].labelId; + if (ASMJIT_UNLIKELY(labelId >= self->labelCount())) { + return DebugUtils::errored(kErrorInvalidLabel); + } + + LabelEntry& le = self->_labelEntries[labelId]; + if (!le.isBound()) { return DebugUtils::errored(kErrorExpressionLabelNotBound); - v = le->section()->offset() + le->offset(); + } + + v = self->_sections[le.sectionId()]->offset() + le.offset(); break; } @@ -863,14 +1089,16 @@ Error CodeHolder::flatten() noexcept { uint64_t realSize = section->realSize(); if (realSize) { uint64_t alignedOffset = Support::alignUp(offset, section->alignment()); - if (ASMJIT_UNLIKELY(alignedOffset < offset)) + if (ASMJIT_UNLIKELY(alignedOffset < offset)) { return DebugUtils::errored(kErrorTooLarge); + } Support::FastUInt8 of = 0; offset = Support::addOverflow(alignedOffset, realSize, &of); - if (ASMJIT_UNLIKELY(of)) + if (ASMJIT_UNLIKELY(of)) { return DebugUtils::errored(kErrorTooLarge); + } } } @@ -879,13 +1107,15 @@ Error CodeHolder::flatten() noexcept { offset = 0; for (Section* section : _sectionsByOrder) { uint64_t realSize = section->realSize(); - if (realSize) + if (realSize) { offset = Support::alignUp(offset, section->alignment()); + } section->_offset = offset; // Make sure the previous section extends a bit to cover the alignment. - if (prev) + if (prev) { prev->_virtualSize = offset - prev->_offset; + } prev = section; offset += realSize; @@ -908,16 +1138,27 @@ size_t CodeHolder::codeSize() const noexcept { } } - if ((sizeof(uint64_t) > sizeof(size_t) && offset > SIZE_MAX) || of) + if ((sizeof(uint64_t) > sizeof(size_t) && offset > uint64_t(SIZE_MAX)) || of) { return SIZE_MAX; + } return size_t(offset); } -Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { +Error CodeHolder::relocateToBase(uint64_t baseAddress, RelocationSummary* summaryOut) noexcept { + // Make sure `summaryOut` pointer is always valid as we want to fill it. + RelocationSummary summaryTmp; + if (summaryOut == nullptr) { + summaryOut = &summaryTmp; + } + + // Fill `summaryOut` defaults. + summaryOut->codeSizeReduction = 0u; + // Base address must be provided. - if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress)) + if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress)) { return DebugUtils::errored(kErrorInvalidArgument); + } _baseAddress = baseAddress; uint32_t addressSize = _environment.registerSize(); @@ -927,22 +1168,23 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { uint8_t* addressTableEntryData = nullptr; if (addressTableSection) { - ASMJIT_PROPAGATE( - reserveBuffer(&addressTableSection->_buffer, size_t(addressTableSection->virtualSize()))); + ASMJIT_PROPAGATE(reserveBuffer(&addressTableSection->_buffer, size_t(addressTableSection->virtualSize()))); addressTableEntryData = addressTableSection->_buffer.data(); } // Relocate all recorded locations. for (const RelocEntry* re : _relocations) { // Possibly deleted or optimized-out entry. - if (re->relocType() == RelocType::kNone) + if (re->relocType() == RelocType::kNone) { continue; + } Section* sourceSection = sectionById(re->sourceSectionId()); Section* targetSection = nullptr; - if (re->targetSectionId() != Globals::kInvalidId) + if (re->targetSectionId() != Globals::kInvalidId) { targetSection = sectionById(re->targetSectionId()); + } uint64_t value = re->payload(); uint64_t sectionOffset = sourceSection->offset(); @@ -951,8 +1193,9 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { // Make sure that the `RelocEntry` doesn't go out of bounds. size_t regionSize = re->format().regionSize(); if (ASMJIT_UNLIKELY(re->sourceOffset() >= sourceSection->bufferSize() || - sourceSection->bufferSize() - size_t(re->sourceOffset()) < regionSize)) + sourceSection->bufferSize() - size_t(re->sourceOffset()) < regionSize)) { return DebugUtils::errored(kErrorInvalidRelocEntry); + } uint8_t* buffer = sourceSection->data(); @@ -970,8 +1213,9 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { case RelocType::kRelToAbs: { // Value is currently a relative offset from the start of its section. // We have to convert it to an absolute offset (including base address). - if (ASMJIT_UNLIKELY(!targetSection)) + if (ASMJIT_UNLIKELY(!targetSection)) { return DebugUtils::errored(kErrorInvalidRelocEntry); + } //value += baseAddress + sectionOffset + sourceOffset + regionSize; value += baseAddress + targetSection->offset(); @@ -982,40 +1226,46 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { value -= baseAddress + sectionOffset + sourceOffset + regionSize; // Sign extend as we are not interested in the high 32-bit word in a 32-bit address space. - if (addressSize <= 4) + if (addressSize <= 4) { value = uint64_t(int64_t(int32_t(value & 0xFFFFFFFFu))); - else if (!Support::isInt32(int64_t(value))) + } + else if (!Support::isInt32(int64_t(value))) { return DebugUtils::errored(kErrorRelocOffsetOutOfRange); + } break; } case RelocType::kX64AddressEntry: { size_t valueOffset = size_t(re->sourceOffset()) + re->format().valueOffset(); - if (re->format().valueSize() != 4 || valueOffset < 2) + if (re->format().valueSize() != 4 || valueOffset < 2) { return DebugUtils::errored(kErrorInvalidRelocEntry); + } // First try whether a relative 32-bit displacement would work. value -= baseAddress + sectionOffset + sourceOffset + regionSize; if (!Support::isInt32(int64_t(value))) { // Relative 32-bit displacement is not possible, use '.addrtab' section. AddressTableEntry* atEntry = _addressTableEntries.get(re->payload()); - if (ASMJIT_UNLIKELY(!atEntry)) + if (ASMJIT_UNLIKELY(!atEntry)) { return DebugUtils::errored(kErrorInvalidRelocEntry); + } // Cannot be null as we have just matched the `AddressTableEntry`. ASMJIT_ASSERT(addressTableSection != nullptr); - if (!atEntry->hasAssignedSlot()) + if (!atEntry->hasAssignedSlot()) { atEntry->_slot = addressTableEntryCount++; + } size_t atEntryIndex = size_t(atEntry->slot()) * addressSize; uint64_t addrSrc = sectionOffset + sourceOffset + regionSize; uint64_t addrDst = addressTableSection->offset() + uint64_t(atEntryIndex); value = addrDst - addrSrc; - if (!Support::isInt32(int64_t(value))) + if (!Support::isInt32(int64_t(value))) { return DebugUtils::errored(kErrorRelocOffsetOutOfRange); + } // Bytes that replace [REX, OPCODE] bytes. uint32_t byte0 = 0xFF; @@ -1037,7 +1287,7 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { buffer[valueOffset - 2] = uint8_t(byte0); buffer[valueOffset - 1] = uint8_t(byte1); - Support::writeU64uLE(addressTableEntryData + atEntryIndex, re->payload()); + Support::storeu_u64_le(addressTableEntryData + atEntryIndex, re->payload()); } break; } @@ -1055,23 +1305,32 @@ Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { if (_sectionsByOrder.last() == addressTableSection) { ASMJIT_ASSERT(addressTableSection != nullptr); + size_t reservedSize = size_t(addressTableSection->_virtualSize); size_t addressTableSize = addressTableEntryCount * addressSize; + addressTableSection->_buffer._size = addressTableSize; addressTableSection->_virtualSize = addressTableSize; + + ASMJIT_ASSERT(reservedSize >= addressTableSize); + size_t codeSizeReduction = reservedSize - addressTableSize; + + summaryOut->codeSizeReduction = codeSizeReduction; } return kErrorOk; } Error CodeHolder::copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags) noexcept { - if (ASMJIT_UNLIKELY(!isSectionValid(sectionId))) + if (ASMJIT_UNLIKELY(!isSectionValid(sectionId))) { return DebugUtils::errored(kErrorInvalidSection); + } Section* section = sectionById(sectionId); size_t bufferSize = section->bufferSize(); - if (ASMJIT_UNLIKELY(dstSize < bufferSize)) + if (ASMJIT_UNLIKELY(dstSize < bufferSize)) { return DebugUtils::errored(kErrorInvalidArgument); + } memcpy(dst, section->data(), bufferSize); @@ -1086,14 +1345,16 @@ Error CodeHolder::copySectionData(void* dst, size_t dstSize, uint32_t sectionId, Error CodeHolder::copyFlattenedData(void* dst, size_t dstSize, CopySectionFlags copyFlags) noexcept { size_t end = 0; for (Section* section : _sectionsByOrder) { - if (section->offset() > dstSize) + if (section->offset() > dstSize) { return DebugUtils::errored(kErrorInvalidArgument); + } size_t bufferSize = section->bufferSize(); size_t offset = size_t(section->offset()); - if (ASMJIT_UNLIKELY(dstSize - offset < bufferSize)) + if (ASMJIT_UNLIKELY(dstSize - offset < bufferSize)) { return DebugUtils::errored(kErrorInvalidArgument); + } uint8_t* dstTarget = static_cast(dst) + offset; size_t paddingSize = 0; @@ -1129,10 +1390,30 @@ UNIT(code_holder) { EXPECT_EQ(code.arch(), Arch::kX86); INFO("Verifying named labels"); - LabelEntry* le; - EXPECT_EQ(code.newNamedLabelEntry(&le, "NamedLabel", SIZE_MAX, LabelType::kGlobal), kErrorOk); - EXPECT_EQ(strcmp(le->name(), "NamedLabel"), 0); - EXPECT_EQ(code.labelIdByName("NamedLabel"), le->id()); + uint32_t dummyId; + uint32_t labelId1; + uint32_t labelId2; + + // Anonymous labels can have no-name (this is basically like calling `code.newLabelId()`). + EXPECT_EQ(code.newNamedLabelId(&dummyId, "", SIZE_MAX, LabelType::kAnonymous), kErrorOk); + + // Global labels must have a name - not providing one is an error. + EXPECT_EQ(code.newNamedLabelId(&dummyId, "", SIZE_MAX, LabelType::kGlobal), kErrorInvalidLabelName); + + // A name of a global label cannot repeat. + EXPECT_EQ(code.newNamedLabelId(&labelId1, "NamedLabel1", SIZE_MAX, LabelType::kGlobal), kErrorOk); + EXPECT_EQ(code.newNamedLabelId(&dummyId, "NamedLabel1", SIZE_MAX, LabelType::kGlobal), kErrorLabelAlreadyDefined); + EXPECT_TRUE(code.isLabelValid(labelId1)); + EXPECT_EQ(code.labelEntry(labelId1).nameSize(), 11u); + EXPECT_EQ(strcmp(code.labelEntry(labelId1).name(), "NamedLabel1"), 0); + EXPECT_EQ(code.labelIdByName("NamedLabel1"), labelId1); + + EXPECT_EQ(code.newNamedLabelId(&labelId2, "NamedLabel2", SIZE_MAX, LabelType::kGlobal), kErrorOk); + EXPECT_EQ(code.newNamedLabelId(&dummyId, "NamedLabel2", SIZE_MAX, LabelType::kGlobal), kErrorLabelAlreadyDefined); + EXPECT_TRUE(code.isLabelValid(labelId2)); + EXPECT_EQ(code.labelEntry(labelId2).nameSize(), 11u); + EXPECT_EQ(strcmp(code.labelEntry(labelId2).name(), "NamedLabel2"), 0); + EXPECT_EQ(code.labelIdByName("NamedLabel2"), labelId2); INFO("Verifying section ordering"); Section* section1; diff --git a/pe-packer/asmjit/core/codeholder.h b/pe-packer/asmjit/core/codeholder.h index da661be..dd60999 100644 --- a/pe-packer/asmjit/core/codeholder.h +++ b/pe-packer/asmjit/core/codeholder.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEHOLDER_H_INCLUDED @@ -9,6 +9,7 @@ #include "../core/archtraits.h" #include "../core/codebuffer.h" #include "../core/errorhandler.h" +#include "../core/fixup.h" #include "../core/operand.h" #include "../core/string.h" #include "../core/support.h" @@ -45,7 +46,7 @@ enum class ExpressionOpType : uint8_t { kSra = 5 }; -//! Value tyoe that can be used within an \ref Expression. +//! Value type that can be used within an \ref Expression. enum class ExpressionValueType : uint8_t { //! No value or invalid. kNone = 0, @@ -65,8 +66,8 @@ struct Expression { uint64_t constant; //! Pointer to another expression. Expression* expression; - //! Pointer to \ref LabelEntry. - LabelEntry* label; + //! Label identifier + uint32_t labelId; }; //! \name Members @@ -98,9 +99,9 @@ struct Expression { } //! Sets the value type at `index` to \ref ExpressionValueType::kLabel and its content to `labelEntry`. - ASMJIT_INLINE_NODEBUG void setValueAsLabel(size_t index, LabelEntry* labelEntry) noexcept { + ASMJIT_INLINE_NODEBUG void setValueAsLabelId(size_t index, uint32_t labelId) noexcept { valueType[index] = ExpressionValueType::kLabel; - value[index].label = labelEntry; + value[index].labelId = labelId; } //! Sets the value type at `index` to \ref ExpressionValueType::kExpression and its content to `expression`. @@ -112,20 +113,64 @@ struct Expression { //! \} }; +//! Relocation type. +enum class RelocType : uint32_t { + //! None/deleted (no relocation). + kNone = 0, + //! Expression evaluation, `_payload` is pointer to `Expression`. + kExpression = 1, + //! Relative relocation from one section to another. + kSectionRelative = 2, + //! Relocate absolute to absolute. + kAbsToAbs = 3, + //! Relocate relative to absolute. + kRelToAbs = 4, + //! Relocate absolute to relative. + kAbsToRel = 5, + //! Relocate absolute to relative or use trampoline. + kX64AddressEntry = 6 +}; + +//! Type of the \ref Label. +enum class LabelType : uint8_t { + //! Anonymous label that can optionally have a name, which is only used for debugging purposes. + kAnonymous = 0u, + //! Local label (always has parentId). + kLocal = 1u, + //! Global label (never has parentId). + kGlobal = 2u, + //! External label (references an external symbol). + kExternal = 3u, + + //! Maximum value of `LabelType`. + kMaxValue = kExternal +}; + +//! Label flags describe some details about labels, mostly for AsmJit's own use. +enum class LabelFlags : uint8_t { + kNone = 0x00u, + kHasOwnExtraData = 0x01u, + kHasName = 0x02u, + kHasParent = 0x04u +}; +ASMJIT_DEFINE_ENUM_FLAGS(LabelFlags) + //! Section flags, used by \ref Section. enum class SectionFlags : uint32_t { //! No flags. kNone = 0, //! Executable (.text sections). - kExecutable = 0x00000001u, + kExecutable = 0x0001u, //! Read-only (.text and .data sections). - kReadOnly = 0x00000002u, + kReadOnly = 0x0002u, //! Zero initialized by the loader (BSS). - kZeroInitialized = 0x00000004u, + kZeroInitialized = 0x0004u, //! Info / comment flag. - kComment = 0x00000008u, + kComment = 0x0008u, + //! Section is built in and created by default (.text section). + kBuiltIn = 0x4000u, //! Section created implicitly, can be deleted by \ref Target. - kImplicit = 0x80000000u + kImplicit = 0x8000u }; ASMJIT_DEFINE_ENUM_FLAGS(SectionFlags) @@ -146,16 +191,36 @@ enum class CopySectionFlags : uint32_t { }; ASMJIT_DEFINE_ENUM_FLAGS(CopySectionFlags) -//! Section entry. -class Section { +//! Base class for both \ref Section and \ref LabelEntry::ExtraData. +class SectionOrLabelEntryExtraHeader { +public: + //! \name Members + //! \{ + + //! Section id - describes either a section where a \ref Label is bound or it's a real section id of \ref Section. + uint32_t _sectionId; + + //! Internal label type is only used by \ref LabelEntry::ExtraData. \ref Section always leaves this field zero, + //! which describes an anonymous label. Anonymous labels are default and always used when there is no + //! \ref LabelEntry::ExtraData + LabelType _internalLabelType; + + //! Internal label flags, used by \ref LabelEntry::ExtraData. \ref Section doesn't use these flags and sets them + //! to zero. + LabelFlags _internalLabelFlags; + + //! Internal data used freely by \ref Section and \ref LabelEntry::ExtraData. + uint16_t _internalUInt16Data; + + //! \} +}; + +//! Section entry. +class Section : public SectionOrLabelEntryExtraHeader { public: //! \name Members //! \{ - //! Section id. - uint32_t _id; - //! Section flags. - SectionFlags _flags; //! Section alignment requirements (0 if no requirements). uint32_t _alignment; //! Order (lower value means higher priority). @@ -175,34 +240,53 @@ public: //! \{ //! Returns the section id. - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _sectionId; } + //! Returns the section name, as a null terminated string. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return _name.str; } //! Returns the section data. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t* data() noexcept { return _buffer.data(); } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const uint8_t* data() const noexcept { return _buffer.data(); } //! Returns the section flags. - ASMJIT_INLINE_NODEBUG SectionFlags flags() const noexcept { return _flags; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG SectionFlags flags() const noexcept { return SectionFlags(_internalUInt16Data); } + //! Tests whether the section has the given `flag`. - ASMJIT_INLINE_NODEBUG bool hasFlag(SectionFlags flag) const noexcept { return Support::test(_flags, flag); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasFlag(SectionFlags flag) const noexcept { return Support::test(_internalUInt16Data, uint32_t(flag)); } + + //! Assigns `flags` to the section (replaces all existing flags). + ASMJIT_INLINE_NODEBUG void assignFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(flags); } + //! Adds `flags` to the section flags. - ASMJIT_INLINE_NODEBUG void addFlags(SectionFlags flags) noexcept { _flags |= flags; } + ASMJIT_INLINE_NODEBUG void addFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(_internalUInt16Data | uint32_t(flags)); } + //! Removes `flags` from the section flags. - ASMJIT_INLINE_NODEBUG void clearFlags(SectionFlags flags) noexcept { _flags &= ~flags; } + ASMJIT_INLINE_NODEBUG void clearFlags(SectionFlags flags) noexcept { _internalUInt16Data = uint16_t(_internalUInt16Data | ~uint32_t(flags)); } //! Returns the minimum section alignment + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t alignment() const noexcept { return _alignment; } + //! Sets the minimum section alignment ASMJIT_INLINE_NODEBUG void setAlignment(uint32_t alignment) noexcept { _alignment = alignment; } //! Returns the section order, which has a higher priority than section id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG int32_t order() const noexcept { return _order; } //! Returns the section offset, relative to base. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t offset() const noexcept { return _offset; } + //! Set the section offset. ASMJIT_INLINE_NODEBUG void setOffset(uint64_t offset) noexcept { _offset = offset; } @@ -212,18 +296,26 @@ public: //! size returned by `bufferSize()` as the buffer stores real data emitted by assemblers or appended by users. //! //! Use `realSize()` to get the real and final size of this section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t virtualSize() const noexcept { return _virtualSize; } + //! Sets the virtual size of the section. ASMJIT_INLINE_NODEBUG void setVirtualSize(uint64_t virtualSize) noexcept { _virtualSize = virtualSize; } //! Returns the buffer size of the section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t bufferSize() const noexcept { return _buffer.size(); } + //! Returns the real size of the section calculated from virtual and buffer sizes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t realSize() const noexcept { return Support::max(virtualSize(), bufferSize()); } //! Returns the `CodeBuffer` used by this section. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CodeBuffer& buffer() noexcept { return _buffer; } + //! Returns the `CodeBuffer` used by this section (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const CodeBuffer& buffer() const noexcept { return _buffer; } //! \} @@ -256,181 +348,30 @@ public: //! \name Accessors //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t address() const noexcept { return _address; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t slot() const noexcept { return _slot; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAssignedSlot() const noexcept { return _slot != 0xFFFFFFFFu; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool operator<(const AddressTableEntry& other) const noexcept { return _address < other._address; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool operator>(const AddressTableEntry& other) const noexcept { return _address > other._address; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool operator<(uint64_t queryAddress) const noexcept { return _address < queryAddress; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool operator>(uint64_t queryAddress) const noexcept { return _address > queryAddress; } //! \} }; -//! Offset format type, used by \ref OffsetFormat. -enum class OffsetType : uint8_t { - //! A value having `_immBitCount` bits and shifted by `_immBitShift`. - //! - //! This offset type is sufficient for many targets that store offset as a continuous set bits within an - //! instruction word / sequence of bytes. - kSignedOffset, - - //! An unsigned value having `_immBitCount` bits and shifted by `_immBitShift`. - kUnsignedOffset, - - // AArch64 Specific Offset Formats - // ------------------------------- - - //! AARCH64 ADR format of `[.|immlo:2|.....|immhi:19|.....]`. - kAArch64_ADR, - - //! AARCH64 ADRP format of `[.|immlo:2|.....|immhi:19|.....]` (4kB pages). - kAArch64_ADRP, - - //! Maximum value of `OffsetFormatType`. - kMaxValue = kAArch64_ADRP -}; - -//! Provides information about formatting offsets, absolute addresses, or their parts. Offset format is used by both -//! \ref RelocEntry and \ref LabelLink. The illustration below describes the relation of region size and offset size. -//! Region size is the size of the whole unit whereas offset size is the size of the unit that will be patched. -//! -//! ``` -//! +-> Code buffer | The subject of the relocation (region) | -//! | | (Word-Offset) (Word-Size) | -//! |xxxxxxxxxxxxxxx|................|*PATCHED*|................|xxxxxxxxxxxx-> -//! | | -//! [Word Offset points here]----+ +--- [WordOffset + WordSize] -//! ``` -//! -//! Once the offset word has been located it can be patched like this: -//! -//! ``` -//! |ImmDiscardLSB (discard LSB bits). -//! |.. -//! [0000000000000iiiiiiiiiiiiiiiiiDD] - Offset value (32-bit) -//! [000000000000000iiiiiiiiiiiiiiiii] - Offset value after discard LSB. -//! [00000000000iiiiiiiiiiiiiiiii0000] - Offset value shifted by ImmBitShift. -//! [xxxxxxxxxxxiiiiiiiiiiiiiiiiixxxx] - Patched word (32-bit) -//! |...............| -//! (ImmBitCount) +- ImmBitShift -//! ``` -struct OffsetFormat { - //! \name Members - //! \{ - - //! Type of the offset. - OffsetType _type; - //! Encoding flags. - uint8_t _flags; - //! Size of the region (in bytes) containing the offset value, if the offset value is part of an instruction, - //! otherwise it would be the same as `_valueSize`. - uint8_t _regionSize; - //! Size of the offset value, in bytes (1, 2, 4, or 8). - uint8_t _valueSize; - //! Offset of the offset value, in bytes, relative to the start of the region or data. Value offset would be - //! zero if both region size and value size are equal. - uint8_t _valueOffset; - //! Size of the offset immediate value in bits. - uint8_t _immBitCount; - //! Shift of the offset immediate value in bits in the target word. - uint8_t _immBitShift; - //! Number of least significant bits to discard before writing the immediate to the destination. All discarded - //! bits must be zero otherwise the value is invalid. - uint8_t _immDiscardLsb; - - //! \} - - //! \name Accessors - //! \{ - - //! Returns the type of the offset. - ASMJIT_INLINE_NODEBUG OffsetType type() const noexcept { return _type; } - - //! Returns flags. - ASMJIT_INLINE_NODEBUG uint32_t flags() const noexcept { return _flags; } - - //! Returns the size of the region/instruction where the offset is encoded. - ASMJIT_INLINE_NODEBUG uint32_t regionSize() const noexcept { return _regionSize; } - - //! Returns the offset of the word relative to the start of the region where the offset is. - ASMJIT_INLINE_NODEBUG uint32_t valueOffset() const noexcept { return _valueOffset; } - - //! Returns the size of the data-type (word) that contains the offset, in bytes. - ASMJIT_INLINE_NODEBUG uint32_t valueSize() const noexcept { return _valueSize; } - //! Returns the count of bits of the offset value in the data it's stored in. - ASMJIT_INLINE_NODEBUG uint32_t immBitCount() const noexcept { return _immBitCount; } - //! Returns the bit-shift of the offset value in the data it's stored in. - ASMJIT_INLINE_NODEBUG uint32_t immBitShift() const noexcept { return _immBitShift; } - //! Returns the number of least significant bits of the offset value, that must be zero and that are not part of - //! the encoded data. - ASMJIT_INLINE_NODEBUG uint32_t immDiscardLsb() const noexcept { return _immDiscardLsb; } - - //! Resets this offset format to a simple data value of `dataSize` bytes. - //! - //! The region will be the same size as data and immediate bits would correspond to `dataSize * 8`. There will be - //! no immediate bit shift or discarded bits. - inline void resetToSimpleValue(OffsetType type, size_t valueSize) noexcept { - ASMJIT_ASSERT(valueSize <= 8u); - - _type = type; - _flags = uint8_t(0); - _regionSize = uint8_t(valueSize); - _valueSize = uint8_t(valueSize); - _valueOffset = uint8_t(0); - _immBitCount = uint8_t(valueSize * 8u); - _immBitShift = uint8_t(0); - _immDiscardLsb = uint8_t(0); - } - - inline void resetToImmValue(OffsetType type, size_t valueSize, uint32_t immBitShift, uint32_t immBitCount, uint32_t immDiscardLsb) noexcept { - ASMJIT_ASSERT(valueSize <= 8u); - ASMJIT_ASSERT(immBitShift < valueSize * 8u); - ASMJIT_ASSERT(immBitCount <= 64u); - ASMJIT_ASSERT(immDiscardLsb <= 64u); - - _type = type; - _flags = uint8_t(0); - _regionSize = uint8_t(valueSize); - _valueSize = uint8_t(valueSize); - _valueOffset = uint8_t(0); - _immBitCount = uint8_t(immBitCount); - _immBitShift = uint8_t(immBitShift); - _immDiscardLsb = uint8_t(immDiscardLsb); - } - - inline void setRegion(size_t regionSize, size_t valueOffset) noexcept { - _regionSize = uint8_t(regionSize); - _valueOffset = uint8_t(valueOffset); - } - - inline void setLeadingAndTrailingSize(size_t leadingSize, size_t trailingSize) noexcept { - _regionSize = uint8_t(leadingSize + trailingSize + _valueSize); - _valueOffset = uint8_t(leadingSize); - } - - //! \} -}; - -//! Relocation type. -enum class RelocType : uint32_t { - //! None/deleted (no relocation). - kNone = 0, - //! Expression evaluation, `_payload` is pointer to `Expression`. - kExpression = 1, - //! Relocate absolute to absolute. - kAbsToAbs = 2, - //! Relocate relative to absolute. - kRelToAbs = 3, - //! Relocate absolute to relative. - kAbsToRel = 4, - //! Relocate absolute to relative or use trampoline. - kX64AddressEntry = 5 -}; - //! Relocation entry. struct RelocEntry { //! \name Members @@ -456,17 +397,28 @@ struct RelocEntry { //! \name Accessors //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG RelocType relocType() const noexcept { return _relocType; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const OffsetFormat& format() const noexcept { return _format; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t sourceSectionId() const noexcept { return _sourceSectionId; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t targetSectionId() const noexcept { return _targetSectionId; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t sourceOffset() const noexcept { return _sourceOffset; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t payload() const noexcept { return _payload; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG Expression* payloadAsExpression() const noexcept { return reinterpret_cast(uintptr_t(_payload)); } @@ -474,141 +426,158 @@ struct RelocEntry { //! \} }; -//! Type of the \ref Label. -enum class LabelType : uint8_t { - //! Anonymous label that can optionally have a name, which is only used for debugging purposes. - kAnonymous = 0, - //! Local label (always has parentId). - kLocal = 1, - //! Global label (never has parentId). - kGlobal = 2, - //! External label (references an external symbol). - kExternal = 3, - - //! Maximum value of `LabelType`. - kMaxValue = kExternal -}; - -//! Data structure used to link either unbound labels or cross-section links. -struct LabelLink { - //! Next link (single-linked list). - LabelLink* next; - //! Section id where the label is bound. - uint32_t sectionId; - //! Relocation id or Globals::kInvalidId. - uint32_t relocId; - //! Label offset relative to the start of the section. - size_t offset; - //! Inlined rel8/rel32. - intptr_t rel; - //! Offset format information. - OffsetFormat format; -}; - -//! Label entry. +//! Label entry provides data stored by \ref CodeHolder for each \ref Label. //! -//! Contains the following properties: -//! - Label id - This is the only thing that is set to the `Label` operand. -//! - Label name - Optional, used mostly to create executables and libraries. -//! - Label type - Type of the label, default `LabelType::kAnonymous`. -//! - Label parent id - Derived from many assemblers that allow to define a local label that falls under a global -//! label. This allows to define many labels of the same name that have different parent (global) label. -//! - Offset - offset of the label bound by `Assembler`. -//! - Links - single-linked list that contains locations of code that has to be patched when the label gets bound. -//! Every use of unbound label adds one link to `_links` list. -//! - HVal - Hash value of label's name and optionally parentId. -//! - HashNext - Hash-table implementation detail. -class LabelEntry : public ZoneHashNode { +//! Label entry is used mostly internall by AsmJit, but it's possibly to use it to query various information about +//! a label. For example to get its type, flags, name, and fixups (if the label is not bound) or offset (if the label +//! is bound). +//! +//! To make the entry small, it's currently split into two data structures - \ref LabelEntry, which is stored in an +//! array as a value, and \ref LabelEntry::ExtraData, which can be pointed to via \ref LabelEntry::_objectData. Extra +//! data of unnamed anonymous labels is shared (and immutable), thus all unnamed anonymous labels would only use +//! \ref LabelEntry (16 bytes per label). +class LabelEntry { public: - //! \name Constants - //! \{ + //! Contains extra data that is only created when the label is not anonymous or has a name. + struct ExtraData : public SectionOrLabelEntryExtraHeader { + //! Label parent id or zero. + uint32_t _parentId; + //! Label name length. + uint32_t _nameSize; - enum : uint32_t { - //! SSO size of \ref _name. - //! - //! \cond INTERNAL - //! Let's round the size of `LabelEntry` to 64 bytes (as `ZoneAllocator` has granularity of 32 bytes anyway). This - //! gives `_name` the remaining space, which is should be 16 bytes on 64-bit and 28 bytes on 32-bit architectures. - //! \endcond - kStaticNameSize = 64 - (sizeof(ZoneHashNode) + 8 + sizeof(Section*) + sizeof(size_t) + sizeof(LabelLink*)) + //! Returns a name associated with this extra data - a valid pointer is only returned when the label has a name, which + //! is marked by \ref LabelFlags::kHasName flag. + ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return Support::offsetPtr(this, sizeof(ExtraData)); } }; - //! \} - //! \name Members //! \{ - //! Type of the label. - LabelType _type; - //! Must be zero. - uint8_t _reserved[3]; - //! Label parent id or zero. - uint32_t _parentId; - //! Label offset relative to the start of the `_section`. - uint64_t _offset; - //! Section where the label was bound. - Section* _section; - //! Label links. - LabelLink* _links; - //! Label name. - ZoneString _name; + // Either references a \ref Section where the label is bound or \ref ExtraData. + SectionOrLabelEntryExtraHeader* _objectData; + + //! Label entry payload. + //! + //! When a Label is bound, `_offsetOrFixups` is the relative offset from the start of the section where + //! the \ref Label has been bound, otherwise `_offsetOrFixups` is a pointer to the first \ref Fixup. + uint64_t _offsetOrFixups; //! \} //! \name Accessors //! \{ - // NOTE: Label id is stored in `_customData`, which is provided by ZoneHashNode to fill a padding that a C++ - // compiler targeting 64-bit CPU will add to align the structure to 64-bits. + //! Returns the type of the label. + //! + //! The type of the label depends on how it was created. Most JIT code uses unnamed anonymous labels created by + //! emitters, for example \ref BaseEmitter::newLabel() returns a \ref Label instance having id that was created + //! by \ref CodeHolder::newLabelId. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG LabelType labelType() const noexcept { return _objectData->_internalLabelType; } - //! Returns label id. - ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _customData; } - //! Sets label id (internal, used only by `CodeHolder`). - ASMJIT_INLINE_NODEBUG void _setId(uint32_t id) noexcept { _customData = id; } + //! Returns label flags. + //! + //! \note Label flags are mostly for internal use, there is probably no reason to use them in user code. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG LabelFlags labelFlags() const noexcept { return _objectData->_internalLabelFlags; } - //! Returns label type. - ASMJIT_INLINE_NODEBUG LabelType type() const noexcept { return _type; } + //! Tests whether the label has the given `flag` set. + //! + //! \note Using other getters instead is advised, for example using \ref hasName() and \ref hasParent() is better + //! (and shorter) than checking label flags. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasLabelFlag(LabelFlags flag) const noexcept { return Support::test(_objectData->_internalLabelFlags, flag); } - //! Tests whether the label has a parent label. - ASMJIT_INLINE_NODEBUG bool hasParent() const noexcept { return _parentId != Globals::kInvalidId; } - //! Returns label's parent id. - ASMJIT_INLINE_NODEBUG uint32_t parentId() const noexcept { return _parentId; } + //! Tests whether the LabelEntry has own extra data (see \ref LabelEntry::ExtraData). + //! + //! \note This should only be used by AsmJit for internal purposes. Own extra data means that the LabelEntry has + //! a mutable extra data separately allocated. This information should not be necessary to users as LabelEntry + //! getters should encapsulate label introspection. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool _hasOwnExtraData() const noexcept { return hasLabelFlag(LabelFlags::kHasOwnExtraData); } + + //! Tests whether the Label represented by this LabelEntry has a name. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasName() const noexcept { return hasLabelFlag(LabelFlags::kHasName); } + + //! Tests whether the Label represented by this LabelEntry has a parent label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasParent() const noexcept { return hasLabelFlag(LabelFlags::kHasParent); } + + //! Tests whether the label represented by this LabelEntry is bound. + //! + //! Bound label means that it has an associated \ref Section and a position in such section. Labels are bound by + //! calling \ref BaseEmitter::bind() method with \ref Label operand. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBound() const noexcept { return _objectData->_sectionId != Globals::kInvalidId; } + + //! Tests whether the label is bound to a the given `section`. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBoundTo(const Section* section) const noexcept { return _objectData->_sectionId == section->sectionId(); } + + //! Tests whether the label is bound to a the given `sectionId`. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isBoundTo(uint32_t sectionId) const noexcept { return _objectData->_sectionId == sectionId; } //! Returns the section where the label was bound. //! //! If the label was not yet bound the return value is `nullptr`. - ASMJIT_INLINE_NODEBUG Section* section() const noexcept { return _section; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t sectionId() const noexcept { return _objectData->_sectionId; } - //! Tests whether the label has name. - ASMJIT_INLINE_NODEBUG bool hasName() const noexcept { return !_name.empty(); } + [[nodiscard]] + ASMJIT_INLINE ExtraData* _ownExtraData() const noexcept { + ASMJIT_ASSERT(_hasOwnExtraData()); + return static_cast(_objectData); + } + + //! Returns label's parent id or \ref Globals::kInvalidId if the label has no parent. + [[nodiscard]] + ASMJIT_INLINE uint32_t parentId() const noexcept { + return _hasOwnExtraData() ? _ownExtraData()->_parentId : Globals::kInvalidId; + } //! Returns the label's name. //! //! \note Local labels will return their local name without their parent part, for example ".L1". - ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return _name.data(); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const char* name() const noexcept { + return hasName() ? _ownExtraData()->name() : nullptr; + } //! Returns size of label's name. //! //! \note Label name is always null terminated, so you can use `strlen()` to get it, however, it's also cached in //! `LabelEntry` itself, so if you want to know the size the fastest way is to call `LabelEntry::nameSize()`. - ASMJIT_INLINE_NODEBUG uint32_t nameSize() const noexcept { return _name.size(); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t nameSize() const noexcept { + return hasName() ? _ownExtraData()->_nameSize : uint32_t(0); + } - //! Returns links associated with this label. - ASMJIT_INLINE_NODEBUG LabelLink* links() const noexcept { return _links; } + //! Returns unresolved fixups associated with this label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasFixups() const noexcept { + return Support::bool_and(!isBound(), _offsetOrFixups != 0u); + } - //! Tests whether the label is bound. - ASMJIT_INLINE_NODEBUG bool isBound() const noexcept { return _section != nullptr; } - //! Tests whether the label is bound to a the given `sectionId`. - ASMJIT_INLINE_NODEBUG bool isBoundTo(Section* section) const noexcept { return _section == section; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG Fixup* _getFixups() const noexcept { return reinterpret_cast(uintptr_t(_offsetOrFixups)); } - //! Returns the label offset (only useful if the label is bound). - ASMJIT_INLINE_NODEBUG uint64_t offset() const noexcept { return _offset; } + ASMJIT_INLINE_NODEBUG void _setFixups(Fixup* first) noexcept { _offsetOrFixups = reinterpret_cast(first); } - //! Returns the hash-value of label's name and its parent label (if any). + //! Returns unresolved fixups associated with this label. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG Fixup* unresolvedLinks() const noexcept { return !isBound() ? _getFixups() : nullptr; } + + //! Returns the label offset (can only be used after the label is bound). //! - //! Label hash is calculated as `HASH(Name) ^ ParentId`. The hash function is implemented in `Support::hashString()` - //! and `Support::hashRound()`. - ASMJIT_INLINE_NODEBUG uint32_t hashCode() const noexcept { return _hashCode; } + //! \note This would trigger an assertion failure in debug builds when called on an unbound label. When accessing + //! offsets, always check whether the label is bound. Unbound labels don't have offsets. + [[nodiscard]] + ASMJIT_INLINE uint64_t offset() const noexcept { + ASMJIT_ASSERT(isBound()); + return _offsetOrFixups; + } //! \} }; @@ -629,12 +598,84 @@ public: //! emitters. After the end of code generation it can be used to query physical locations of labels and to relocate //! the assembled code into the right address. //! +//! CodeHolder Reusability +//! ---------------------- +//! +//! If you intend to generate a lot of code, or tiny code, it's advised to reuse CodeHolder and emitter instances. +//! There are currently two ways of reusing CodeHolder and emitters - one is using \ref CodeHolder::init() followed +//! by \ref CodeHolder::reset(), and another is initializing once by \ref CodeHolder::init() and then reinitializing +//! by \ref CodeHolder::reinit(). The first strategy is shown below: +//! +//! ``` +//! // All of them will be reused for code generation by using an 'init()/reset()' strategy. +//! Environment env = ...; // Environment to use, for example from JitRuntime. +//! CodeHolder code; // CodeHolder to reuse (all allocated memory will be held by it until it's destroyed). +//! x86::Compiler cc; // Emitter to reuse (for example x86::Compiler). +//! +//! for (size_t i = 0; i < ...; i++) { +//! // Initialize the CodeHolder first. +//! code.init(env); +//! code.attach(&emitter); +//! +//! [[code generation as usual]] +//! +//! code.reset(); +//! } +//! ``` +//! +//! While this approach is good for many use-cases, there is even a faster strategy called reinitialization, which is +//! provided by \ref CodeHolder::reinit(). The idea of reinit is to reinitialize the CodeHolder into a state, which +//! was achieved by initializing it by \ref CodeHolder::init(), by optionally attaching \ref Logger, \ref ErrorHandler, +//! and emitters of any kind. See an example below: +//! +//! ``` +//! // All of them will be reused for code generation by using a 'reinit()' strategy. +//! Environment env = ...; // Environment to use, for example from JitRuntime. +//! CodeHolder code; // CodeHolder to reuse (all allocated memory will be held by it until it's destroyed). +//! x86::Compiler cc; // Emitter to reuse (for example x86::Compiler). +//! +//! // Initialize the CodeHolder and attach emitters to it (attaching ErrorHandler is advised!) +//! code.init(env); +//! code.attach(&emitter); +//! +//! for (size_t i = 0; i < ...; i++) { +//! [[code generation as usual]] +//! +//! // Optionally you can start the loop with 'code.reinit()', but this is cleaner as it wipes out all intermediate +//! // states of CodeHolder and the attached emitters. It won't detach Logger, ErrorHandler, nor attached emitters. +//! code.reinit(); +//! } +//! ``` +//! //! \note \ref CodeHolder has an ability to attach an \ref ErrorHandler, however, the error handler is not triggered //! by \ref CodeHolder itself, it's instead propagated to all emitters that attach to it. class CodeHolder { public: ASMJIT_NONCOPYABLE(CodeHolder) + //! \name Types + //! \{ + + //! \cond INTERNAL + struct NamedLabelExtraData : public ZoneHashNode { + LabelEntry::ExtraData extraData; + + ASMJIT_INLINE_NODEBUG uint32_t labelId() const noexcept { return _customData; } + }; + //! \endcond + + //! An informative data structure that is filled with some details that happened during \ref relocateToBase(). + struct RelocationSummary { + //! The number of bytes the final code has been reduced by. + //! + //! At the moment this is the same as the number of bytes that the address table was shrunk, because it was + //! possible to avoid certain entries during relocation - the functions that would be otherwise present were + //! close enough to avoid them in the .addrtab section. + size_t codeSizeReduction; + }; + + //! \} + //! \name Members //! \{ @@ -655,21 +696,32 @@ public: //! Zone allocator, used to manage internal containers. ZoneAllocator _allocator; - //! Attached emitters. - ZoneVector _emitters; + //! First emitter attached to this CodeHolder (double-linked list). + BaseEmitter* _attachedFirst; + //! Last emitter attached to this CodeHolder (double-linked list). + BaseEmitter* _attachedLast; + //! Section entries. ZoneVector _sections; //! Section entries sorted by section order and then section id. ZoneVector _sectionsByOrder; + //! Label entries. - ZoneVector _labelEntries; + ZoneVector _labelEntries; //! Relocation entries. ZoneVector _relocations; - //! Label name -> LabelEntry (only named labels). - ZoneHash _namedLabels; + //! Label name -> LabelEntry::ExtraData (only used by labels that have a name and are not anonymous). + ZoneHash _namedLabels; + //! Unresolved fixups that are most likely references across sections. + Fixup* _fixups; + //! Pool containing \ref Fixup instances for quickly recycling them. + ZonePool _fixupDataPool; + //! Count of unresolved fixups of unbound labels (at the end of assembling this should be zero). + size_t _unresolvedFixupCount; + + //! Text section - always one part of a CodeHolder itself. + Section _textSection; - //! Count of label links, which are not resolved. - size_t _unresolvedLinkCount; //! Pointer to an address table section (or null if this section doesn't exist). Section* _addressTableSection; //! Address table entries. @@ -696,12 +748,29 @@ public: //! Tests whether the `CodeHolder` has been initialized. //! //! Emitters can be only attached to initialized `CodeHolder` instances. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInitialized() const noexcept { return _environment.isInitialized(); } //! Initializes CodeHolder to hold code described by the given `environment` and `baseAddress`. ASMJIT_API Error init(const Environment& environment, uint64_t baseAddress = Globals::kNoBaseAddress) noexcept; //! Initializes CodeHolder to hold code described by the given `environment`, `cpuFeatures`, and `baseAddress`. ASMJIT_API Error init(const Environment& environment, const CpuFeatures& cpuFeatures, uint64_t baseAddress = Globals::kNoBaseAddress) noexcept; + + //! Reinitializes CodeHolder with the same environment, cpu features, and base address as it had, and notifies + //! all attached emitters of reinitialization. If the \ref CodeHolder was not initialized, \ref kErrorNotInitialized + //! is returned. + //! + //! Reinitialization is designed to be a faster alternative compared to \ref reset() followed by \ref init() chain. + //! The purpose of reinitialization is a very quick reuse of \ref CodeHolder and all attached emitters (most likely + //! Assembler or Compiler) without paying the cost of complete initialization and then assignment of all the loggers, + //! error handlers, and emitters. + //! + //! \note Semantically reinit() is the same as using \ref reset(ResetPolicy::kSoft), followed by \ref init(), and + //! then by attaching loggers, error handlers, and emitters that were attached previously. This means that after + //! reinitialization you will get a clean and ready for use \ref CodeHolder, which was initialized the same way as + //! before. + ASMJIT_API Error reinit() noexcept; + //! Detaches all code-generators attached and resets the `CodeHolder`. ASMJIT_API void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept; @@ -717,7 +786,7 @@ public: //! \} - //! \name Allocators + //! \name Memory Allocators //! \{ //! Returns the allocator that the `CodeHolder` uses. @@ -725,6 +794,7 @@ public: //! \note This should be only used for AsmJit's purposes. Code holder uses arena allocator to allocate everything, //! so anything allocated through this allocator will be invalidated by \ref CodeHolder::reset() or by CodeHolder's //! destructor. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ZoneAllocator* allocator() const noexcept { return const_cast(&_allocator); } //! \} @@ -733,28 +803,46 @@ public: //! \{ //! Returns the target environment information. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const Environment& environment() const noexcept { return _environment; } //! Returns the target architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return environment().arch(); } + //! Returns the target sub-architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SubArch subArch() const noexcept { return environment().subArch(); } //! Returns the minimum CPU features of the target architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const CpuFeatures& cpuFeatures() const noexcept { return _cpuFeatures; } //! Tests whether a static base-address is set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasBaseAddress() const noexcept { return _baseAddress != Globals::kNoBaseAddress; } + //! Returns a static base-address or \ref Globals::kNoBaseAddress, if not set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t baseAddress() const noexcept { return _baseAddress; } //! \} - //! \name Emitters + //! \name Attached Emitters //! \{ //! Returns a vector of attached emitters. - ASMJIT_INLINE_NODEBUG const ZoneVector& emitters() const noexcept { return _emitters; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG BaseEmitter* attachedFirst() noexcept { return _attachedFirst; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG BaseEmitter* attachedLast() noexcept { return _attachedLast; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const BaseEmitter* attachedFirst() const noexcept { return _attachedFirst; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const BaseEmitter* attachedLast() const noexcept { return _attachedLast; } //! \} @@ -762,9 +850,12 @@ public: //! \{ //! Returns the attached logger. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Logger* logger() const noexcept { return _logger; } + //! Attaches a `logger` to CodeHolder and propagates it to all attached emitters. ASMJIT_API void setLogger(Logger* logger) noexcept; + //! Resets the logger to none. ASMJIT_INLINE_NODEBUG void resetLogger() noexcept { setLogger(nullptr); } @@ -772,11 +863,16 @@ public: //! \{ //! Tests whether the CodeHolder has an attached error handler, see \ref ErrorHandler. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasErrorHandler() const noexcept { return _errorHandler != nullptr; } + //! Returns the attached error handler. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ErrorHandler* errorHandler() const noexcept { return _errorHandler; } + //! Attach an error handler to this `CodeHolder`. ASMJIT_API void setErrorHandler(ErrorHandler* errorHandler) noexcept; + //! Resets the error handler to none. ASMJIT_INLINE_NODEBUG void resetErrorHandler() noexcept { setErrorHandler(nullptr); } @@ -801,13 +897,19 @@ public: //! \{ //! Returns an array of `Section*` records. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& sections() const noexcept { return _sections; } + //! Returns an array of `Section*` records sorted according to section order first, then section id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& sectionsByOrder() const noexcept { return _sectionsByOrder; } + //! Returns the number of sections. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t sectionCount() const noexcept { return _sections.size(); } //! Tests whether the given `sectionId` is valid. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isSectionValid(uint32_t sectionId) const noexcept { return sectionId < _sections.size(); } //! Creates a new section and return its pointer in `sectionOut`. @@ -816,19 +918,23 @@ public: ASMJIT_API Error newSection(Section** sectionOut, const char* name, size_t nameSize = SIZE_MAX, SectionFlags flags = SectionFlags::kNone, uint32_t alignment = 1, int32_t order = 0) noexcept; //! Returns a section entry of the given index. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Section* sectionById(uint32_t sectionId) const noexcept { return _sections[sectionId]; } //! Returns section-id that matches the given `name`. //! //! If there is no such section `Section::kInvalidId` is returned. + [[nodiscard]] ASMJIT_API Section* sectionByName(const char* name, size_t nameSize = SIZE_MAX) const noexcept; //! Returns '.text' section (section that commonly represents code). //! //! \note Text section is always the first section in \ref CodeHolder::sections() array. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Section* textSection() const noexcept { return _sections[0]; } //! Tests whether '.addrtab' section exists. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAddressTable() const noexcept { return _addressTableSection != nullptr; } //! Returns '.addrtab' section. @@ -837,10 +943,12 @@ public: //! addresses that cannot be encoded in instructions like 'jmp' or 'call'. //! //! \note This section is created on demand, the returned pointer can be null. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Section* addressTableSection() const noexcept { return _addressTableSection; } //! Ensures that '.addrtab' section exists (creates it if it doesn't) and //! returns it. Can return `nullptr` on out of memory condition. + [[nodiscard]] ASMJIT_API Section* ensureAddressTableSection() noexcept; //! Used to add an address to an address table. @@ -850,7 +958,7 @@ public: //! use the same slot. //! //! This function should be considered internal as it's used by assemblers to insert an absolute address into the - //! address table. Inserting address into address table without creating a particula relocation entry makes no sense. + //! address table. Inserting address into address table without creating a particular relocation entry makes no sense. ASMJIT_API Error addAddressToAddressTable(uint64_t address) noexcept; //! \} @@ -858,77 +966,118 @@ public: //! \name Labels & Symbols //! \{ - //! Returns array of `LabelEntry*` records. - ASMJIT_INLINE_NODEBUG const ZoneVector& labelEntries() const noexcept { return _labelEntries; } + //! Returns array of `LabelEntry` records. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const ZoneVector& labelEntries() const noexcept { return _labelEntries; } //! Returns number of labels created. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t labelCount() const noexcept { return _labelEntries.size(); } - //! Tests whether the label having `id` is valid (i.e. created by `newLabelEntry()`). + //! Tests whether the label having `labelId` is valid (i.e. created by `newLabelId()`). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelValid(uint32_t labelId) const noexcept { return labelId < _labelEntries.size(); } - //! Tests whether the `label` is valid (i.e. created by `newLabelEntry()`). + //! Tests whether the `label` is valid (i.e. created by `newLabelId()`). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelValid(const Label& label) const noexcept { - return label.id() < _labelEntries.size(); + return isLabelValid(label.id()); } - //! \overload + //! Tests whether a label having `labelId` is already bound. + //! + //! Returns `false` if the `labelId` is not valid. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelBound(uint32_t labelId) const noexcept { - return isLabelValid(labelId) && _labelEntries[labelId]->isBound(); + return isLabelValid(labelId) && _labelEntries[labelId].isBound(); } //! Tests whether the `label` is already bound. //! //! Returns `false` if the `label` is not valid. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelBound(const Label& label) const noexcept { return isLabelBound(label.id()); } - //! Returns LabelEntry of the given label `id`. - ASMJIT_INLINE_NODEBUG LabelEntry* labelEntry(uint32_t labelId) const noexcept { - return isLabelValid(labelId) ? _labelEntries[labelId] : static_cast(nullptr); + //! Returns LabelEntry of the given label identifier `labelId` (or `label` if you are using overloads). + //! + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG LabelEntry& labelEntry(uint32_t labelId) noexcept { + return _labelEntries[labelId]; } - //! Returns LabelEntry of the given `label`. - ASMJIT_INLINE_NODEBUG LabelEntry* labelEntry(const Label& label) const noexcept { + //! \overload + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const LabelEntry& labelEntry(uint32_t labelId) const noexcept { + return _labelEntries[labelId]; + } + + //! \overload + [[nodiscard]] + ASMJIT_INLINE_NODEBUG LabelEntry& labelEntry(const Label& label) noexcept { + return labelEntry(label.id()); + } + + //! \overload + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const LabelEntry& labelEntry(const Label& label) const noexcept { return labelEntry(label.id()); } //! Returns offset of a `Label` by its `labelId`. //! - //! The offset returned is relative to the start of the section. Zero offset is returned for unbound labels, - //! which is their initial offset value. + //! The offset returned is relative to the start of the section where the label is bound. Zero offset is returned + //! for unbound labels, which is their initial offset value. + //! + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t labelOffset(uint32_t labelId) const noexcept { ASMJIT_ASSERT(isLabelValid(labelId)); - return _labelEntries[labelId]->offset(); + return _labelEntries[labelId].offset(); } //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t labelOffset(const Label& label) const noexcept { return labelOffset(label.id()); } //! Returns offset of a label by it's `labelId` relative to the base offset. //! - //! \remarks The offset of the section where the label is bound must be valid in order to use this function, - //! otherwise the value returned will not be reliable. + //! \attention The passed `labelId` must be valid as it's used as an index to `_labelEntries[]` array. In debug + //! builds the array access uses an assertion, but such assertion is not present in release builds. To get whether + //! a label is valid, check out \ref CodeHolder::isLabelValid() function. + //! + //! \note The offset of the section where the label is bound must be valid in order to use this function, otherwise + //! the value returned will not be reliable. Typically, sections have offsets when they are flattened, see \ref + //! CodeHolder::flatten() function for more details. + [[nodiscard]] inline uint64_t labelOffsetFromBase(uint32_t labelId) const noexcept { ASMJIT_ASSERT(isLabelValid(labelId)); - const LabelEntry* le = _labelEntries[labelId]; - return (le->isBound() ? le->section()->offset() : uint64_t(0)) + le->offset(); + + const LabelEntry& le = _labelEntries[labelId]; + return (le.isBound() ? _sections[le.sectionId()]->offset() : uint64_t(0)) + le.offset(); } //! \overload + [[nodiscard]] inline uint64_t labelOffsetFromBase(const Label& label) const noexcept { return labelOffsetFromBase(label.id()); } - //! Creates a new anonymous label and return its id in `idOut`. + //! Creates a new anonymous label and return its id in `labelIdOut`. //! //! Returns `Error`, does not report error to `ErrorHandler`. - ASMJIT_API Error newLabelEntry(LabelEntry** entryOut) noexcept; + [[nodiscard]] + ASMJIT_API Error newLabelId(uint32_t* labelIdOut) noexcept; //! Creates a new named \ref LabelEntry of the given label `type`. //! @@ -945,12 +1094,14 @@ public: //! addition, AsmJit supports named anonymous labels, which are useful only for debugging purposes as the //! anonymous name will have a name, which will be formatted, but the label itself cannot be queried by such //! name. - ASMJIT_API Error newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId = Globals::kInvalidId) noexcept; + [[nodiscard]] + ASMJIT_API Error newNamedLabelId(uint32_t* labelIdOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId = Globals::kInvalidId) noexcept; //! Returns a label by name. //! //! If the named label doesn't a default constructed \ref Label is returned, //! which has its id set to \ref Globals::kInvalidId. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label labelByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept { return Label(labelIdByName(name, nameSize, parentId)); } @@ -958,22 +1109,27 @@ public: //! Returns a label id by name. //! //! If the named label doesn't exist \ref Globals::kInvalidId is returned. + [[nodiscard]] ASMJIT_API uint32_t labelIdByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept; - //! Tests whether there are any unresolved label links. - ASMJIT_INLINE_NODEBUG bool hasUnresolvedLinks() const noexcept { return _unresolvedLinkCount != 0; } - //! Returns the number of label links, which are unresolved. - ASMJIT_INLINE_NODEBUG size_t unresolvedLinkCount() const noexcept { return _unresolvedLinkCount; } + //! Tests whether there are any unresolved fixups related to unbound labels. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasUnresolvedFixups() const noexcept { return _unresolvedFixupCount != 0u; } + + //! Returns the number of unresolved fixups. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG size_t unresolvedFixupCount() const noexcept { return _unresolvedFixupCount; } //! Creates a new label-link used to store information about yet unbound labels. //! //! Returns `null` if the allocation failed. - ASMJIT_API LabelLink* newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept; + [[nodiscard]] + ASMJIT_API Fixup* newFixup(LabelEntry& le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept; - //! Resolves cross-section links (`LabelLink`) associated with each label that was used as a destination in code - //! of a different section. It's only useful to people that use multiple sections as it will do nothing if the code - //! only contains a single section in which cross-section links are not possible. - ASMJIT_API Error resolveUnresolvedLinks() noexcept; + //! Resolves cross-section fixups associated with each label that was used as a destination in code of a different + //! section. It's only useful to people that use multiple sections as it will do nothing if the code only contains + //! a single section in which cross-section fixups are not possible. + ASMJIT_API Error resolveCrossSectionFixups() noexcept; //! Binds a label to a given `sectionId` and `offset` (relative to start of the section). //! @@ -986,16 +1142,21 @@ public: //! \{ //! Tests whether the code contains relocation entries. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRelocEntries() const noexcept { return !_relocations.empty(); } + //! Returns array of `RelocEntry*` records. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& relocEntries() const noexcept { return _relocations; } //! Returns a RelocEntry of the given `id`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG RelocEntry* relocEntry(uint32_t id) const noexcept { return _relocations[id]; } //! Creates a new relocation entry of type `relocType`. //! //! Additional fields can be set after the relocation entry was created. + [[nodiscard]] ASMJIT_API Error newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept; //! \} @@ -1013,15 +1174,17 @@ public: //! \note All sections will be iterated over and the code size returned would represent the minimum code size of //! all combined sections after applying minimum alignment. Code size may decrease after calling `flatten()` and //! `relocateToBase()`. + [[nodiscard]] ASMJIT_API size_t codeSize() const noexcept; //! Relocates the code to the given `baseAddress`. //! //! \param baseAddress Absolute base address where the code will be relocated to. Please note that nothing is - //! copied to such base address, it's just an absolute value used by the relocator to resolve all stored relocations. + //! copied to such base address, it's just an absolute value used by the relocation code to resolve all stored + //! relocations. //! //! \note This should never be called more than once. - ASMJIT_API Error relocateToBase(uint64_t baseAddress) noexcept; + ASMJIT_API Error relocateToBase(uint64_t baseAddress, RelocationSummary* summaryOut = nullptr) noexcept; //! Copies a single section into `dst`. ASMJIT_API Error copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags = CopySectionFlags::kNone) noexcept; diff --git a/pe-packer/asmjit/core/codewriter.cpp b/pe-packer/asmjit/core/codewriter.cpp index 1babc5f..b9edeea 100644 --- a/pe-packer/asmjit/core/codewriter.cpp +++ b/pe-packer/asmjit/core/codewriter.cpp @@ -1,11 +1,12 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #include "../core/codeholder.h" #include "../core/codewriter_p.h" +#include "../arm/armutils.h" ASMJIT_BEGIN_NAMESPACE @@ -15,39 +16,57 @@ bool CodeWriterUtils::encodeOffset32(uint32_t* dst, int64_t offset64, const Offs uint32_t discardLsb = format.immDiscardLsb(); // Invalid offset (should not happen). - if (!bitCount || bitCount > format.valueSize() * 8u) + if (!bitCount || bitCount > format.valueSize() * 8u) { return false; + } uint32_t value; + uint32_t u = 0; + bool unsignedLogic = format.type() == OffsetType::kUnsignedOffset; + + // First handle all offsets that use additional field for their sign and the offset is encoded as its + // absolute value. + if (format.hasSignBit()) { + u = uint32_t(offset64 >= 0); + if (u == 0) { + offset64 = -offset64; + } + unsignedLogic = true; + } // First handle all unsigned offset types. - if (format.type() == OffsetType::kUnsignedOffset) { + if (unsignedLogic) { if (discardLsb) { ASMJIT_ASSERT(discardLsb <= 32); - if ((offset64 & Support::lsbMask(discardLsb)) != 0) + if ((offset64 & Support::lsbMask(discardLsb)) != 0) { return false; + } offset64 = int64_t(uint64_t(offset64) >> discardLsb); } value = uint32_t(offset64 & Support::lsbMask(bitCount)); - if (value != offset64) + if (value != offset64) { return false; + } } else { // The rest of OffsetType options are all signed. if (discardLsb) { ASMJIT_ASSERT(discardLsb <= 32); - if ((offset64 & Support::lsbMask(discardLsb)) != 0) + if ((offset64 & Support::lsbMask(discardLsb)) != 0) { return false; + } offset64 >>= discardLsb; } - if (!Support::isInt32(offset64)) + if (!Support::isInt32(offset64)) { return false; + } value = uint32_t(int32_t(offset64)); - if (!Support::isEncodableOffset32(int32_t(value), bitCount)) + if (!Support::isEncodableOffset32(int32_t(value), bitCount)) { return false; + } } switch (format.type()) { @@ -57,11 +76,109 @@ bool CodeWriterUtils::encodeOffset32(uint32_t* dst, int64_t offset64, const Offs return true; } + // Opcode: {.....|imm:1|..N.N|......|imm:3|....|imm:8} + case OffsetType::kThumb32_ADR: { + // Sanity checks. + if (format.valueSize() != 4 || bitCount != 12 || bitShift != 0) { + return false; + } + + uint32_t imm8 = (value & 0x00FFu); + uint32_t imm3 = (value & 0x0700u) << (12 - 8); + uint32_t imm1 = (value & 0x0800u) << (26 - 11); + uint32_t n = u ^ 1u; + + *dst = imm8 | imm3 | imm1 | (n << 21) | (n << 23); + return true; + } + + // Opcode: {....|.|imm[22]|imm[19:10]|..|ja|.|jb|imm[9:0]|.} + case OffsetType::kThumb32_BLX: + // The calculation is the same as `B`, but the first LSB bit must be zero, so account for that. + value <<= 1; + [[fallthrough]]; + + // Opcode: {....|.|imm[23]|imm[20:11]|..|ja|.|jb|imm[10:0]} + case OffsetType::kThumb32_B: { + // Sanity checks. + if (format.valueSize() != 4) { + return false; + } + + uint32_t ia = (value & 0x0007FFu); + uint32_t ib = (value & 0x1FF800u) << (16 - 11); + uint32_t ic = (value & 0x800000u) << (26 - 23); + uint32_t ja = ((~value >> 23) ^ (value >> 22)) & 1u; + uint32_t jb = ((~value >> 23) ^ (value >> 21)) & 1u; + + *dst = ia | ib | ic | (ja << 14) | (jb << 11); + return true; + } + + // Opcode: {....|.|imm[19]|....|imm[16:11]|..|ja|.|jb|imm[10:0]} + case OffsetType::kThumb32_BCond: { + // Sanity checks. + if (format.valueSize() != 4 || bitCount != 20 || bitShift != 0) { + return false; + } + + uint32_t ia = (value & 0x0007FFu); + uint32_t ib = (value & 0x01F800u) << (16 - 11); + uint32_t ic = (value & 0x080000u) << (26 - 19); + uint32_t ja = ((~value >> 19) ^ (value >> 22)) & 1u; + uint32_t jb = ((~value >> 19) ^ (value >> 21)) & 1u; + + *dst = ia | ib | ic | (ja << 14) | (jb << 11); + return true; + } + + case OffsetType::kAArch32_ADR: { + uint32_t encodedImm; + if (!arm::Utils::encodeAArch32Imm(value, &encodedImm)) { + return false; + } + + *dst = (Support::bitMask(22) << u) | (encodedImm << bitShift); + return true; + } + + case OffsetType::kAArch32_U23_SignedOffset: { + *dst = (value << bitShift) | (u << 23); + return true; + } + + case OffsetType::kAArch32_U23_0To3At0_4To7At8: { + // Sanity checks. + if (format.valueSize() != 4 || bitCount != 8 || bitShift != 0) { + return false; + } + + uint32_t immLo = (value & 0x0Fu); + uint32_t immHi = (value & 0xF0u) << (8 - 4); + + *dst = immLo | immHi | (u << 23); + return true; + } + + case OffsetType::kAArch32_1To24At0_0At24: { + // Sanity checks. + if (format.valueSize() != 4 || bitCount != 25 || bitShift != 0) { + return false; + } + + uint32_t immLo = (value & 0x0000001u) << 24; + uint32_t immHi = (value & 0x1FFFFFEu) >> 1; + + *dst = immLo | immHi; + return true; + } + case OffsetType::kAArch64_ADR: case OffsetType::kAArch64_ADRP: { // Sanity checks. - if (format.valueSize() != 4 || bitCount != 21 || bitShift != 5) + if (format.valueSize() != 4 || bitCount != 21 || bitShift != 5) { return false; + } uint32_t immLo = value & 0x3u; uint32_t immHi = (value >> 2) & Support::lsbMask(19); @@ -79,8 +196,9 @@ bool CodeWriterUtils::encodeOffset64(uint64_t* dst, int64_t offset64, const Offs uint32_t bitCount = format.immBitCount(); uint32_t discardLsb = format.immDiscardLsb(); - if (!bitCount || bitCount > format.valueSize() * 8u) + if (!bitCount || bitCount > format.valueSize() * 8u) { return false; + } uint64_t value; @@ -88,26 +206,30 @@ bool CodeWriterUtils::encodeOffset64(uint64_t* dst, int64_t offset64, const Offs if (format.type() == OffsetType::kUnsignedOffset) { if (discardLsb) { ASMJIT_ASSERT(discardLsb <= 32); - if ((offset64 & Support::lsbMask(discardLsb)) != 0) + if ((offset64 & Support::lsbMask(discardLsb)) != 0) { return false; + } offset64 = int64_t(uint64_t(offset64) >> discardLsb); } value = uint64_t(offset64) & Support::lsbMask(bitCount); - if (value != uint64_t(offset64)) + if (value != uint64_t(offset64)) { return false; + } } else { // The rest of OffsetType options are all signed. if (discardLsb) { ASMJIT_ASSERT(discardLsb <= 32); - if ((offset64 & Support::lsbMask(discardLsb)) != 0) + if ((offset64 & Support::lsbMask(discardLsb)) != 0) { return false; + } offset64 >>= discardLsb; } - if (!Support::isEncodableOffset64(offset64, bitCount)) + if (!Support::isEncodableOffset64(offset64, bitCount)) { return false; + } value = uint64_t(offset64); } @@ -132,19 +254,21 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma switch (format.valueSize()) { case 1: { uint32_t mask; - if (!encodeOffset32(&mask, offset64, format)) + if (!encodeOffset32(&mask, offset64, format)) { return false; + } - Support::writeU8(dst, uint8_t(Support::readU8(dst) | mask)); + Support::store_u8(dst, uint8_t(Support::load_u8(dst) | mask)); return true; } case 2: { uint32_t mask; - if (!encodeOffset32(&mask, offset64, format)) + if (!encodeOffset32(&mask, offset64, format)) { return false; + } - Support::writeU16uLE(dst, uint16_t(Support::readU16uLE(dst) | mask)); + Support::storeu_u16_le(dst, uint16_t(Support::loadu_u16_le(dst) | mask)); return true; } @@ -154,16 +278,17 @@ bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetForma return false; } - Support::writeU32uLE(dst, Support::readU32uLE(dst) | mask); + Support::storeu_u32_le(dst, Support::loadu_u32_le(dst) | mask); return true; } case 8: { uint64_t mask; - if (!encodeOffset64(&mask, offset64, format)) + if (!encodeOffset64(&mask, offset64, format)) { return false; + } - Support::writeU64uLE(dst, Support::readU64uLE(dst) | mask); + Support::storeu_u64_le(dst, Support::loadu_u64_le(dst) | mask); return true; } diff --git a/pe-packer/asmjit/core/codewriter_p.h b/pe-packer/asmjit/core/codewriter_p.h index c799241..efb6667 100644 --- a/pe-packer/asmjit/core/codewriter_p.h +++ b/pe-packer/asmjit/core/codewriter_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CODEBUFFERWRITER_P_H_INCLUDED @@ -23,10 +23,11 @@ class CodeWriter { public: uint8_t* _cursor; - ASMJIT_FORCE_INLINE explicit CodeWriter(BaseAssembler* a) noexcept + ASMJIT_INLINE_NODEBUG explicit CodeWriter(BaseAssembler* a) noexcept : _cursor(a->_bufferPtr) {} - ASMJIT_FORCE_INLINE Error ensureSpace(BaseAssembler* a, size_t n) noexcept { + [[nodiscard]] + ASMJIT_INLINE Error ensureSpace(BaseAssembler* a, size_t n) noexcept { size_t remainingSpace = (size_t)(a->_bufferEnd - _cursor); if (ASMJIT_UNLIKELY(remainingSpace < n)) { CodeBuffer& buffer = a->_section->_buffer; @@ -38,25 +39,28 @@ public: return kErrorOk; } - ASMJIT_FORCE_INLINE uint8_t* cursor() const noexcept { return _cursor; } - ASMJIT_FORCE_INLINE void setCursor(uint8_t* cursor) noexcept { _cursor = cursor; } - ASMJIT_FORCE_INLINE void advance(size_t n) noexcept { _cursor += n; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint8_t* cursor() const noexcept { return _cursor; } - ASMJIT_FORCE_INLINE size_t offsetFrom(uint8_t* from) const noexcept { + ASMJIT_INLINE_NODEBUG void setCursor(uint8_t* cursor) noexcept { _cursor = cursor; } + ASMJIT_INLINE_NODEBUG void advance(size_t n) noexcept { _cursor += n; } + + [[nodiscard]] + ASMJIT_INLINE size_t offsetFrom(uint8_t* from) const noexcept { ASMJIT_ASSERT(_cursor >= from); return (size_t)(_cursor - from); } template - ASMJIT_FORCE_INLINE void emit8(T val) noexcept { - typedef typename std::make_unsigned::type U; + ASMJIT_INLINE void emit8(T val) noexcept { + using U = std::make_unsigned_t; _cursor[0] = uint8_t(U(val) & U(0xFF)); _cursor++; } template - ASMJIT_FORCE_INLINE void emit8If(T val, Y cond) noexcept { - typedef typename std::make_unsigned::type U; + ASMJIT_INLINE void emit8If(T val, Y cond) noexcept { + using U = std::make_unsigned_t; ASMJIT_ASSERT(size_t(cond) <= 1u); _cursor[0] = uint8_t(U(val) & U(0xFF)); @@ -64,42 +68,42 @@ public: } template - ASMJIT_FORCE_INLINE void emit16uLE(T val) noexcept { - typedef typename std::make_unsigned::type U; - Support::writeU16uLE(_cursor, uint16_t(U(val) & 0xFFFFu)); + ASMJIT_INLINE void emit16uLE(T val) noexcept { + using U = std::make_unsigned_t; + Support::storeu_u16_le(_cursor, uint16_t(U(val) & 0xFFFFu)); _cursor += 2; } template - ASMJIT_FORCE_INLINE void emit16uBE(T val) noexcept { - typedef typename std::make_unsigned::type U; - Support::writeU16uBE(_cursor, uint16_t(U(val) & 0xFFFFu)); + ASMJIT_INLINE void emit16uBE(T val) noexcept { + using U = std::make_unsigned_t; + Support::storeu_u16_be(_cursor, uint16_t(U(val) & 0xFFFFu)); _cursor += 2; } template - ASMJIT_FORCE_INLINE void emit32uLE(T val) noexcept { - typedef typename std::make_unsigned::type U; - Support::writeU32uLE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + ASMJIT_INLINE void emit32uLE(T val) noexcept { + using U = std::make_unsigned_t; + Support::storeu_u32_le(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); _cursor += 4; } template - ASMJIT_FORCE_INLINE void emit32uBE(T val) noexcept { - typedef typename std::make_unsigned::type U; - Support::writeU32uBE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + ASMJIT_INLINE void emit32uBE(T val) noexcept { + using U = std::make_unsigned_t; + Support::storeu_u32_be(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); _cursor += 4; } - ASMJIT_FORCE_INLINE void emitData(const void* data, size_t size) noexcept { + ASMJIT_INLINE void emitData(const void* data, size_t size) noexcept { ASMJIT_ASSERT(size != 0); memcpy(_cursor, data, size); _cursor += size; } template - ASMJIT_FORCE_INLINE void emitValueLE(const T& value, size_t size) noexcept { - typedef typename std::make_unsigned::type U; + ASMJIT_INLINE void emitValueLE(const T& value, size_t size) noexcept { + using U = std::make_unsigned_t; ASMJIT_ASSERT(size <= sizeof(T)); U v = U(value); @@ -111,8 +115,8 @@ public: } template - ASMJIT_FORCE_INLINE void emitValueBE(const T& value, size_t size) noexcept { - typedef typename std::make_unsigned::type U; + ASMJIT_INLINE void emitValueBE(const T& value, size_t size) noexcept { + using U = std::make_unsigned_t; ASMJIT_ASSERT(size <= sizeof(T)); U v = U(value); @@ -123,13 +127,13 @@ public: _cursor += size; } - ASMJIT_FORCE_INLINE void emitZeros(size_t size) noexcept { + ASMJIT_INLINE void emitZeros(size_t size) noexcept { ASMJIT_ASSERT(size != 0); memset(_cursor, 0, size); _cursor += size; } - ASMJIT_FORCE_INLINE void remove8(uint8_t* where) noexcept { + ASMJIT_INLINE void remove8(uint8_t* where) noexcept { ASMJIT_ASSERT(where < _cursor); uint8_t* p = where; @@ -139,7 +143,7 @@ public: } template - ASMJIT_FORCE_INLINE void insert8(uint8_t* where, T val) noexcept { + ASMJIT_INLINE void insert8(uint8_t* where, T val) noexcept { uint8_t* p = _cursor; while (p != where) { @@ -151,7 +155,7 @@ public: _cursor++; } - ASMJIT_FORCE_INLINE void done(BaseAssembler* a) noexcept { + ASMJIT_INLINE void done(BaseAssembler* a) noexcept { CodeBuffer& buffer = a->_section->_buffer; size_t newSize = (size_t)(_cursor - a->_bufferData); ASMJIT_ASSERT(newSize <= buffer.capacity()); @@ -164,9 +168,13 @@ public: //! Code writer utilities. namespace CodeWriterUtils { +[[nodiscard]] bool encodeOffset32(uint32_t* dst, int64_t offset64, const OffsetFormat& format) noexcept; + +[[nodiscard]] bool encodeOffset64(uint64_t* dst, int64_t offset64, const OffsetFormat& format) noexcept; +[[nodiscard]] bool writeOffset(void* dst, int64_t offset64, const OffsetFormat& format) noexcept; } // {CodeWriterUtils} diff --git a/pe-packer/asmjit/core/compiler.cpp b/pe-packer/asmjit/core/compiler.cpp index f7cca47..82dc579 100644 --- a/pe-packer/asmjit/core/compiler.cpp +++ b/pe-packer/asmjit/core/compiler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -22,11 +22,11 @@ ASMJIT_BEGIN_NAMESPACE // =================== class GlobalConstPoolPass : public Pass { -public: - typedef Pass Base; -public: ASMJIT_NONCOPYABLE(GlobalConstPoolPass) +public: + using Base = Pass; + GlobalConstPoolPass() noexcept : Pass("GlobalConstPoolPass") {} Error run(Zone* zone, Logger* logger) override { @@ -51,7 +51,6 @@ public: BaseCompiler::BaseCompiler() noexcept : BaseBuilder(), _func(nullptr), - _vRegZone(4096 - Zone::kBlockOverhead), _vRegArray(), _constPools { nullptr, nullptr } { _emitterType = EmitterType::kCompiler; @@ -66,34 +65,38 @@ Error BaseCompiler::newFuncNode(FuncNode** out, const FuncSignature& signature) *out = nullptr; // Create FuncNode together with all the required surrounding nodes. - FuncNode* funcNode; + FuncNode* funcNode = nullptr; ASMJIT_PROPAGATE(_newNodeT(&funcNode)); ASMJIT_PROPAGATE(newLabelNode(&funcNode->_exitNode)); ASMJIT_PROPAGATE(_newNodeT(&funcNode->_end, SentinelType::kFuncEnd)); // Initialize the function's detail info. Error err = funcNode->detail().init(signature, environment()); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } // If the Target guarantees greater stack alignment than required by the calling convention // then override it as we can prevent having to perform dynamic stack alignment uint32_t environmentStackAlignment = _environment.stackAlignment(); - if (funcNode->_funcDetail._callConv.naturalStackAlignment() < environmentStackAlignment) + if (funcNode->_funcDetail._callConv.naturalStackAlignment() < environmentStackAlignment) { funcNode->_funcDetail._callConv.setNaturalStackAlignment(environmentStackAlignment); + } // Initialize the function frame. err = funcNode->_frame.init(funcNode->_funcDetail); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } // Allocate space for function arguments. funcNode->_args = nullptr; if (funcNode->argCount() != 0) { - funcNode->_args = _allocator.allocT(funcNode->argCount() * sizeof(FuncNode::ArgPack)); - if (ASMJIT_UNLIKELY(!funcNode->_args)) + funcNode->_args = _codeZone.alloc(funcNode->argCount() * sizeof(FuncNode::ArgPack)); + if (ASMJIT_UNLIKELY(!funcNode->_args)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } memset(funcNode->_args, 0, funcNode->argCount() * sizeof(FuncNode::ArgPack)); } @@ -117,7 +120,7 @@ Error BaseCompiler::addFuncNode(FuncNode** out, const FuncSignature& signature) Error BaseCompiler::newFuncRetNode(FuncRetNode** out, const Operand_& o0, const Operand_& o1) { uint32_t opCount = !o1.isNone() ? 2u : !o0.isNone() ? 1u : 0u; - FuncRetNode* node; + FuncRetNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node)); ASMJIT_ASSUME(node != nullptr); @@ -146,10 +149,10 @@ Error BaseCompiler::addFuncRetNode(FuncRetNode** out, const Operand_& o0, const FuncNode* BaseCompiler::addFunc(FuncNode* func) { _func = func; - addNode(func); // Function node. - BaseNode* prev = cursor(); // {CURSOR}. - addNode(func->exitNode()); // Function exit label. - addNode(func->endNode()); // Function end sentinel. + addNode(func); // Function node. + BaseNode* prev = cursor(); // {CURSOR}. + addNode(func->exitNode()); // Function exit label. + addNode(func->endNode()); // Function end sentinel. _setCursor(prev); return func; @@ -159,8 +162,9 @@ Error BaseCompiler::endFunc() { FuncNode* func = _func; resetState(); - if (ASMJIT_UNLIKELY(!func)) + if (ASMJIT_UNLIKELY(!func)) { return reportError(DebugUtils::errored(kErrorInvalidState)); + } // Add the local constant pool at the end of the function (if exists). ConstPoolNode* localConstPool = _constPools[uint32_t(ConstPoolScope::kLocal)]; @@ -183,7 +187,7 @@ Error BaseCompiler::endFunc() { // ================================== Error BaseCompiler::newInvokeNode(InvokeNode** out, InstId instId, const Operand_& o0, const FuncSignature& signature) { - InvokeNode* node; + InvokeNode* node = nullptr; ASMJIT_PROPAGATE(_newNodeT(&node, instId, InstOptions::kNone)); node->setOpCount(1); @@ -191,15 +195,17 @@ Error BaseCompiler::newInvokeNode(InvokeNode** out, InstId instId, const Operand node->resetOpRange(1, node->opCapacity()); Error err = node->detail().init(signature, environment()); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } // Skip the allocation if there are no arguments. uint32_t argCount = signature.argCount(); if (argCount) { - node->_args = static_cast(_allocator.alloc(argCount * sizeof(InvokeNode::OperandPack))); - if (!node->_args) + node->_args = _codeZone.alloc(argCount * sizeof(InvokeNode::OperandPack)); + if (!node->_args) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } memset(node->_args, 0, argCount * sizeof(InvokeNode::OperandPack)); } @@ -221,40 +227,31 @@ Error BaseCompiler::addInvokeNode(InvokeNode** out, InstId instId, const Operand // BaseCompiler - Virtual Registers // ================================ -static void BaseCompiler_assignGenericName(BaseCompiler* self, VirtReg* vReg) { - uint32_t index = unsigned(Operand::virtIdToIndex(vReg->_id)); - - char buf[64]; - int size = snprintf(buf, ASMJIT_ARRAY_SIZE(buf), "%%%u", unsigned(index)); - - ASMJIT_ASSERT(size > 0 && size < int(ASMJIT_ARRAY_SIZE(buf))); - vReg->_name.setData(&self->_dataZone, buf, unsigned(size)); -} - Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature signature, const char* name) { *out = nullptr; uint32_t index = _vRegArray.size(); - if (ASMJIT_UNLIKELY(index >= uint32_t(Operand::kVirtIdCount))) + if (ASMJIT_UNLIKELY(index >= uint32_t(Operand::kVirtIdCount))) { return reportError(DebugUtils::errored(kErrorTooManyVirtRegs)); + } - if (ASMJIT_UNLIKELY(_vRegArray.willGrow(&_allocator) != kErrorOk)) + if (ASMJIT_UNLIKELY(_vRegArray.willGrow(&_allocator) != kErrorOk)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } - VirtReg* vReg = _vRegZone.allocZeroedT(); - if (ASMJIT_UNLIKELY(!vReg)) + void* vRegPtr = _codeZone.alloc(Zone::alignedSizeOf()); + if (ASMJIT_UNLIKELY(!vRegPtr)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } uint32_t size = TypeUtils::sizeOf(typeId); uint32_t alignment = Support::min(size, 64); - - vReg = new(vReg) VirtReg(signature, Operand::indexToVirtId(index), size, alignment, typeId); + VirtReg* vReg = new(Support::PlacementNew{vRegPtr}) VirtReg(signature, Operand::indexToVirtId(index), size, alignment, typeId); #ifndef ASMJIT_NO_LOGGING - if (name && name[0] != '\0') - vReg->_name.setData(&_dataZone, name, SIZE_MAX); - else - BaseCompiler_assignGenericName(this, vReg); + if (name && name[0] != '\0') { + vReg->_name.setData(&_codeZone, name, SIZE_MAX); + } #else DebugUtils::unused(name); #endif @@ -265,13 +262,14 @@ Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature si return kErrorOk; } -Error BaseCompiler::_newReg(BaseReg* out, TypeId typeId, const char* name) { +Error BaseCompiler::_newReg(Reg* out, TypeId typeId, const char* name) { OperandSignature regSignature; out->reset(); Error err = ArchUtils::typeIdToRegSignature(arch(), typeId, &typeId, ®Signature); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } VirtReg* vReg; ASMJIT_PROPAGATE(newVirtReg(&vReg, typeId, regSignature, name)); @@ -281,7 +279,7 @@ Error BaseCompiler::_newReg(BaseReg* out, TypeId typeId, const char* name) { return kErrorOk; } -Error BaseCompiler::_newRegFmt(BaseReg* out, TypeId typeId, const char* fmt, ...) { +Error BaseCompiler::_newRegFmt(Reg* out, TypeId typeId, const char* fmt, ...) { va_list ap; StringTmp<256> sb; @@ -292,7 +290,7 @@ Error BaseCompiler::_newRegFmt(BaseReg* out, TypeId typeId, const char* fmt, ... return _newReg(out, typeId, sb.data()); } -Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) { +Error BaseCompiler::_newReg(Reg* out, const Reg& ref, const char* name) { out->reset(); OperandSignature regSignature; @@ -345,17 +343,19 @@ Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) } } - if (typeId == TypeId::kVoid) + if (typeId == TypeId::kVoid) { return reportError(DebugUtils::errored(kErrorInvalidState)); + } } } else { - typeId = ArchTraits::byArch(arch()).regTypeToTypeId(ref.type()); + typeId = RegUtils::typeIdOf(ref.regType()); } Error err = ArchUtils::typeIdToRegSignature(arch(), typeId, &typeId, ®Signature); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } VirtReg* vReg; ASMJIT_PROPAGATE(newVirtReg(&vReg, typeId, regSignature, name)); @@ -365,7 +365,7 @@ Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) return kErrorOk; } -Error BaseCompiler::_newRegFmt(BaseReg* out, const BaseReg& ref, const char* fmt, ...) { +Error BaseCompiler::_newRegFmt(Reg* out, const Reg& ref, const char* fmt, ...) { va_list ap; StringTmp<256> sb; @@ -379,17 +379,17 @@ Error BaseCompiler::_newRegFmt(BaseReg* out, const BaseReg& ref, const char* fmt Error BaseCompiler::_newStack(BaseMem* out, uint32_t size, uint32_t alignment, const char* name) { out->reset(); - if (size == 0) + if (ASMJIT_UNLIKELY(Support::bool_or(size == 0, !Support::isZeroOrPowerOf2(alignment)))) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } - if (alignment == 0) - alignment = 1; + if (alignment == 0u) { + alignment = 1u; + } - if (!Support::isPowerOf2(alignment)) - return reportError(DebugUtils::errored(kErrorInvalidArgument)); - - if (alignment > 64) - alignment = 64; + if (alignment > 64u) { + alignment = 64u; + } VirtReg* vReg; ASMJIT_PROPAGATE(newVirtReg(&vReg, TypeId::kVoid, OperandSignature{0}, name)); @@ -408,21 +408,26 @@ Error BaseCompiler::_newStack(BaseMem* out, uint32_t size, uint32_t alignment, c } Error BaseCompiler::setStackSize(uint32_t virtId, uint32_t newSize, uint32_t newAlignment) { - if (!isVirtIdValid(virtId)) + if (!isVirtIdValid(virtId)) { return DebugUtils::errored(kErrorInvalidVirtId); + } - if (newAlignment && !Support::isPowerOf2(newAlignment)) + if (!Support::isZeroOrPowerOf2(newAlignment)) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } - if (newAlignment > 64) - newAlignment = 64; + if (newAlignment > 64u) { + newAlignment = 64u; + } VirtReg* vReg = virtRegById(virtId); - if (newSize) + if (newSize) { vReg->_virtSize = newSize; + } - if (newAlignment) + if (newAlignment) { vReg->_alignment = uint8_t(newAlignment); + } // This is required if the RAPass is already running. There is a chance that a stack-slot has been already // allocated and in that case it has to be updated as well, otherwise we would allocate wrong amount of memory. @@ -438,18 +443,21 @@ Error BaseCompiler::setStackSize(uint32_t virtId, uint32_t newSize, uint32_t new Error BaseCompiler::_newConst(BaseMem* out, ConstPoolScope scope, const void* data, size_t size) { out->reset(); - if (uint32_t(scope) > 1) + if (uint32_t(scope) > 1) { return reportError(DebugUtils::errored(kErrorInvalidArgument)); + } - if (!_constPools[uint32_t(scope)]) + if (!_constPools[uint32_t(scope)]) { ASMJIT_PROPAGATE(newConstPoolNode(&_constPools[uint32_t(scope)])); + } ConstPoolNode* pool = _constPools[uint32_t(scope)]; size_t off; Error err = pool->add(data, size, off); - if (ASMJIT_UNLIKELY(err)) + if (ASMJIT_UNLIKELY(err)) { return reportError(err); + } *out = BaseMem(OperandSignature::fromOpType(OperandType::kMem) | OperandSignature::fromMemBaseType(RegType::kLabelTag) | @@ -458,11 +466,13 @@ Error BaseCompiler::_newConst(BaseMem* out, ConstPoolScope scope, const void* da return kErrorOk; } -void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { +void BaseCompiler::rename(const Reg& reg, const char* fmt, ...) { if (!reg.isVirtReg()) return; VirtReg* vReg = virtRegById(reg.id()); - if (!vReg) return; + if (!vReg) { + return; + } if (fmt && fmt[0] != '\0') { char buf[128]; @@ -472,10 +482,7 @@ void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf), fmt, ap); va_end(ap); - vReg->_name.setData(&_dataZone, buf, SIZE_MAX); - } - else { - BaseCompiler_assignGenericName(this, vReg); + vReg->_name.setData(&_codeZone, buf, SIZE_MAX); } } @@ -483,14 +490,15 @@ void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { // =============================== Error BaseCompiler::newJumpNode(JumpNode** out, InstId instId, InstOptions instOptions, const Operand_& o0, JumpAnnotation* annotation) { - JumpNode* node = _allocator.allocT(); - uint32_t opCount = 1; + JumpNode* node = _codeZone.alloc(); *out = node; - if (ASMJIT_UNLIKELY(!node)) + if (ASMJIT_UNLIKELY(!node)) { return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } - node = new(node) JumpNode(this, instId, instOptions, opCount, annotation); + uint32_t opCount = 1; + node = new(Support::PlacementNew{node}) JumpNode(instId, instOptions, opCount, annotation); node->setOp(0, o0); node->resetOpRange(opCount, JumpNode::kBaseOpCapacity); @@ -517,7 +525,7 @@ JumpAnnotation* BaseCompiler::newJumpAnnotation() { } uint32_t id = _jumpAnnotations.size(); - JumpAnnotation* jumpAnnotation = _allocator.newT(this, id); + JumpAnnotation* jumpAnnotation = _codeZone.newT(this, id); if (!jumpAnnotation) { reportError(DebugUtils::errored(kErrorOutOfMemory)); @@ -531,33 +539,49 @@ JumpAnnotation* BaseCompiler::newJumpAnnotation() { // BaseCompiler - Events // ===================== -Error BaseCompiler::onAttach(CodeHolder* code) noexcept { +static ASMJIT_INLINE void BaseCompiler_clear(BaseCompiler* self) noexcept { + self->_func = nullptr; + self->_constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; + self->_constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; + self->_vRegArray.reset(); +} + +static ASMJIT_INLINE Error BaseCompiler_initDefaultPasses(BaseCompiler* self) noexcept { + return self->addPassT(); +} + + +Error BaseCompiler::onAttach(CodeHolder& code) noexcept { ASMJIT_PROPAGATE(Base::onAttach(code)); - const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); - RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; - _gpSignature = archTraits.regTypeToSignature(nativeRegType); - - Error err = addPassT(); + Error err = BaseCompiler_initDefaultPasses(this); if (ASMJIT_UNLIKELY(err)) { onDetach(code); return err; } - return kErrorOk; } -Error BaseCompiler::onDetach(CodeHolder* code) noexcept { - _func = nullptr; - _constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; - _constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; - - _vRegArray.reset(); - _vRegZone.reset(); - +Error BaseCompiler::onDetach(CodeHolder& code) noexcept { + BaseCompiler_clear(this); return Base::onDetach(code); } +Error BaseCompiler::onReinit(CodeHolder& code) noexcept { + BaseCompiler_clear(this); + Error err = Base::onReinit(code); + + if (ASMJIT_LIKELY(err == kErrorOk)) { + err = BaseCompiler_initDefaultPasses(this); + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + return err; + } + } + + return err; +} + // FuncPass - Construction & Destruction // ===================================== @@ -569,9 +593,8 @@ FuncPass::FuncPass(const char* name) noexcept Error FuncPass::run(Zone* zone, Logger* logger) { BaseNode* node = cb()->firstNode(); - if (!node) return kErrorOk; - do { + while (node) { if (node->type() == NodeType::kFunc) { FuncNode* func = node->as(); node = func->endNode(); @@ -582,7 +605,7 @@ Error FuncPass::run(Zone* zone, Logger* logger) { do { node = node->next(); } while (node && node->type() != NodeType::kFunc); - } while (node); + } return kErrorOk; } diff --git a/pe-packer/asmjit/core/compiler.h b/pe-packer/asmjit/core/compiler.h index 6c32133..ed3388e 100644 --- a/pe-packer/asmjit/core/compiler.h +++ b/pe-packer/asmjit/core/compiler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_COMPILER_H_INCLUDED @@ -47,18 +47,17 @@ class InvokeNode; //! Check out architecture specific compilers for more details and examples: //! //! - \ref x86::Compiler - X86/X64 compiler implementation. +//! - \ref a64::Compiler - AArch64 compiler implementation. class ASMJIT_VIRTAPI BaseCompiler : public BaseBuilder { public: ASMJIT_NONCOPYABLE(BaseCompiler) - typedef BaseBuilder Base; + using Base = BaseBuilder; //! \name Members //! \{ //! Current function. FuncNode* _func; - //! Allocates `VirtReg` objects. - Zone _vRegZone; //! Stores array of `VirtReg` pointers. ZoneVector _vRegArray; //! Stores jump annotations. @@ -77,7 +76,7 @@ public: //! Creates a new `BaseCompiler` instance. ASMJIT_API BaseCompiler() noexcept; //! Destroys the `BaseCompiler` instance. - ASMJIT_API virtual ~BaseCompiler() noexcept; + ASMJIT_API ~BaseCompiler() noexcept override; //! \} @@ -95,6 +94,7 @@ public: ASMJIT_API Error addFuncRetNode(FuncRetNode** ASMJIT_NONNULL(out), const Operand_& o0, const Operand_& o1); //! Returns the current function. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncNode* func() const noexcept { return _func; } //! Creates a new \ref FuncNode with the given `signature` and returns it. @@ -117,18 +117,6 @@ public: //! Emits a sentinel that marks the end of the current function. ASMJIT_API Error endFunc(); -#if !defined(ASMJIT_NO_DEPRECATED) - inline Error _setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg); - - //! Sets a function argument at `argIndex` to `reg`. - ASMJIT_DEPRECATED("Setting arguments through Compiler is deprecated, use FuncNode->setArg() instead") - inline Error setArg(size_t argIndex, const BaseReg& reg) { return _setArg(argIndex, 0, reg); } - - //! Sets a function argument at `argIndex` at `valueIndex` to `reg`. - ASMJIT_DEPRECATED("Setting arguments through Compiler is deprecated, use FuncNode->setArg() instead") - inline Error setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) { return _setArg(argIndex, valueIndex, reg); } -#endif - inline Error addRet(const Operand_& o0, const Operand_& o1) { FuncRetNode* node; return addFuncRetNode(&node, o0, o1); @@ -156,47 +144,56 @@ public: ASMJIT_API Error newVirtReg(VirtReg** ASMJIT_NONNULL(out), TypeId typeId, OperandSignature signature, const char* name); //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. - ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* name = nullptr); + ASMJIT_API Error _newReg(Reg* ASMJIT_NONNULL(out), TypeId typeId, const char* name = nullptr); //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. //! //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. - ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* fmt, ...); + ASMJIT_API Error _newRegFmt(Reg* ASMJIT_NONNULL(out), TypeId typeId, const char* fmt, ...); + //! \overload + inline Error _newRegFmt(Reg* ASMJIT_NONNULL(out), TypeId typeId) { return _newRegFmt(out, typeId, nullptr); } //! Creates a new virtual register compatible with the provided reference register `ref`. - ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* name = nullptr); + ASMJIT_API Error _newReg(Reg* ASMJIT_NONNULL(out), const Reg& ref, const char* name = nullptr); //! Creates a new virtual register compatible with the provided reference register `ref`. //! //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. - ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* fmt, ...); + ASMJIT_API Error _newRegFmt(Reg* ASMJIT_NONNULL(out), const Reg& ref, const char* fmt, ...); //! Tests whether the given `id` is a valid virtual register id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isVirtIdValid(uint32_t id) const noexcept { uint32_t index = Operand::virtIdToIndex(id); return index < _vRegArray.size(); } + //! Tests whether the given `reg` is a virtual register having a valid id. - ASMJIT_INLINE_NODEBUG bool isVirtRegValid(const BaseReg& reg) const noexcept { + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isVirtRegValid(const Reg& reg) const noexcept { return isVirtIdValid(reg.id()); } //! Returns \ref VirtReg associated with the given `id`. + [[nodiscard]] inline VirtReg* virtRegById(uint32_t id) const noexcept { ASMJIT_ASSERT(isVirtIdValid(id)); return _vRegArray[Operand::virtIdToIndex(id)]; } //! Returns \ref VirtReg associated with the given `reg`. - ASMJIT_INLINE_NODEBUG VirtReg* virtRegByReg(const BaseReg& reg) const noexcept { return virtRegById(reg.id()); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG VirtReg* virtRegByReg(const Reg& reg) const noexcept { return virtRegById(reg.id()); } //! Returns \ref VirtReg associated with the given virtual register `index`. //! //! \note This is not the same as virtual register id. The conversion between id and its index is implemented //! by \ref Operand_::virtIdToIndex() and \ref Operand_::indexToVirtId() functions. + [[nodiscard]] ASMJIT_INLINE_NODEBUG VirtReg* virtRegByIndex(uint32_t index) const noexcept { return _vRegArray[index]; } //! Returns an array of all virtual registers managed by the Compiler. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& virtRegs() const noexcept { return _vRegArray; } //! \name Stack @@ -232,13 +229,14 @@ public: //! \{ //! Rename the given virtual register `reg` to a formatted string `fmt`. - ASMJIT_API void rename(const BaseReg& reg, const char* fmt, ...); + ASMJIT_API void rename(const Reg& reg, const char* fmt, ...); //! \} //! \name Jump Annotations //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& jumpAnnotations() const noexcept { return _jumpAnnotations; } @@ -248,6 +246,7 @@ public: //! Returns a new `JumpAnnotation` instance, which can be used to aggregate possible targets of a jump where the //! target is not a label, for example to implement jump tables. + [[nodiscard]] ASMJIT_API JumpAnnotation* newJumpAnnotation(); //! \} @@ -255,8 +254,9 @@ public: //! \name Events //! \{ - ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; - ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + ASMJIT_API Error onAttach(CodeHolder& code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder& code) noexcept override; + ASMJIT_API Error onReinit(CodeHolder& code) noexcept override; //! \} }; @@ -296,15 +296,23 @@ public: //! \{ //! Returns the compiler that owns this JumpAnnotation. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseCompiler* compiler() const noexcept { return _compiler; } + //! Returns the annotation id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t annotationId() const noexcept { return _annotationId; } + //! Returns a vector of label identifiers that lists all targets of the jump. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ZoneVector& labelIds() const noexcept { return _labelIds; } //! Tests whether the given `label` is a target of this JumpAnnotation. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasLabel(const Label& label) const noexcept { return hasLabelId(label.id()); } + //! Tests whether the given `labelId` is a target of this JumpAnnotation. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasLabelId(uint32_t labelId) const noexcept { return _labelIds.contains(labelId); } //! \} @@ -325,7 +333,7 @@ public: //! \note This node should be only used to represent jump where the jump target cannot be deduced by examining //! instruction operands. For example if the jump target is register or memory location. This pattern is often //! used to perform indirect jumps that use jump table, e.g. to implement `switch{}` statement. -class JumpNode : public InstNode { +class JumpNode : public InstNodeWithOperands { public: ASMJIT_NONCOPYABLE(JumpNode) @@ -339,10 +347,10 @@ public: //! \name Construction & Destruction //! \{ - inline JumpNode(BaseCompiler* ASMJIT_NONNULL(cc), InstId instId, InstOptions options, uint32_t opCount, JumpAnnotation* annotation) noexcept - : InstNode(cc, instId, options, opCount, kBaseOpCapacity), + inline JumpNode(InstId instId, InstOptions options, uint32_t opCount, JumpAnnotation* annotation) noexcept + : InstNodeWithOperands(instId, options, opCount), _annotation(annotation) { - setType(NodeType::kJump); + _setType(NodeType::kJump); } //! \} @@ -351,9 +359,13 @@ public: //! \{ //! Tests whether this JumpNode has associated a \ref JumpAnnotation. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAnnotation() const noexcept { return _annotation != nullptr; } + //! Returns the \ref JumpAnnotation associated with this jump, or `nullptr`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG JumpAnnotation* annotation() const noexcept { return _annotation; } + //! Sets the \ref JumpAnnotation associated with this jump to `annotation`. ASMJIT_INLINE_NODEBUG void setAnnotation(JumpAnnotation* annotation) noexcept { _annotation = annotation; } @@ -439,14 +451,14 @@ public: //! Creates a new `FuncNode` instance. //! //! Always use `BaseCompiler::addFunc()` to create a new `FuncNode`. - inline FuncNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept - : LabelNode(cb), + inline explicit FuncNode(uint32_t labelId = Globals::kInvalidId) noexcept + : LabelNode(labelId), _funcDetail(), _frame(), _exitNode(nullptr), _end(nullptr), _args(nullptr) { - setType(NodeType::kFunc); + _setType(NodeType::kFunc); } //! \} @@ -455,44 +467,61 @@ public: //! \name Accessors //! Returns function exit `LabelNode`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG LabelNode* exitNode() const noexcept { return _exitNode; } + //! Returns function exit label. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label exitLabel() const noexcept { return _exitNode->label(); } //! Returns "End of Func" sentinel node. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SentinelNode* endNode() const noexcept { return _end; } //! Returns function detail. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncDetail& detail() noexcept { return _funcDetail; } + //! Returns function detail. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncDetail& detail() const noexcept { return _funcDetail; } //! Returns function frame. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncFrame& frame() noexcept { return _frame; } + //! Returns function frame. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncFrame& frame() const noexcept { return _frame; } //! Returns function attributes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncAttributes attributes() const noexcept { return _frame.attributes(); } + //! Adds `attrs` to the function attributes. ASMJIT_INLINE_NODEBUG void addAttributes(FuncAttributes attrs) noexcept { _frame.addAttributes(attrs); } //! Returns arguments count. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t argCount() const noexcept { return _funcDetail.argCount(); } + //! Returns argument packs. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ArgPack* argPacks() const noexcept { return _args; } //! Tests whether the function has a return value. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRet() const noexcept { return _funcDetail.hasRet(); } //! Returns argument pack at `argIndex`. + [[nodiscard]] inline ArgPack& argPack(size_t argIndex) const noexcept { ASMJIT_ASSERT(argIndex < argCount()); return _args[argIndex]; } //! Sets argument at `argIndex`. - inline void setArg(size_t argIndex, const BaseReg& vReg) noexcept { + inline void setArg(size_t argIndex, const Reg& vReg) noexcept { ASMJIT_ASSERT(argIndex < argCount()); _args[argIndex][0].init(vReg); } @@ -504,7 +533,7 @@ public: } //! Sets argument at `argIndex` and `valueIndex`. - inline void setArg(size_t argIndex, size_t valueIndex, const BaseReg& vReg) noexcept { + inline void setArg(size_t argIndex, size_t valueIndex, const Reg& vReg) noexcept { ASMJIT_ASSERT(argIndex < argCount()); _args[argIndex][valueIndex].init(vReg); } @@ -531,7 +560,7 @@ public: }; //! Function return, used by \ref BaseCompiler. -class FuncRetNode : public InstNode { +class FuncRetNode : public InstNodeWithOperands { public: ASMJIT_NONCOPYABLE(FuncRetNode) @@ -539,20 +568,21 @@ public: //! \{ //! Creates a new `FuncRetNode` instance. - inline FuncRetNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept : InstNode(cb, BaseInst::kIdAbstract, InstOptions::kNone, 0) { - _any._nodeType = NodeType::kFuncRet; + inline FuncRetNode() noexcept + : InstNodeWithOperands(BaseInst::kIdAbstract, InstOptions::kNone, 0) { + _nodeType = NodeType::kFuncRet; } //! \} }; //! Function invocation, used by \ref BaseCompiler. -class InvokeNode : public InstNode { +class InvokeNode : public InstNodeWithOperands { public: ASMJIT_NONCOPYABLE(InvokeNode) //! Operand pack provides multiple operands that can be associated with a single return value of function - //! argument. Sometims this is necessary to express an argument or return value that requires multiple + //! argument. Sometimes this is necessary to express an argument or return value that requires multiple //! registers, for example 64-bit value in 32-bit mode or passing / returning homogeneous data structures. struct OperandPack { //! Operands. @@ -565,12 +595,14 @@ public: } //! Returns an operand at the given `valueIndex`. + [[nodiscard]] inline Operand& operator[](size_t valueIndex) noexcept { ASMJIT_ASSERT(valueIndex < Globals::kMaxValuePack); return _data[valueIndex].as(); } //! Returns an operand at the given `valueIndex` (const). + [[nodiscard]] const inline Operand& operator[](size_t valueIndex) const noexcept { ASMJIT_ASSERT(valueIndex < Globals::kMaxValuePack); return _data[valueIndex].as(); @@ -593,14 +625,14 @@ public: //! \{ //! Creates a new `InvokeNode` instance. - inline InvokeNode(BaseBuilder* ASMJIT_NONNULL(cb), InstId instId, InstOptions options) noexcept - : InstNode(cb, instId, options, kBaseOpCapacity), + inline InvokeNode(InstId instId, InstOptions options) noexcept + : InstNodeWithOperands(instId, options, 0), _funcDetail(), _args(nullptr) { - setType(NodeType::kInvoke); + _setType(NodeType::kInvoke); _resetOps(); _rets.reset(); - addFlags(NodeFlags::kIsRemovable); + _addFlags(NodeFlags::kIsRemovable); } //! \} @@ -609,52 +641,74 @@ public: //! \{ //! Sets the function signature. + [[nodiscard]] inline Error init(const FuncSignature& signature, const Environment& environment) noexcept { return _funcDetail.init(signature, environment); } //! Returns the function detail. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncDetail& detail() noexcept { return _funcDetail; } + //! Returns the function detail. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncDetail& detail() const noexcept { return _funcDetail; } //! Returns the target operand. - ASMJIT_INLINE_NODEBUG Operand& target() noexcept { return _opArray[0].as(); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG Operand& target() noexcept { return op(0); } + //! \overload - ASMJIT_INLINE_NODEBUG const Operand& target() const noexcept { return _opArray[0].as(); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const Operand& target() const noexcept { return op(0); } //! Returns the number of function return values. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRet() const noexcept { return _funcDetail.hasRet(); } + //! Returns the number of function arguments. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t argCount() const noexcept { return _funcDetail.argCount(); } //! Returns operand pack representing function return value(s). + [[nodiscard]] ASMJIT_INLINE_NODEBUG OperandPack& retPack() noexcept { return _rets; } + //! Returns operand pack representing function return value(s). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const OperandPack& retPack() const noexcept { return _rets; } //! Returns the return value at the given `valueIndex`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Operand& ret(size_t valueIndex = 0) noexcept { return _rets[valueIndex]; } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const Operand& ret(size_t valueIndex = 0) const noexcept { return _rets[valueIndex]; } //! Returns operand pack representing function return value(s). + [[nodiscard]] inline OperandPack& argPack(size_t argIndex) noexcept { ASMJIT_ASSERT(argIndex < argCount()); return _args[argIndex]; } + //! \overload + [[nodiscard]] inline const OperandPack& argPack(size_t argIndex) const noexcept { ASMJIT_ASSERT(argIndex < argCount()); return _args[argIndex]; } //! Returns a function argument at the given `argIndex`. + [[nodiscard]] inline Operand& arg(size_t argIndex, size_t valueIndex) noexcept { ASMJIT_ASSERT(argIndex < argCount()); return _args[argIndex][valueIndex]; } + //! \overload + [[nodiscard]] inline const Operand& arg(size_t argIndex, size_t valueIndex) const noexcept { ASMJIT_ASSERT(argIndex < argCount()); return _args[argIndex][valueIndex]; @@ -669,15 +723,15 @@ public: } //! Sets the function return value at `valueIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setRet(size_t valueIndex, const BaseReg& reg) noexcept { _setRet(valueIndex, reg); } + ASMJIT_INLINE_NODEBUG void setRet(size_t valueIndex, const Reg& reg) noexcept { _setRet(valueIndex, reg); } //! Sets the first function argument in a value-pack at `argIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const BaseReg& reg) noexcept { _setArg(argIndex, 0, reg); } + ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const Reg& reg) noexcept { _setArg(argIndex, 0, reg); } //! Sets the first function argument in a value-pack at `argIndex` to `imm`. ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, const Imm& imm) noexcept { _setArg(argIndex, 0, imm); } //! Sets the function argument at `argIndex` and `valueIndex` to `reg`. - ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) noexcept { _setArg(argIndex, valueIndex, reg); } + ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const Reg& reg) noexcept { _setArg(argIndex, valueIndex, reg); } //! Sets the function argument at `argIndex` and `valueIndex` to `imm`. ASMJIT_INLINE_NODEBUG void setArg(size_t argIndex, size_t valueIndex, const Imm& imm) noexcept { _setArg(argIndex, valueIndex, imm); } @@ -688,7 +742,7 @@ public: class ASMJIT_VIRTAPI FuncPass : public Pass { public: ASMJIT_NONCOPYABLE(FuncPass) - typedef Pass Base; + using Base = Pass; //! \name Construction & Destruction //! \{ @@ -701,6 +755,7 @@ public: //! \{ //! Returns the associated `BaseCompiler`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseCompiler* cc() const noexcept { return static_cast(_cb); } //! \} @@ -712,23 +767,11 @@ public: ASMJIT_API Error run(Zone* zone, Logger* logger) override; //! Called once per `FuncNode`. - ASMJIT_API virtual Error runOnFunction(Zone* zone, Logger* logger, FuncNode* func) = 0; + ASMJIT_API virtual Error runOnFunction(Zone* zone, Logger* logger, FuncNode* func); //! \} }; -#if !defined(ASMJIT_NO_DEPRECATED) -inline Error BaseCompiler::_setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) { - FuncNode* func = _func; - - if (ASMJIT_UNLIKELY(!func)) - return reportError(DebugUtils::errored(kErrorInvalidState)); - - func->setArg(argIndex, valueIndex, reg); - return kErrorOk; -} -#endif - //! \} ASMJIT_END_NAMESPACE diff --git a/pe-packer/asmjit/core/compilerdefs.h b/pe-packer/asmjit/core/compilerdefs.h index e2e74ce..14b7346 100644 --- a/pe-packer/asmjit/core/compilerdefs.h +++ b/pe-packer/asmjit/core/compilerdefs.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_COMPILERDEFS_H_INCLUDED @@ -18,7 +18,18 @@ class RAWorkReg; //! \addtogroup asmjit_compiler //! \{ -//! Virtual register data, managed by \ref BaseCompiler. +//! Public virtual register interface, managed by \ref BaseCompiler. +//! +//! When a virtual register is created by \ref BaseCompiler a `VirtReg` is linked with the register operand id it +//! returns. This `VirtReg` can be accessed via \ref BaseCompiler::virtRegByReg() function, which returns a pointer +//! to `VirtReg`. +//! +//! In general, `VirtReg` should be only introspected as it contains important variables that are needed and managed +//! by AsmJit, however, the `VirtReg` API can also be used to influence register allocation. For example there is +//! a \ref VirtReg::setWeight() function, which could be used to increase a weight of a virtual register (thus make +//! it hard to spill, for example). In addition, there is a \ref VirtReg::setHomeIdHint() function, which can be used +//! to do an initial assignment of a physical register of a virtual register. However, AsmJit could still override +//! the physical register assigned in some special cases. class VirtReg { public: ASMJIT_NONCOPYABLE(VirtReg) @@ -45,6 +56,8 @@ public: //! True if this virtual register has assigned stack offset (can be only valid after register allocation pass). uint8_t _hasStackSlot : 1; uint8_t _reservedBits : 5; + //! Home register hint for the register allocator (initially unassigned). + uint8_t _homeIdHint = Reg::kIdBad; //! Stack offset assigned by the register allocator relative to stack pointer (can be negative as well). int32_t _stackOffset = 0; @@ -52,7 +65,7 @@ public: //! Reserved for future use (padding). uint32_t _reservedU32 = 0; - //! Virtual register name (user provided or automatically generated). + //! Virtual register name (either empty or user provided). ZoneString<16> _name {}; // The following members are used exclusively by RAPass. They are initialized when the VirtReg is created to @@ -83,18 +96,27 @@ public: //! \{ //! Returns the virtual register id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t id() const noexcept { return _id; } //! Returns the virtual register name. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* name() const noexcept { return _name.data(); } + //! Returns the size of the virtual register name. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t nameSize() const noexcept { return _name.size(); } //! Returns a register signature of this virtual register. + [[nodiscard]] ASMJIT_INLINE_NODEBUG OperandSignature signature() const noexcept { return _signature; } + //! Returns a virtual register type (maps to the physical register type as well). + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegType type() const noexcept { return _signature.regType(); } + //! Returns a virtual register group (maps to the physical register group as well). + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegGroup group() const noexcept { return _signature.regGroup(); } //! Returns a real size of the register this virtual register maps to. @@ -102,23 +124,29 @@ public: //! For example if this is a 128-bit SIMD register used for a scalar single precision floating point value then //! its virtSize would be 4, however, the `regSize` would still say 16 (128-bits), because it's the smallest size //! of that register type. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t regSize() const noexcept { return _signature.size(); } //! Returns the virtual register size. //! //! The virtual register size describes how many bytes the virtual register needs to store its content. It can be //! smaller than the physical register size, see `regSize()`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t virtSize() const noexcept { return _virtSize; } //! Returns the virtual register alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t alignment() const noexcept { return _alignment; } //! Returns the virtual register type id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG TypeId typeId() const noexcept { return _typeId; } //! Returns the virtual register weight - the register allocator can use it as explicit hint for alloc/spill //! decisions. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t weight() const noexcept { return _weight; } + //! Sets the virtual register weight (0 to 255) - the register allocator can use it as explicit hint for //! alloc/spill decisions and initial bin-packing. ASMJIT_INLINE_NODEBUG void setWeight(uint32_t weight) noexcept { _weight = uint8_t(weight); } @@ -126,17 +154,20 @@ public: //! Returns whether the virtual register is always allocated to a fixed physical register (and never reallocated). //! //! \note This is only used for special purposes and it's mostly internal. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFixed() const noexcept { return bool(_isFixed); } //! Tests whether the virtual register is in fact a stack that only uses the virtual register id. //! //! \note It's an error if a stack is accessed as a register. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isStack() const noexcept { return bool(_isStack); } //! Tests whether this virtual register (or stack) has assigned a stack offset. //! //! If this is a virtual register that was never allocated on stack, it would return false, otherwise if //! it's a virtual register that was spilled or explicitly allocated stack, the return value would be true. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasStackSlot() const noexcept { return bool(_hasStackSlot); } //! Assigns a stack offset of this virtual register to `stackOffset` and sets `_hasStackSlot` to true. @@ -145,18 +176,37 @@ public: _stackOffset = stackOffset; } + //! Tests whether this virtual register has assigned a physical register as a hint to the register allocator. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasHomeIdHint() const noexcept { return _homeIdHint != Reg::kIdBad; } + + //! Returns a physical register hint, which will be used by the register allocator. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t homeIdHint() const noexcept { return _homeIdHint; } + + //! Assigns a physical register hint, which will be used by the register allocator. + ASMJIT_INLINE_NODEBUG void setHomeIdHint(uint32_t homeId) noexcept { _homeIdHint = uint8_t(homeId); } + //! Resets a physical register hint. + ASMJIT_INLINE_NODEBUG void resetHomeIdHint() noexcept { _homeIdHint = Reg::kIdBad; } + //! Returns a stack offset associated with a virtual register or explicit stack allocation. //! //! \note Always verify that the stack offset has been assigned by calling \ref hasStackSlot(). The return //! value will be zero when the stack offset was not assigned. + [[nodiscard]] ASMJIT_INLINE_NODEBUG int32_t stackOffset() const noexcept { return _stackOffset; } //! Tests whether the virtual register has an associated `RAWorkReg` at the moment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasWorkReg() const noexcept { return _workReg != nullptr; } + //! Returns an associated RAWorkReg with this virtual register (only valid during register allocation). + [[nodiscard]] ASMJIT_INLINE_NODEBUG RAWorkReg* workReg() const noexcept { return _workReg; } + //! Associates a RAWorkReg with this virtual register (used by register allocator). ASMJIT_INLINE_NODEBUG void setWorkReg(RAWorkReg* workReg) noexcept { _workReg = workReg; } + //! Reset the RAWorkReg association (used by register allocator). ASMJIT_INLINE_NODEBUG void resetWorkReg() noexcept { _workReg = nullptr; } diff --git a/pe-packer/asmjit/core/constpool.cpp b/pe-packer/asmjit/core/constpool.cpp index 2ac5edf..8531e11 100644 --- a/pe-packer/asmjit/core/constpool.cpp +++ b/pe-packer/asmjit/core/constpool.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -40,8 +40,10 @@ void ConstPool::reset(Zone* zone) noexcept { static inline ConstPool::Gap* ConstPool_allocGap(ConstPool* self) noexcept { ConstPool::Gap* gap = self->_gapPool; - if (!gap) - return self->_zone->allocT(); + + if (!gap) { + return self->_zone->alloc(); + } self->_gapPool = gap->_next; return gap; @@ -87,8 +89,9 @@ static void ConstPool_addGap(ConstPool* self, size_t offset, size_t size) noexce // We don't have to check for errors here, if this failed nothing really happened (just the gap won't be // visible) and it will fail again at place where the same check would generate `kErrorOutOfMemory` error. ConstPool::Gap* gap = ConstPool_allocGap(self); - if (!gap) + if (!gap) { return; + } gap->_next = self->_gaps[gapIndex]; self->_gaps[gapIndex] = gap; @@ -102,24 +105,19 @@ static void ConstPool_addGap(ConstPool* self, size_t offset, size_t size) noexce } Error ConstPool::add(const void* data, size_t size, size_t& dstOffset) noexcept { - size_t treeIndex; + constexpr size_t kMaxSize = size_t(1) << (kIndexCount - 1); - if (size == 64) - treeIndex = kIndex64; - else if (size == 32) - treeIndex = kIndex32; - else if (size == 16) - treeIndex = kIndex16; - else if (size == 8) - treeIndex = kIndex8; - else if (size == 4) - treeIndex = kIndex4; - else if (size == 2) - treeIndex = kIndex2; - else if (size == 1) - treeIndex = kIndex1; - else + // Avoid sizes outside of the supported range. + if (ASMJIT_UNLIKELY(size == 0 || size > kMaxSize)) { return DebugUtils::errored(kErrorInvalidArgument); + } + + size_t treeIndex = Support::ctz(size); + + // Avoid sizes, which are not aligned to power of 2. + if (ASMJIT_UNLIKELY((size_t(1) << treeIndex) != size)) { + return DebugUtils::errored(kErrorInvalidArgument); + } ConstPool::Node* node = _tree[treeIndex].get(data); if (node) { @@ -147,8 +145,9 @@ Error ConstPool::add(const void* data, size_t size, size_t& dstOffset) noexcept ASMJIT_ASSERT(Support::isAligned(offset, size)); gapSize -= size; - if (gapSize > 0) + if (gapSize > 0) { ConstPool_addGap(this, gapOffset, gapSize); + } } gapIndex++; @@ -169,8 +168,9 @@ Error ConstPool::add(const void* data, size_t size, size_t& dstOffset) noexcept // Add the initial node to the right index. node = ConstPool::Tree::_newNode(_zone, data, size, offset, false); - if (ASMJIT_UNLIKELY(!node)) + if (ASMJIT_UNLIKELY(!node)) { return DebugUtils::errored(kErrorOutOfMemory); + } _tree[treeIndex].insert(node); _alignment = Support::max(_alignment, size); @@ -192,18 +192,16 @@ Error ConstPool::add(const void* data, size_t size, size_t& dstOffset) noexcept const uint8_t* pData = static_cast(data); for (size_t i = 0; i < pCount; i++, pData += smallerSize) { node = _tree[treeIndex].get(pData); - if (node) continue; + if (node) { + continue; + } node = ConstPool::Tree::_newNode(_zone, pData, smallerSize, offset + (i * smallerSize), true); _tree[treeIndex].insert(node); } } - if (_minItemSize == 0) - _minItemSize = size; - else - _minItemSize = Support::min(_minItemSize, size); - + _minItemSize = !_minItemSize ? size : Support::min(_minItemSize, size); return kErrorOk; } @@ -216,8 +214,9 @@ struct ConstPoolFill { _dataSize(dataSize) {} inline void operator()(const ConstPool::Node* node) noexcept { - if (!node->_shared) + if (!node->_shared) { memcpy(_dst + node->_offset, node->data(), _dataSize); + } } uint8_t* _dst; @@ -240,7 +239,7 @@ void ConstPool::fill(void* dst) const noexcept { #if defined(ASMJIT_TEST) UNIT(const_pool) { - Zone zone(32384 - Zone::kBlockOverhead); + Zone zone(32u * 1024u); ConstPool pool(&zone); uint32_t i; diff --git a/pe-packer/asmjit/core/constpool.h b/pe-packer/asmjit/core/constpool.h index 036b80f..9cd838b 100644 --- a/pe-packer/asmjit/core/constpool.h +++ b/pe-packer/asmjit/core/constpool.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CONSTPOOL_H_INCLUDED @@ -27,6 +27,9 @@ enum class ConstPoolScope : uint32_t { }; //! Constant pool. +//! +//! Constant pool is designed to hold 1, 2, 4, 8, 16, 32, and 64 byte constants. It's not designed to hold constants +//! having arbitrary length like strings and arrays. class ConstPool { public: ASMJIT_NONCOPYABLE(ConstPool) @@ -70,9 +73,11 @@ public: _shared(shared), _offset(uint32_t(offset)) {} - ASMJIT_INLINE_NODEBUG void* data() const noexcept { - return static_cast(const_cast(this) + 1); - } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG void* data() noexcept { return Support::offsetPtr(this, sizeof(*this)); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const void* data() const noexcept { return Support::offsetPtr(this, sizeof(*this)); } }; //! Data comparer used internally. @@ -83,10 +88,12 @@ public: ASMJIT_INLINE_NODEBUG Compare(size_t dataSize) noexcept : _dataSize(dataSize) {} + [[nodiscard]] ASMJIT_INLINE_NODEBUG int operator()(const Node& a, const Node& b) const noexcept { return ::memcmp(a.data(), b.data(), _dataSize); } + [[nodiscard]] ASMJIT_INLINE_NODEBUG int operator()(const Node& a, const void* data) const noexcept { return ::memcmp(a.data(), data, _dataSize); } @@ -111,7 +118,10 @@ public: _size = 0; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return _size == 0; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t size() const noexcept { return _size; } inline void setDataSize(size_t dataSize) noexcept { @@ -119,6 +129,7 @@ public: _dataSize = dataSize; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG Node* get(const void* data) noexcept { Compare cmp(_dataSize); return _tree.get(data, cmp); @@ -163,11 +174,16 @@ public: } } + [[nodiscard]] static inline Node* _newNode(Zone* zone, const void* data, size_t size, size_t offset, bool shared) noexcept { - Node* node = zone->allocT(sizeof(Node) + size); - if (ASMJIT_UNLIKELY(!node)) return nullptr; + size_t nodeSize = Support::alignUp(sizeof(Node) + size, Globals::kZoneAlignment); + Node* node = zone->alloc(nodeSize); - node = new(node) Node(offset, shared); + if (ASMJIT_UNLIKELY(!node)) { + return nullptr; + } + + node = new(Support::PlacementNew{node}) Node(offset, shared); memcpy(node->data(), data, size); return node; } @@ -199,9 +215,17 @@ public: //! \name Construction & Destruction //! \{ - ASMJIT_API ConstPool(Zone* zone) noexcept; + //! Creates a new constant pool that would use `zone` as a memory allocator. + ASMJIT_API explicit ConstPool(Zone* zone) noexcept; + //! Destroys this constant pool. ASMJIT_API ~ConstPool() noexcept; + //! \} + + //! \name Reset + //! \{ + + //! Resets this constant pool and its allocator to `zone`. ASMJIT_API void reset(Zone* zone) noexcept; //! \} @@ -210,12 +234,19 @@ public: //! \{ //! Tests whether the constant-pool is empty. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return _size == 0; } + //! Returns the size of the constant-pool in bytes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t size() const noexcept { return _size; } + //! Returns minimum alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t alignment() const noexcept { return _alignment; } + //! Returns the minimum size of all items added to the constant pool. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t minItemSize() const noexcept { return _minItemSize; } //! \} diff --git a/pe-packer/asmjit/core/cpuinfo.cpp b/pe-packer/asmjit/core/cpuinfo.cpp index 41eb3db..ba5f319 100644 --- a/pe-packer/asmjit/core/cpuinfo.cpp +++ b/pe-packer/asmjit/core/cpuinfo.cpp @@ -1,12 +1,14 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #include "../core/cpuinfo.h" #include "../core/support.h" +#include + // Required by `__cpuidex()` and `_xgetbv()`. #if ASMJIT_ARCH_X86 #if defined(_MSC_VER) @@ -19,7 +21,6 @@ #if !defined(_WIN32) #include #include - #include #endif //! Required to detect CPU and features on Apple platforms. @@ -43,11 +44,20 @@ #endif #if ASMJIT_ARCH_ARM >= 64 && defined(__OpenBSD__) - #include #include + #include #endif + + #if ASMJIT_ARCH_ARM >= 64 && defined(__NetBSD__) + #include + #endif + #endif // ASMJIT_ARCH_ARM +#if !defined(_WIN32) && (ASMJIT_ARCH_X86 || ASMJIT_ARCH_ARM) + #include +#endif + ASMJIT_BEGIN_NAMESPACE // CpuInfo - Detect - Compatibility @@ -56,7 +66,7 @@ ASMJIT_BEGIN_NAMESPACE // CPU features detection is a minefield on non-X86 platforms. The following list describes which // operating systems and architectures are supported and the status of the implementation: // -// * X86, X86_64: +// * X86|X86_64: // - All OSes supported // - Detection is based on using a CPUID instruction, which is a user-space instruction, so there // is no need to use any OS specific APIs or syscalls to detect all features provided by the CPU. @@ -73,7 +83,7 @@ ASMJIT_BEGIN_NAMESPACE // * ARM64: // - Linux - HWCAPS and CPUID based detection. // - FreeBSD - HWCAPS and CPUID based detection (shared with Linux code). -// - NetBSD - NOT IMPLEMENTED! +// - NetBSD - CPUID based detection (reading CPUID via sysctl's cpu0 info) // - OpenBSD - CPUID based detection (reading CPUID via sysctl's CTL_MACHDEP). // - Apple - sysctlbyname() based detection with FamilyId matrix (record for each family id). // - Windows - IsProcessorFeaturePresent() based detection (only detects a subset of features). @@ -111,7 +121,7 @@ static inline uint32_t detectHWThreadCount() noexcept { namespace x86 { -typedef CpuFeatures::X86 Ext; +using Ext = CpuFeatures::X86; struct cpuid_t { uint32_t eax, ebx, ecx, edx; }; struct xgetbv_t { uint32_t eax, edx; }; @@ -173,9 +183,11 @@ static inline void simplifyCpuVendor(CpuInfo& cpu, uint32_t d0, uint32_t d1, uin }; uint32_t i; - for (i = 0; i < ASMJIT_ARRAY_SIZE(table) - 1; i++) - if (table[i].d[0] == d0 && table[i].d[1] == d1 && table[i].d[2] == d2) + for (i = 0; i < ASMJIT_ARRAY_SIZE(table) - 1; i++) { + if (table[i].d[0] == d0 && table[i].d[1] == d1 && table[i].d[2] == d2) { break; + } + } memcpy(cpu._vendor.str, table[i].normalized, 8); } @@ -190,10 +202,11 @@ static ASMJIT_FAVOR_SIZE void simplifyCpuBrand(char* s) noexcept { s[0] = '\0'; for (;;) { - if (!c) + if (!c) { break; + } - if (!(c == ' ' && (prev == '@' || s[1] == ' ' || s[1] == '@'))) { + if (!(c == ' ' && (prev == '@' || s[1] == ' ' || s[1] == '@' || s[1] == '\0'))) { *d++ = c; prev = c; } @@ -218,8 +231,8 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { // We are gonna execute CPUID, which was introduced by I486, so it's the requirement. features.add(Ext::kI486); - // CPUID EAX=0 - // ----------- + // CPUID EAX=0x00 (Basic CPUID Information) + // ---------------------------------------- // Get vendor string/id. cpuidQuery(®s, 0x0); @@ -229,10 +242,10 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { simplifyCpuVendor(cpu, regs.ebx, regs.edx, regs.ecx); - // CPUID EAX=1 - // ----------- + // CPUID EAX=0x01 (Basic CPUID Information) + // ---------------------------------------- - if (maxId >= 0x1) { + if (maxId >= 0x01u) { // Get feature flags in ECX/EDX and family/model in EAX. cpuidQuery(®s, 0x1); @@ -241,11 +254,13 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { uint32_t familyId = (regs.eax >> 8) & 0x0F; // Use extended family and model fields. - if (familyId == 0x06u || familyId == 0x0Fu) + if (familyId == 0x06u || familyId == 0x0Fu) { modelId += (((regs.eax >> 16) & 0x0Fu) << 4); + } - if (familyId == 0x0Fu) + if (familyId == 0x0Fu) { familyId += ((regs.eax >> 20) & 0xFFu); + } cpu._modelId = modelId; cpu._familyId = familyId; @@ -300,39 +315,35 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { } constexpr uint32_t kXCR0_AMX_Bits = 0x3u << 17; - bool amxEnabledByOS = (xcr0.eax & kXCR0_AMX_Bits) == kXCR0_AMX_Bits; + bool amxEnabled = (xcr0.eax & kXCR0_AMX_Bits) == kXCR0_AMX_Bits; #if defined(__APPLE__) // Apple platform provides on-demand AVX512 support. When an AVX512 instruction is used the first time it results // in #UD, which would cause the thread being promoted to use AVX512 support by the OS in addition to enabling the // necessary bits in XCR0 register. - bool avx512EnabledByOS = true; + bool avx512Enabled = true; #else // - XCR0[2:1] == 11b - XMM/YMM states need to be enabled by OS. // - XCR0[7:5] == 111b - Upper 256-bit of ZMM0-XMM15 and ZMM16-ZMM31 need to be enabled by OS. constexpr uint32_t kXCR0_AVX512_Bits = (0x3u << 1) | (0x7u << 5); - bool avx512EnabledByOS = (xcr0.eax & kXCR0_AVX512_Bits) == kXCR0_AVX512_Bits; + bool avx512Enabled = (xcr0.eax & kXCR0_AVX512_Bits) == kXCR0_AVX512_Bits; #endif - // CPUID EAX=7 ECX=0 - // ----------------- + bool avx10Enabled = false; - // Detect new features if the processor supports CPUID-07. - bool maybeMPX = false; + // CPUID EAX=0x07 ECX=0 (Structured Extended Feature Flags Enumeration Leaf) + // ------------------------------------------------------------------------- - if (maxId >= 0x7) { + if (maxId >= 0x07u) { cpuidQuery(®s, 0x7); - maybeMPX = bitTest(regs.ebx, 14); maxSubLeafId_0x7 = regs.eax; features.addIf(bitTest(regs.ebx, 0), Ext::kFSGSBASE); features.addIf(bitTest(regs.ebx, 3), Ext::kBMI); - features.addIf(bitTest(regs.ebx, 4), Ext::kHLE); features.addIf(bitTest(regs.ebx, 7), Ext::kSMEP); features.addIf(bitTest(regs.ebx, 8), Ext::kBMI2); features.addIf(bitTest(regs.ebx, 9), Ext::kERMS); - features.addIf(bitTest(regs.ebx, 11), Ext::kRTM); features.addIf(bitTest(regs.ebx, 18), Ext::kRDSEED); features.addIf(bitTest(regs.ebx, 19), Ext::kADX); features.addIf(bitTest(regs.ebx, 20), Ext::kSMAP); @@ -347,6 +358,7 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.ecx, 9), Ext::kVAES); features.addIf(bitTest(regs.ecx, 10), Ext::kVPCLMULQDQ); features.addIf(bitTest(regs.ecx, 22), Ext::kRDPID); + features.addIf(bitTest(regs.ecx, 23), Ext::kKL); features.addIf(bitTest(regs.ecx, 25), Ext::kCLDEMOTE); features.addIf(bitTest(regs.ecx, 27), Ext::kMOVDIRI); features.addIf(bitTest(regs.ecx, 28), Ext::kMOVDIR64B); @@ -358,22 +370,15 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.edx, 18), Ext::kPCONFIG); features.addIf(bitTest(regs.edx, 20), Ext::kCET_IBT); - // Detect 'TSX' - Requires at least one of `HLE` and `RTM` features. - if (features.hasHLE() || features.hasRTM()) { - features.add(Ext::kTSX); - } - if (bitTest(regs.ebx, 5) && features.hasAVX()) { features.add(Ext::kAVX2); } - if (avx512EnabledByOS && bitTest(regs.ebx, 16)) { + if (avx512Enabled && bitTest(regs.ebx, 16)) { features.add(Ext::kAVX512_F); features.addIf(bitTest(regs.ebx, 17), Ext::kAVX512_DQ); features.addIf(bitTest(regs.ebx, 21), Ext::kAVX512_IFMA); - features.addIf(bitTest(regs.ebx, 26), Ext::kAVX512_PF); - features.addIf(bitTest(regs.ebx, 27), Ext::kAVX512_ER); features.addIf(bitTest(regs.ebx, 28), Ext::kAVX512_CD); features.addIf(bitTest(regs.ebx, 30), Ext::kAVX512_BW); features.addIf(bitTest(regs.ebx, 31), Ext::kAVX512_VL); @@ -382,21 +387,19 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.ecx, 11), Ext::kAVX512_VNNI); features.addIf(bitTest(regs.ecx, 12), Ext::kAVX512_BITALG); features.addIf(bitTest(regs.ecx, 14), Ext::kAVX512_VPOPCNTDQ); - features.addIf(bitTest(regs.edx, 2), Ext::kAVX512_4VNNIW); - features.addIf(bitTest(regs.edx, 3), Ext::kAVX512_4FMAPS); features.addIf(bitTest(regs.edx, 8), Ext::kAVX512_VP2INTERSECT); features.addIf(bitTest(regs.edx, 23), Ext::kAVX512_FP16); } - if (amxEnabledByOS) { + if (amxEnabled) { features.addIf(bitTest(regs.edx, 22), Ext::kAMX_BF16); features.addIf(bitTest(regs.edx, 24), Ext::kAMX_TILE); features.addIf(bitTest(regs.edx, 25), Ext::kAMX_INT8); } } - // CPUID EAX=7 ECX=1 - // ----------------- + // CPUID EAX=0x07 ECX=1 (Structured Extended Feature Enumeration Sub-leaf) + // ----------------------------------------------------------------------- if (maxSubLeafId_0x7 >= 1) { cpuidQuery(®s, 0x7, 1); @@ -413,6 +416,8 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.eax, 22), Ext::kHRESET); features.addIf(bitTest(regs.eax, 26), Ext::kLAM); features.addIf(bitTest(regs.eax, 27), Ext::kMSRLIST); + features.addIf(bitTest(regs.eax, 31), Ext::kMOVRS); + features.addIf(bitTest(regs.ecx, 5), Ext::kMSR_IMM); features.addIf(bitTest(regs.ebx, 1), Ext::kTSE); features.addIf(bitTest(regs.edx, 14), Ext::kPREFETCHI); features.addIf(bitTest(regs.edx, 18), Ext::kCET_SSS); @@ -430,22 +435,20 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.eax, 5), Ext::kAVX512_BF16); } - if (amxEnabledByOS) { + if (features.hasAVX512_F()) { + avx10Enabled = Support::bitTest(regs.edx, 19); + } + + if (amxEnabled) { features.addIf(bitTest(regs.eax, 21), Ext::kAMX_FP16); features.addIf(bitTest(regs.edx, 8), Ext::kAMX_COMPLEX); } } - // CPUID EAX=13 ECX=0 - // ------------------ - - if (maxId >= 0xD) { - cpuidQuery(®s, 0xD, 0); - - // Both CPUID result and XCR0 has to be enabled to have support for MPX. - if (((regs.eax & xcr0.eax) & 0x00000018u) == 0x00000018u && maybeMPX) - features.add(Ext::kMPX); + // CPUID EAX=0x0D ECX=1 (Processor Extended State Enumeration Sub-leaf) + // -------------------------------------------------------------------- + if (maxId >= 0x0Du) { cpuidQuery(®s, 0xD, 1); features.addIf(bitTest(regs.eax, 0), Ext::kXSAVEOPT); @@ -453,15 +456,57 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { features.addIf(bitTest(regs.eax, 3), Ext::kXSAVES); } - // CPUID EAX=14 ECX=0 - // ------------------ + // CPUID EAX=0x0E ECX=0 (Processor Trace Enumeration Main Leaf) + // ------------------------------------------------------------ - if (maxId >= 0xE) { - cpuidQuery(®s, 0xE, 0); + if (maxId >= 0x0Eu) { + cpuidQuery(®s, 0x0E, 0); features.addIf(bitTest(regs.ebx, 4), Ext::kPTWRITE); } + // CPUID EAX=0x19 ECX=0 (Key Locker Leaf) + // -------------------------------------- + + if (maxId >= 0x19u && features.hasKL()) { + cpuidQuery(®s, 0x19, 0); + + features.addIf(bitTest(regs.ebx, 0), Ext::kAESKLE); + features.addIf(bitTest(regs.ebx, 0) && bitTest(regs.ebx, 2), Ext::kAESKLEWIDE_KL); + } + + // CPUID EAX=0x1E ECX=1 (TMUL Information Sub-leaf) + // ------------------------------------------------ + + if (maxId >= 0x1Eu && features.hasAMX_TILE()) { + cpuidQuery(®s, 0x1E, 1); + + // NOTE: Some AMX flags are mirrored here from CPUID[0x07, 0x00]. + features.addIf(bitTest(regs.eax, 0), Ext::kAMX_INT8); + features.addIf(bitTest(regs.eax, 1), Ext::kAMX_BF16); + features.addIf(bitTest(regs.eax, 2), Ext::kAMX_COMPLEX); + features.addIf(bitTest(regs.eax, 3), Ext::kAMX_FP16); + features.addIf(bitTest(regs.eax, 4), Ext::kAMX_FP8); + features.addIf(bitTest(regs.eax, 5), Ext::kAMX_TRANSPOSE); + features.addIf(bitTest(regs.eax, 6), Ext::kAMX_TF32); + features.addIf(bitTest(regs.eax, 7), Ext::kAMX_AVX512); + features.addIf(bitTest(regs.eax, 8), Ext::kAMX_MOVRS); + } + + // CPUID EAX=0x24 ECX=0 (AVX10 Information) + // ---------------------------------------- + + if (maxId >= 0x24u && avx10Enabled) { + // EAX output is the maximum supported sub-leaf. + cpuidQuery(®s, 0x24, 0); + + // AVX10 Converged Vector ISA version. + uint32_t ver = regs.ebx & 0xFFu; + + features.addIf(ver >= 1u, Ext::kAVX10_1); + features.addIf(ver >= 2u, Ext::kAVX10_2); + } + // CPUID EAX=0x80000000...maxId // ---------------------------- @@ -471,7 +516,7 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { // The highest EAX that we understand. constexpr uint32_t kHighestProcessedEAX = 0x8000001Fu; - // Several CPUID calls are required to get the whole branc string. It's easier + // Several CPUID calls are required to get the whole brand string. It's easier // to copy one DWORD at a time instead of copying the string a byte by byte. uint32_t* brand = cpu._brand.u32; do { @@ -566,6 +611,7 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { // Other resources: // https://en.wikipedia.org/wiki/AArch64 // https://en.wikipedia.org/wiki/Apple_silicon#List_of_Apple_processors +// https://developer.arm.com/downloads/-/exploration-tools/feature-names-for-a-profile // https://developer.arm.com/architectures/learn-the-architecture/understanding-the-armv8-x-extensions/single-page #if ASMJIT_ARCH_ARM @@ -573,7 +619,7 @@ static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { namespace arm { // ARM commonly refers to CPU features using FEAT_ prefix, we use Ext:: to make it compatible with other parts. -typedef CpuFeatures::ARM Ext; +using Ext = CpuFeatures::ARM; // CpuInfo - Detect - ARM - OS Kernel Version // ========================================== @@ -584,14 +630,12 @@ struct UNameKernelVersion { inline bool atLeast(int major, int minor, int patch = 0) const noexcept { if (parts[0] >= major) { - if (parts[0] > major) + if (parts[0] > major) { return true; + } if (parts[1] >= minor) { - if (parts[1] > minor) - return true; - - return parts[2] >= patch; + return parts[1] > minor ? true : parts[2] >= patch; } } @@ -599,14 +643,15 @@ struct UNameKernelVersion { } }; -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static UNameKernelVersion getUNameKernelVersion() noexcept { UNameKernelVersion ver{}; ver.parts[0] = -1; utsname buffer; - if (uname(&buffer) != 0) + if (uname(&buffer) != 0) { return ver; + } size_t count = 0; char* p = buffer.release; @@ -614,8 +659,9 @@ static UNameKernelVersion getUNameKernelVersion() noexcept { uint32_t c = uint8_t(*p); if (c >= uint32_t('0') && c <= uint32_t('9')) { ver.parts[count] = int(strtol(p, &p, 10)); - if (++count == 3) + if (++count == 3) { break; + } } else if (c == '.' || c == '-') { p++; @@ -632,13 +678,13 @@ static UNameKernelVersion getUNameKernelVersion() noexcept { // CpuInfo - Detect - ARM - Baseline Features of ARM Architectures // =============================================================== -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static inline void populateBaseAArch32Features(CpuFeatures::ARM& features) noexcept { // No baseline flags at the moment. DebugUtils::unused(features); } -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static inline void populateBaseAArch64Features(CpuFeatures::ARM& features) noexcept { // AArch64 is based on ARMv8.0 and later. features.add(Ext::kARMv6); @@ -659,45 +705,44 @@ static inline void populateBaseARMFeatures(CpuInfo& cpu) noexcept { #endif } -// CpuInfo - Detect - ARM - Madatory Features of ARM Architectures -// =============================================================== +// CpuInfo - Detect - ARM - Mandatory Features of ARM Architectures +// ================================================================ // Populates mandatory ARMv8.[v]A features. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void populateARMv8AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept { +[[maybe_unused]] +static ASMJIT_NOINLINE void populateARMv8AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept { switch (v) { default: - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 9: // ARMv8.9 features.add(Ext::kCLRBHB, Ext::kCSSC, Ext::kPRFMSLC, Ext::kSPECRES2, Ext::kRAS2); - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 8: // ARMv8.8 features.add(Ext::kHBC, Ext::kMOPS, Ext::kNMI); - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 7: // ARMv8.7 features.add(Ext::kHCX, Ext::kPAN3, Ext::kWFXT, Ext::kXS); - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 6: // ARMv8.6 - // Missing: AMUv1p1. - features.add(Ext::kBF16, Ext::kECV, Ext::kFGT, Ext::kI8MM); - ASMJIT_FALLTHROUGH; + features.add(Ext::kAMU1_1, Ext::kBF16, Ext::kECV, Ext::kFGT, Ext::kI8MM); + [[fallthrough]]; case 5: // ARMv8.5 - // Missing: CSV2_2. - features.add(Ext::kBTI, Ext::kDPB2, Ext::kFLAGM2, Ext::kFRINTTS, Ext::kSB, Ext::kSPECRES, Ext::kSSBS); - ASMJIT_FALLTHROUGH; + features.add(Ext::kBTI, Ext::kCSV2, Ext::kDPB2, Ext::kFLAGM2, Ext::kFRINTTS, Ext::kSB, Ext::kSPECRES, Ext::kSSBS); + [[fallthrough]]; case 4: // ARMv8.4 - // Missing: AMUv1, SEL2, TLBIOS, TLBIRANGE. - features.add(Ext::kDIT, Ext::kDOTPROD, Ext::kFLAGM, Ext::kLRCPC2, Ext::kLSE2, Ext::kMPAM, Ext::kNV, Ext::kTRF); - ASMJIT_FALLTHROUGH; + features.add(Ext::kAMU1, Ext::kDIT, Ext::kDOTPROD, Ext::kFLAGM, + Ext::kLRCPC2, Ext::kLSE2, Ext::kMPAM, Ext::kNV, + Ext::kSEL2, Ext::kTLBIOS, Ext::kTLBIRANGE, Ext::kTRF); + [[fallthrough]]; case 3: // ARMv8.3 features.add(Ext::kCCIDX, Ext::kFCMA, Ext::kJSCVT, Ext::kLRCPC, Ext::kPAUTH); - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 2: // ARMv8.2 - features.add(Ext::kCRC32, Ext::kDPB, Ext::kPAN2, Ext::kRAS, Ext::kUAO); - ASMJIT_FALLTHROUGH; + features.add(Ext::kDPB, Ext::kPAN2, Ext::kRAS, Ext::kUAO); + [[fallthrough]]; case 1: // ARMv8.1 features.add(Ext::kCRC32, Ext::kLOR, Ext::kLSE, Ext::kPAN, Ext::kRDM, Ext::kVHE); - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 0: // ARMv8.0 features.add(Ext::kASIMD, Ext::kFP, Ext::kIDIVA, Ext::kVFP_D32); break; @@ -705,21 +750,21 @@ static ASMJIT_FAVOR_SIZE void populateARMv8AFeatures(CpuFeatures::ARM& features, } // Populates mandatory ARMv9.[v] features. -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void populateARMv9AFeatures(CpuFeatures::ARM& features, uint32_t v) noexcept { populateARMv8AFeatures(features, v <= 4u ? 5u + v : 9u); switch (v) { default: - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 4: // ARMv9.4 - based on ARMv8.9. - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 3: // ARMv9.3 - based on ARMv8.8. - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 2: // ARMv9.2 - based on ARMv8.7. - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 1: // ARMv9.1 - based on ARMv8.6. - ASMJIT_FALLTHROUGH; + [[fallthrough]]; case 0: // ARMv9.0 - based on ARMv8.5. features.add(Ext::kRME, Ext::kSVE, Ext::kSVE2); break; @@ -733,28 +778,45 @@ static ASMJIT_FAVOR_SIZE void populateARMv9AFeatures(CpuFeatures::ARM& features, // of the registers so it's an implementation that can theoretically be tested / used in mocks. // Merges a feature that contains 0b1111 when it doesn't exist and starts at 0b0000 when it does. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeatureNA(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, +[[maybe_unused]] +static ASMJIT_INLINE void mergeAArch64CPUIDFeatureNA( + CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f0, Ext::Id f1 = Ext::kNone, Ext::Id f2 = Ext::kNone, Ext::Id f3 = Ext::kNone) noexcept { uint32_t val = uint32_t((regBits >> offset) & 0xFu); - - // If val == 0b1111 then the feature is not implemented in this case (some early extensions). - if (val == 0xFu) + if (val == 0xFu) { + // If val == 0b1111 then the feature is not implemented in this case (some early extensions). return; + } - if (f0 != Ext::kNone) features.add(f0); - if (f1 != Ext::kNone) features.addIf(val >= 1, f1); - if (f2 != Ext::kNone) features.addIf(val >= 2, f2); - if (f3 != Ext::kNone) features.addIf(val >= 3, f3); + features.addIf(f0 != Ext::kNone, f0); + features.addIf(f1 != Ext::kNone && val >= 1, f1); + features.addIf(f2 != Ext::kNone && val >= 2, f2); + features.addIf(f3 != Ext::kNone && val >= 3, f3); } -// Merges a feature that contains 0b0000 when it doesn't exist and starts at 0b0001 when it does. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeatureExt(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, +// Merges a feature identified by a single bit at `offset`. +[[maybe_unused]] +static ASMJIT_INLINE void mergeAArch64CPUIDFeature1B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f1) noexcept { + features.addIf((regBits & (uint64_t(1) << offset)) != 0, f1); +} + +// Merges a feature-list starting from 0b01 when it does (0b00 means feature not supported). +[[maybe_unused]] +static ASMJIT_INLINE void mergeAArch64CPUIDFeature2B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f1, Ext::Id f2, Ext::Id f3) noexcept { + uint32_t val = uint32_t((regBits >> offset) & 0x3u); + + features.addIf(f1 != Ext::kNone && val >= 1, f1); + features.addIf(f2 != Ext::kNone && val >= 2, f2); + features.addIf(f3 != Ext::kNone && val == 3, f3); +} + +// Merges a feature-list starting from 0b0001 when it does (0b0000 means feature not supported). +[[maybe_unused]] +static ASMJIT_INLINE void mergeAArch64CPUIDFeature4B(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, Ext::Id f1, Ext::Id f2 = Ext::kNone, Ext::Id f3 = Ext::kNone, @@ -763,72 +825,76 @@ static ASMJIT_FORCE_INLINE void mergeAArch64CPUIDFeatureExt(CpuFeatures::ARM& fe uint32_t val = uint32_t((regBits >> offset) & 0xFu); // if val == 0 it means that this feature is not supported. - - if (f1 != Ext::kNone) features.addIf(val >= 1, f1); - if (f2 != Ext::kNone) features.addIf(val >= 2, f2); - if (f3 != Ext::kNone) features.addIf(val >= 3, f3); - if (f4 != Ext::kNone) features.addIf(val >= 4, f4); + features.addIf(f1 != Ext::kNone && val >= 1, f1); + features.addIf(f2 != Ext::kNone && val >= 2, f2); + features.addIf(f3 != Ext::kNone && val >= 3, f3); + features.addIf(f4 != Ext::kNone && val >= 4, f4); } -#define MERGE_FEATURE_N_A(identifier, reg, offset, ...) mergeAArch64CPUIDFeatureNA(cpu.features().arm(), reg, offset, __VA_ARGS__) -#define MERGE_FEATURE_EXT(identifier, reg, offset, ...) mergeAArch64CPUIDFeatureExt(cpu.features().arm(), reg, offset, __VA_ARGS__) +// Merges a feature that is identified by an exact bit-combination of 4 bits. +[[maybe_unused]] +static ASMJIT_INLINE void mergeAArch64CPUIDFeature4S(CpuFeatures::ARM& features, uint64_t regBits, uint32_t offset, uint32_t value, Ext::Id f1) noexcept { + features.addIf(uint32_t((regBits >> offset) & 0xFu) == value, f1); +} + +#define MERGE_FEATURE_NA(identifier, reg, offset, ...) mergeAArch64CPUIDFeatureNA(cpu.features().arm(), reg, offset, __VA_ARGS__) +#define MERGE_FEATURE_1B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature1B(cpu.features().arm(), reg, offset, __VA_ARGS__) +#define MERGE_FEATURE_2B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature2B(cpu.features().arm(), reg, offset, __VA_ARGS__) +#define MERGE_FEATURE_4B(identifier, reg, offset, ...) mergeAArch64CPUIDFeature4B(cpu.features().arm(), reg, offset, __VA_ARGS__) +#define MERGE_FEATURE_4S(identifier, reg, offset, ...) mergeAArch64CPUIDFeature4S(cpu.features().arm(), reg, offset, __VA_ARGS__) // Detects features based on the content of ID_AA64PFR0_EL1 and ID_AA64PFR1_EL1 registers. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(CpuInfo& cpu, uint64_t fpr0, uint64_t fpr1) noexcept { +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(CpuInfo& cpu, uint64_t fpr0, uint64_t fpr1) noexcept { // ID_AA64PFR0_EL1 // =============== // FP and AdvSIMD bits should match (i.e. if FP features FP16, ASIMD must feature it too). - MERGE_FEATURE_N_A("FP bits [19:16]" , fpr0, 16, Ext::kFP, Ext::kFP16); - MERGE_FEATURE_N_A("AdvSIMD bits [23:20]" , fpr0, 20, Ext::kASIMD, Ext::kFP16); + MERGE_FEATURE_NA("FP bits [19:16]" , fpr0, 16, Ext::kFP, Ext::kFP16); + MERGE_FEATURE_NA("AdvSIMD bits [23:20]" , fpr0, 20, Ext::kASIMD, Ext::kFP16); /* - MERGE_FEATURE_EXT("GIC bits [27:24]" , fpr0, 24, ...); - */ - MERGE_FEATURE_EXT("RAS bits [31:28]" , fpr0, 28, Ext::kRAS, Ext::kRAS1_1, Ext::kRAS2); - MERGE_FEATURE_EXT("SVE bits [35:32]" , fpr0, 32, Ext::kSVE); - /* - MERGE_FEATURE_EXT("SEL2 bits [39:36]" , fpr0, 36, ...); - MERGE_FEATURE_EXT("MPAM bits [43:40]" , fpr0, 40, ...); - MERGE_FEATURE_EXT("AMU bits [47:44]" , fpr0, 44, ...); - */ - MERGE_FEATURE_EXT("DIT bits [51:48]" , fpr0, 48, Ext::kDIT); - MERGE_FEATURE_EXT("RME bits [55:52]" , fpr0, 52, Ext::kRME); - /* - MERGE_FEATURE_EXT("CSV2 bits [59:56]" , fpr0, 56, ...); - MERGE_FEATURE_EXT("CSV3 bits [63:60]" , fpr0, 60, ...); + MERGE_FEATURE_4B("GIC bits [27:24]" , fpr0, 24, ...); */ + MERGE_FEATURE_4B("RAS bits [31:28]" , fpr0, 28, Ext::kRAS, Ext::kRAS1_1, Ext::kRAS2); + MERGE_FEATURE_4B("SVE bits [35:32]" , fpr0, 32, Ext::kSVE); + MERGE_FEATURE_4B("SEL2 bits [39:36]" , fpr0, 36, Ext::kSEL2); + MERGE_FEATURE_4B("MPAM bits [43:40]" , fpr0, 40, Ext::kMPAM); + MERGE_FEATURE_4B("AMU bits [47:44]" , fpr0, 44, Ext::kAMU1, Ext::kAMU1_1); + MERGE_FEATURE_4B("DIT bits [51:48]" , fpr0, 48, Ext::kDIT); + MERGE_FEATURE_4B("RME bits [55:52]" , fpr0, 52, Ext::kRME); + MERGE_FEATURE_4B("CSV2 bits [59:56]" , fpr0, 56, Ext::kCSV2, Ext::kCSV2, Ext::kCSV2, Ext::kCSV2_3); + MERGE_FEATURE_4B("CSV3 bits [63:60]" , fpr0, 60, Ext::kCSV3); // ID_AA64PFR1_EL1 // =============== - MERGE_FEATURE_EXT("BT bits [3:0]" , fpr1, 0, Ext::kBTI); - MERGE_FEATURE_EXT("SSBS bits [7:4]" , fpr1, 4, Ext::kSSBS, Ext::kSSBS2); - MERGE_FEATURE_EXT("MTE bits [11:8]" , fpr1, 8, Ext::kMTE, Ext::kMTE2, Ext::kMTE3); + MERGE_FEATURE_4B("BT bits [3:0]" , fpr1, 0, Ext::kBTI); + MERGE_FEATURE_4B("SSBS bits [7:4]" , fpr1, 4, Ext::kSSBS, Ext::kSSBS2); + MERGE_FEATURE_4B("MTE bits [11:8]" , fpr1, 8, Ext::kMTE, Ext::kMTE2, Ext::kMTE3); /* - MERGE_FEATURE_EXT("RAS_frac bits [15:12]" , fpr1, 12, ...); - MERGE_FEATURE_EXT("MPAM_frac bits [19:16]" , fpr1, 16, ...); + MERGE_FEATURE_4B("RAS_frac bits [15:12]" , fpr1, 12, ...); + MERGE_FEATURE_4B("MPAM_frac bits [19:16]" , fpr1, 16, ...); */ - MERGE_FEATURE_EXT("SME bits [27:24]" , fpr1, 24, Ext::kSME, Ext::kSME2); - MERGE_FEATURE_EXT("RNDR_trap bits [31:28]" , fpr1, 28, Ext::kRNG_TRAP); + MERGE_FEATURE_4B("SME bits [27:24]" , fpr1, 24, Ext::kSME, Ext::kSME2); + MERGE_FEATURE_4B("RNDR_trap bits [31:28]" , fpr1, 28, Ext::kRNG_TRAP); /* - MERGE_FEATURE_EXT("CSV2_frac bits [35:32]" , fpr1, 32, ...); + MERGE_FEATURE_4B("CSV2_frac bits [35:32]" , fpr1, 32, ...); */ - MERGE_FEATURE_EXT("NMI bits [39:36]" , fpr1, 36, Ext::kNMI); + MERGE_FEATURE_4B("NMI bits [39:36]" , fpr1, 36, Ext::kNMI); /* - MERGE_FEATURE_EXT("MTE_frac bits [43:40]" , fpr1, 40, ...); - MERGE_FEATURE_EXT("GCS bits [47:44]" , fpr1, 44, ...); + MERGE_FEATURE_4B("MTE_frac bits [43:40]" , fpr1, 40, ...); */ - MERGE_FEATURE_EXT("THE bits [51:48]" , fpr1, 48, Ext::kTHE); + MERGE_FEATURE_4B("GCS bits [47:44]" , fpr1, 44, Ext::kGCS); + MERGE_FEATURE_4B("THE bits [51:48]" , fpr1, 48, Ext::kTHE); // MTEX extensions are only available when MTE3 is available. if (cpu.features().arm().hasMTE3()) - MERGE_FEATURE_EXT("MTEX bits [55:52]" , fpr1, 52, Ext::kMTE4); + MERGE_FEATURE_4B("MTEX bits [55:52]" , fpr1, 52, Ext::kMTE4); /* - MERGE_FEATURE_EXT("DF2 bits [59:56]" , fpr1, 56, ...); - MERGE_FEATURE_EXT("PFAR bits [63:60]" , fpr1, 60, ...); + MERGE_FEATURE_4B("DF2 bits [59:56]" , fpr1, 56, ...); */ + MERGE_FEATURE_4B("PFAR bits [63:60]" , fpr1, 60, Ext::kPFAR); // ID_AA64PFR0_EL1 + ID_AA64PFR1_EL1 // ================================= @@ -843,182 +909,234 @@ static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(Cp uint32_t mpamMain = uint32_t((fpr0 >> 40) & 0xFu); uint32_t mpamFrac = uint32_t((fpr1 >> 16) & 0xFu); - if (mpamMain || mpamFrac) + if (mpamMain || mpamFrac) { cpu.features().arm().add(Ext::kMPAM); + } } // Detects features based on the content of ID_AA64ISAR0_EL1 and ID_AA64ISAR1_EL1 registers. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(CpuInfo& cpu, uint64_t isar0, uint64_t isar1) noexcept { +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(CpuInfo& cpu, uint64_t isar0, uint64_t isar1) noexcept { // ID_AA64ISAR0_EL1 // ================ - MERGE_FEATURE_EXT("AES bits [7:4]" , isar0, 4, Ext::kAES, Ext::kPMULL); - MERGE_FEATURE_EXT("SHA1 bits [11:8]" , isar0, 8, Ext::kSHA1); - MERGE_FEATURE_EXT("SHA2 bits [15:12]" , isar0, 12, Ext::kSHA256, Ext::kSHA512); - MERGE_FEATURE_EXT("CRC32 bits [19:16]" , isar0, 16, Ext::kCRC32); - MERGE_FEATURE_EXT("Atomic bits [23:20]" , isar0, 20, Ext::kNone, Ext::kLSE, Ext::kLSE128); - MERGE_FEATURE_EXT("TME bits [27:24]" , isar0, 24, Ext::kTME); - MERGE_FEATURE_EXT("RDM bits [31:28]" , isar0, 28, Ext::kRDM); - MERGE_FEATURE_EXT("SHA3 bits [35:32]" , isar0, 32, Ext::kSHA3); - MERGE_FEATURE_EXT("SM3 bits [39:36]" , isar0, 36, Ext::kSM3); - MERGE_FEATURE_EXT("SM4 bits [43:40]" , isar0, 40, Ext::kSM4); - MERGE_FEATURE_EXT("DP bits [47:44]" , isar0, 44, Ext::kDOTPROD); - MERGE_FEATURE_EXT("FHM bits [51:48]" , isar0, 48, Ext::kFHM); - MERGE_FEATURE_EXT("TS bits [55:52]" , isar0, 52, Ext::kFLAGM, Ext::kFLAGM2); + MERGE_FEATURE_4B("AES bits [7:4]" , isar0, 4, Ext::kAES, Ext::kPMULL); + MERGE_FEATURE_4B("SHA1 bits [11:8]" , isar0, 8, Ext::kSHA1); + MERGE_FEATURE_4B("SHA2 bits [15:12]" , isar0, 12, Ext::kSHA256, Ext::kSHA512); + MERGE_FEATURE_4B("CRC32 bits [19:16]" , isar0, 16, Ext::kCRC32); + MERGE_FEATURE_4B("Atomic bits [23:20]" , isar0, 20, Ext::kNone, Ext::kLSE, Ext::kLSE128); + MERGE_FEATURE_4B("TME bits [27:24]" , isar0, 24, Ext::kTME); + MERGE_FEATURE_4B("RDM bits [31:28]" , isar0, 28, Ext::kRDM); + MERGE_FEATURE_4B("SHA3 bits [35:32]" , isar0, 32, Ext::kSHA3); + MERGE_FEATURE_4B("SM3 bits [39:36]" , isar0, 36, Ext::kSM3); + MERGE_FEATURE_4B("SM4 bits [43:40]" , isar0, 40, Ext::kSM4); + MERGE_FEATURE_4B("DP bits [47:44]" , isar0, 44, Ext::kDOTPROD); + MERGE_FEATURE_4B("FHM bits [51:48]" , isar0, 48, Ext::kFHM); + MERGE_FEATURE_4B("TS bits [55:52]" , isar0, 52, Ext::kFLAGM, Ext::kFLAGM2); /* - MERGE_FEATURE_EXT("TLB bits [59:56]" , isar0, 56, ...); + MERGE_FEATURE_4B("TLB bits [59:56]" , isar0, 56, ...); */ - MERGE_FEATURE_EXT("RNDR bits [63:60]" , isar0, 60, Ext::kFLAGM, Ext::kRNG); + MERGE_FEATURE_4B("RNDR bits [63:60]" , isar0, 60, Ext::kFLAGM, Ext::kRNG); // ID_AA64ISAR1_EL1 // ================ - MERGE_FEATURE_EXT("DPB bits [3:0]" , isar1, 0, Ext::kDPB, Ext::kDPB2); + MERGE_FEATURE_4B("DPB bits [3:0]" , isar1, 0, Ext::kDPB, Ext::kDPB2); /* - MERGE_FEATURE_EXT("APA bits [7:4]" , isar1, 4, ...); - MERGE_FEATURE_EXT("API bits [11:8]" , isar1, 8, ...); + MERGE_FEATURE_4B("APA bits [7:4]" , isar1, 4, ...); + MERGE_FEATURE_4B("API bits [11:8]" , isar1, 8, ...); */ - MERGE_FEATURE_EXT("JSCVT bits [15:12]" , isar1, 12, Ext::kJSCVT); - MERGE_FEATURE_EXT("FCMA bits [19:16]" , isar1, 16, Ext::kFCMA); - MERGE_FEATURE_EXT("LRCPC bits [23:20]" , isar1, 20, Ext::kLRCPC, Ext::kLRCPC2, Ext::kLRCPC3); + MERGE_FEATURE_4B("JSCVT bits [15:12]" , isar1, 12, Ext::kJSCVT); + MERGE_FEATURE_4B("FCMA bits [19:16]" , isar1, 16, Ext::kFCMA); + MERGE_FEATURE_4B("LRCPC bits [23:20]" , isar1, 20, Ext::kLRCPC, Ext::kLRCPC2, Ext::kLRCPC3); /* - MERGE_FEATURE_EXT("GPA bits [27:24]" , isar1, 24, ...); - MERGE_FEATURE_EXT("GPI bits [31:28]" , isar1, 28, ...); + MERGE_FEATURE_4B("GPA bits [27:24]" , isar1, 24, ...); + MERGE_FEATURE_4B("GPI bits [31:28]" , isar1, 28, ...); */ - MERGE_FEATURE_EXT("FRINTTS bits [35:32]" , isar1, 32, Ext::kFRINTTS); - MERGE_FEATURE_EXT("SB bits [39:36]" , isar1, 36, Ext::kSB); - MERGE_FEATURE_EXT("SPECRES bits [43:40]" , isar1, 40, Ext::kSPECRES, Ext::kSPECRES2); - MERGE_FEATURE_EXT("BF16 bits [47:44]" , isar1, 44, Ext::kBF16, Ext::kEBF16); - MERGE_FEATURE_EXT("DGH bits [51:48]" , isar1, 48, Ext::kDGH); - MERGE_FEATURE_EXT("I8MM bits [55:52]" , isar1, 52, Ext::kI8MM); - MERGE_FEATURE_EXT("XS bits [59:56]" , isar1, 56, Ext::kXS); - MERGE_FEATURE_EXT("LS64 bits [63:60]" , isar1, 60, Ext::kLS64, Ext::kLS64_V, Ext::kLS64_ACCDATA); + MERGE_FEATURE_4B("FRINTTS bits [35:32]" , isar1, 32, Ext::kFRINTTS); + MERGE_FEATURE_4B("SB bits [39:36]" , isar1, 36, Ext::kSB); + MERGE_FEATURE_4B("SPECRES bits [43:40]" , isar1, 40, Ext::kSPECRES, Ext::kSPECRES2); + MERGE_FEATURE_4B("BF16 bits [47:44]" , isar1, 44, Ext::kBF16, Ext::kEBF16); + MERGE_FEATURE_4B("DGH bits [51:48]" , isar1, 48, Ext::kDGH); + MERGE_FEATURE_4B("I8MM bits [55:52]" , isar1, 52, Ext::kI8MM); + MERGE_FEATURE_4B("XS bits [59:56]" , isar1, 56, Ext::kXS); + MERGE_FEATURE_4B("LS64 bits [63:60]" , isar1, 60, Ext::kLS64, Ext::kLS64_V, Ext::kLS64_ACCDATA); } // Detects features based on the content of ID_AA64ISAR2_EL1 register. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64ISAR2(CpuInfo& cpu, uint64_t isar2) noexcept { - MERGE_FEATURE_EXT("WFxT bits [3:0]" , isar2, 0, Ext::kNone, Ext::kWFXT); - MERGE_FEATURE_EXT("RPRES bits [7:4]" , isar2, 4, Ext::kRPRES); +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64ISAR2(CpuInfo& cpu, uint64_t isar2) noexcept { + MERGE_FEATURE_4B("WFxT bits [3:0]" , isar2, 0, Ext::kNone, Ext::kWFXT); + MERGE_FEATURE_4B("RPRES bits [7:4]" , isar2, 4, Ext::kRPRES); /* - MERGE_FEATURE_EXT("GPA3 bits [11:8]" , isar2, 8, ...); - MERGE_FEATURE_EXT("APA3 bits [15:12]" , isar2, 12, ...); + MERGE_FEATURE_4B("GPA3 bits [11:8]" , isar2, 8, ...); + MERGE_FEATURE_4B("APA3 bits [15:12]" , isar2, 12, ...); */ - MERGE_FEATURE_EXT("MOPS bits [19:16]" , isar2, 16, Ext::kMOPS); - MERGE_FEATURE_EXT("BC bits [23:20]" , isar2, 20, Ext::kHBC); + MERGE_FEATURE_4B("MOPS bits [19:16]" , isar2, 16, Ext::kMOPS); + MERGE_FEATURE_4B("BC bits [23:20]" , isar2, 20, Ext::kHBC); + MERGE_FEATURE_4B("PAC_frac bits [27:24]" , isar2, 24, Ext::kCONSTPACFIELD); + MERGE_FEATURE_4B("CLRBHB bits [31:28]" , isar2, 28, Ext::kCLRBHB); + MERGE_FEATURE_4B("SYSREG128 bits [35:32]" , isar2, 32, Ext::kSYSREG128); + MERGE_FEATURE_4B("SYSINSTR128 bits [39:36]" , isar2, 36, Ext::kSYSINSTR128); + MERGE_FEATURE_4B("PRFMSLC bits [43:40]" , isar2, 40, Ext::kPRFMSLC); + MERGE_FEATURE_4B("RPRFM bits [51:48]" , isar2, 48, Ext::kRPRFM); + MERGE_FEATURE_4B("CSSC bits [55:52]" , isar2, 52, Ext::kCSSC); + MERGE_FEATURE_4B("LUT bits [59:56]" , isar2, 56, Ext::kLUT); /* - MERGE_FEATURE_EXT("PAC_frac bits [27:24]" , isar2, 24, ...); + MERGE_FEATURE_4B("ATS1A bits [63:60]" , isar2, 60, ...); */ - MERGE_FEATURE_EXT("CLRBHB bits [31:28]" , isar2, 28, Ext::kCLRBHB); - MERGE_FEATURE_EXT("SYSREG128 bits [35:32]" , isar2, 32, Ext::kSYSREG128); - MERGE_FEATURE_EXT("SYSINSTR128 bits [39:36]" , isar2, 36, Ext::kSYSINSTR128); - MERGE_FEATURE_EXT("PRFMSLC bits [43:40]" , isar2, 40, Ext::kPRFMSLC); - MERGE_FEATURE_EXT("RPRFM bits [51:48]" , isar2, 48, Ext::kRPRFM); - MERGE_FEATURE_EXT("CSSC bits [55:52]" , isar2, 52, Ext::kCSSC); } -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64MMFR0(CpuInfo& cpu, uint64_t mmfr0) noexcept { +// TODO: This register is not accessed at the moment. +#if 0 +// Detects features based on the content of ID_AA64ISAR3_EL1register. +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64ISAR3(CpuInfo& cpu, uint64_t isar3) noexcept { + // ID_AA64ISAR3_EL1 + // ================ + + MERGE_FEATURE_4B("CPA bits [3:0]" , isar3, 0, Ext::kCPA, Ext::kCPA2); + MERGE_FEATURE_4B("FAMINMAX bits [7:4]" , isar3, 4, Ext::kFAMINMAX); + MERGE_FEATURE_4B("TLBIW bits [11:8]" , isar3, 8, Ext::kTLBIW); + /* + MERGE_FEATURE_4B("PACM bits [15:12]" , isar3, 12, ...); + */ + MERGE_FEATURE_4B("LSFE bits [19:16]" , isar3, 16, Ext::kLSFE); + MERGE_FEATURE_4B("OCCMO bits [23:20]" , isar3, 20, Ext::kOCCMO); + MERGE_FEATURE_4B("LSUI bits [27:24]" , isar3, 24, Ext::kLSUI); +} +#endif + +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64MMFR0(CpuInfo& cpu, uint64_t mmfr0) noexcept { // ID_AA64MMFR0_EL1 // ================ /* - MERGE_FEATURE_EXT("PARange bits [3:0]" , mmfr0, 0, ...); - MERGE_FEATURE_EXT("ASIDBits bits [7:4]" , mmfr0, 4, ...); - MERGE_FEATURE_EXT("BigEnd bits [11:8]" , mmfr0, 8, ...); - MERGE_FEATURE_EXT("SNSMem bits [15:12]" , mmfr0, 12, ...); - MERGE_FEATURE_EXT("BigEndEL0 bits [19:16]" , mmfr0, 16, ...); - MERGE_FEATURE_EXT("TGran16 bits [23:20]" , mmfr0, 20, ...); - MERGE_FEATURE_EXT("TGran64 bits [27:24]" , mmfr0, 24, ...); - MERGE_FEATURE_EXT("TGran4 bits [31:28]" , mmfr0, 28, ...); - MERGE_FEATURE_EXT("TGran16_2 bits [35:32]" , mmfr0, 32, ...); - MERGE_FEATURE_EXT("TGran64_2 bits [39:36]" , mmfr0, 36, ...); - MERGE_FEATURE_EXT("TGran4_2 bits [43:40]" , mmfr0, 40, ...); - MERGE_FEATURE_EXT("ExS bits [47:44]" , mmfr0, 44, ...); + MERGE_FEATURE_4B("PARange bits [3:0]" , mmfr0, 0, ...); + MERGE_FEATURE_4B("ASIDBits bits [7:4]" , mmfr0, 4, ...); + MERGE_FEATURE_4B("BigEnd bits [11:8]" , mmfr0, 8, ...); + MERGE_FEATURE_4B("SNSMem bits [15:12]" , mmfr0, 12, ...); + MERGE_FEATURE_4B("BigEndEL0 bits [19:16]" , mmfr0, 16, ...); + MERGE_FEATURE_4B("TGran16 bits [23:20]" , mmfr0, 20, ...); + MERGE_FEATURE_4B("TGran64 bits [27:24]" , mmfr0, 24, ...); + MERGE_FEATURE_4B("TGran4 bits [31:28]" , mmfr0, 28, ...); + MERGE_FEATURE_4B("TGran16_2 bits [35:32]" , mmfr0, 32, ...); + MERGE_FEATURE_4B("TGran64_2 bits [39:36]" , mmfr0, 36, ...); + MERGE_FEATURE_4B("TGran4_2 bits [43:40]" , mmfr0, 40, ...); + MERGE_FEATURE_4B("ExS bits [47:44]" , mmfr0, 44, ...); */ - MERGE_FEATURE_EXT("FGT bits [59:56]" , mmfr0, 56, Ext::kFGT, Ext::kFGT2); - MERGE_FEATURE_EXT("ECV bits [63:60]" , mmfr0, 60, Ext::kECV); + MERGE_FEATURE_4B("FGT bits [59:56]" , mmfr0, 56, Ext::kFGT, Ext::kFGT2); + MERGE_FEATURE_4B("ECV bits [63:60]" , mmfr0, 60, Ext::kECV); } -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64MMFR1(CpuInfo& cpu, uint64_t mmfr1) noexcept { +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64MMFR1(CpuInfo& cpu, uint64_t mmfr1) noexcept { // ID_AA64MMFR1_EL1 // ================ + MERGE_FEATURE_4B("HAFDBS bits [3:0]" , mmfr1, 0, Ext::kHAFDBS, Ext::kNone, Ext::kHAFT, Ext::kHDBSS); + MERGE_FEATURE_4B("VMIDBits bits [7:4]" , mmfr1, 4, Ext::kVMID16); + MERGE_FEATURE_4B("VH bits [11:8]" , mmfr1, 8, Ext::kVHE); + MERGE_FEATURE_4B("HPDS bits [15:12]" , mmfr1, 12, Ext::kHPDS, Ext::kHPDS2); + MERGE_FEATURE_4B("LO bits [19:16]" , mmfr1, 16, Ext::kLOR); + MERGE_FEATURE_4B("PAN bits [23:20]" , mmfr1, 20, Ext::kPAN, Ext::kPAN2, Ext::kPAN3); /* - MERGE_FEATURE_EXT("HAFDBS bits [3:0]" , mmfr1, 0, ...); - MERGE_FEATURE_EXT("VMIDBits bits [7:4]" , mmfr1, 4, ...); + MERGE_FEATURE_4B("SpecSEI bits [27:24]" , mmfr1, 24, ...); */ - MERGE_FEATURE_EXT("VH bits [11:8]" , mmfr1, 8, Ext::kVHE); + MERGE_FEATURE_4B("XNX bits [31:28]" , mmfr1, 28, Ext::kXNX); /* - MERGE_FEATURE_EXT("HPDS bits [15:12]" , mmfr1, 12, ...); + MERGE_FEATURE_4B("TWED bits [35:32]" , mmfr1, 32, ...); + MERGE_FEATURE_4B("ETS bits [39:36]" , mmfr1, 36, ...); */ - MERGE_FEATURE_EXT("LO bits [19:16]" , mmfr1, 16, Ext::kLOR); - MERGE_FEATURE_EXT("PAN bits [23:20]" , mmfr1, 20, Ext::kPAN, Ext::kPAN2, Ext::kPAN3); + MERGE_FEATURE_4B("HCX bits [43:40]" , mmfr1, 40, Ext::kHCX); + MERGE_FEATURE_4B("AFP bits [47:44]" , mmfr1, 44, Ext::kAFP); /* - MERGE_FEATURE_EXT("SpecSEI bits [27:24]" , mmfr1, 24, ...); - MERGE_FEATURE_EXT("XNX bits [31:28]" , mmfr1, 28, ...); - MERGE_FEATURE_EXT("TWED bits [35:32]" , mmfr1, 32, ...); - MERGE_FEATURE_EXT("ETS bits [39:36]" , mmfr1, 36, ...); - */ - MERGE_FEATURE_EXT("HCX bits [43:40]" , mmfr1, 40, Ext::kHCX); - MERGE_FEATURE_EXT("AFP bits [47:44]" , mmfr1, 44, Ext::kAFP); - /* - MERGE_FEATURE_EXT("nTLBPA bits [51:48]" , mmfr1, 48, ...); - MERGE_FEATURE_EXT("TIDCP1 bits [55:52]" , mmfr1, 52, ...); - MERGE_FEATURE_EXT("CMOW bits [59:56]" , mmfr1, 56, ...); - MERGE_FEATURE_EXT("ECBHB bits [63:60]" , mmfr1, 60, ...); + MERGE_FEATURE_4B("nTLBPA bits [51:48]" , mmfr1, 48, ...); + MERGE_FEATURE_4B("TIDCP1 bits [55:52]" , mmfr1, 52, ...); */ + MERGE_FEATURE_4B("CMOW bits [59:56]" , mmfr1, 56, Ext::kCMOW); + MERGE_FEATURE_4B("ECBHB bits [63:60]" , mmfr1, 60, Ext::kECBHB); } -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64MMFR2(CpuInfo& cpu, uint64_t mmfr2) noexcept { +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64MMFR2(CpuInfo& cpu, uint64_t mmfr2) noexcept { // ID_AA64MMFR2_EL1 // ================ /* - MERGE_FEATURE_EXT("CnP bits [3:0]" , mmfr2, 0, ...); + MERGE_FEATURE_4B("CnP bits [3:0]" , mmfr2, 0, ...); */ - MERGE_FEATURE_EXT("UAO bits [7:4]" , mmfr2, 4, Ext::kUAO); + MERGE_FEATURE_4B("UAO bits [7:4]" , mmfr2, 4, Ext::kUAO); /* - MERGE_FEATURE_EXT("LSM bits [11:8]" , mmfr2, 8, ...); - MERGE_FEATURE_EXT("IESB bits [15:12]" , mmfr2, 12, ...); - MERGE_FEATURE_EXT("VARange bits [19:16]" , mmfr2, 16, ...); + MERGE_FEATURE_4B("LSM bits [11:8]" , mmfr2, 8, ...); + MERGE_FEATURE_4B("IESB bits [15:12]" , mmfr2, 12, ...); */ - MERGE_FEATURE_EXT("CCIDX bits [23:20]" , mmfr2, 20, Ext::kCCIDX); - MERGE_FEATURE_EXT("NV bits [27:24]" , mmfr2, 24, Ext::kNV, Ext::kNV2); + MERGE_FEATURE_4B("VARange bits [19:16]" , mmfr2, 16, Ext::kLVA, Ext::kLVA3); + MERGE_FEATURE_4B("CCIDX bits [23:20]" , mmfr2, 20, Ext::kCCIDX); + MERGE_FEATURE_4B("NV bits [27:24]" , mmfr2, 24, Ext::kNV, Ext::kNV2); /* - MERGE_FEATURE_EXT("ST bits [31:28]" , mmfr2, 28, ...); + MERGE_FEATURE_4B("ST bits [31:28]" , mmfr2, 28, ...); */ - MERGE_FEATURE_EXT("AT bits [35:32]" , mmfr2, 32, Ext::kLSE2); + MERGE_FEATURE_4B("AT bits [35:32]" , mmfr2, 32, Ext::kLSE2); /* - MERGE_FEATURE_EXT("IDS bits [39:36]" , mmfr2, 36, ...); - MERGE_FEATURE_EXT("FWB bits [43:40]" , mmfr2, 40, ...); - MERGE_FEATURE_EXT("TTL bits [51:48]" , mmfr2, 48, ...); - MERGE_FEATURE_EXT("BBM bits [55:52]" , mmfr2, 52, ...); - MERGE_FEATURE_EXT("EVT bits [59:56]" , mmfr2, 56, ...); - MERGE_FEATURE_EXT("E0PD bits [63:60]" , mmfr2, 60, ...); + MERGE_FEATURE_4B("IDS bits [39:36]" , mmfr2, 36, ...); + MERGE_FEATURE_4B("FWB bits [43:40]" , mmfr2, 40, ...); + MERGE_FEATURE_4B("TTL bits [51:48]" , mmfr2, 48, ...); + MERGE_FEATURE_4B("BBM bits [55:52]" , mmfr2, 52, ...); + MERGE_FEATURE_4B("EVT bits [59:56]" , mmfr2, 56, ...); + MERGE_FEATURE_4B("E0PD bits [63:60]" , mmfr2, 60, ...); */ } -// Detects features based on the content of ID_AA64ZFR0_EL1 register. -ASMJIT_MAYBE_UNUSED -static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cpu, uint64_t zfr0) noexcept { - MERGE_FEATURE_EXT("SVEver bits [3:0]" , zfr0, 0, Ext::kSVE2, Ext::kSVE2_1); - MERGE_FEATURE_EXT("AES bits [7:4]" , zfr0, 4, Ext::kSVE_AES, Ext::kSVE_PMULL128); - MERGE_FEATURE_EXT("BitPerm bits [19:16]" , zfr0, 16, Ext::kSVE_BITPERM); - MERGE_FEATURE_EXT("BF16 bits [23:20]" , zfr0, 20, Ext::kSVE_BF16, Ext::kSVE_EBF16); - MERGE_FEATURE_EXT("B16B16 bits [27:24]" , zfr0, 24, Ext::kSVE_B16B16); - MERGE_FEATURE_EXT("SHA3 bits [35:32]" , zfr0, 32, Ext::kSVE_SHA3); - MERGE_FEATURE_EXT("SM4 bits [43:40]" , zfr0, 40, Ext::kSVE_SM4); - MERGE_FEATURE_EXT("I8MM bits [47:44]" , zfr0, 44, Ext::kSVE_I8MM); - MERGE_FEATURE_EXT("F32MM bits [55:52]" , zfr0, 52, Ext::kSVE_F32MM); - MERGE_FEATURE_EXT("F64MM bits [59:56]" , zfr0, 56, Ext::kSVE_F64MM); +// Detects features based on the content of ID_AA64ZFR0_EL1 (SVE Feature ID register 0). +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cpu, uint64_t zfr0) noexcept { + MERGE_FEATURE_4B("SVEver bits [3:0]" , zfr0, 0, Ext::kSVE2, Ext::kSVE2_1, Ext::kSVE2_2); + MERGE_FEATURE_4B("AES bits [7:4]" , zfr0, 4, Ext::kSVE_AES, Ext::kSVE_PMULL128); + MERGE_FEATURE_4B("EltPerm bits [15:12]" , zfr0, 12, Ext::kSVE_ELTPERM); + MERGE_FEATURE_4B("BitPerm bits [19:16]" , zfr0, 16, Ext::kSVE_BITPERM); + MERGE_FEATURE_4B("BF16 bits [23:20]" , zfr0, 20, Ext::kSVE_BF16, Ext::kSVE_EBF16); + MERGE_FEATURE_4B("B16B16 bits [27:24]" , zfr0, 24, Ext::kSVE_B16B16); + MERGE_FEATURE_4B("SHA3 bits [35:32]" , zfr0, 32, Ext::kSVE_SHA3); + MERGE_FEATURE_4B("SM4 bits [43:40]" , zfr0, 40, Ext::kSVE_SM4); + MERGE_FEATURE_4B("I8MM bits [47:44]" , zfr0, 44, Ext::kSVE_I8MM); + MERGE_FEATURE_4B("F32MM bits [55:52]" , zfr0, 52, Ext::kSVE_F32MM); + MERGE_FEATURE_4B("F64MM bits [59:56]" , zfr0, 56, Ext::kSVE_F64MM); } -#undef MERGE_FEATURE_EXT -#undef MERGE_FEATURE_N_A +[[maybe_unused]] +static inline void detectAArch64FeaturesViaCPUID_AA64SMFR0(CpuInfo& cpu, uint64_t smfr0) noexcept { + MERGE_FEATURE_1B("SMOP4 bit [0]" , smfr0, 0, Ext::kSME_MOP4); + MERGE_FEATURE_1B("STMOP bit [16]" , smfr0, 16, Ext::kSME_TMOP); + MERGE_FEATURE_1B("SFEXPA bit [23]" , smfr0, 23, Ext::kSSVE_FEXPA); + MERGE_FEATURE_1B("AES bit [24]" , smfr0, 24, Ext::kSSVE_AES); + MERGE_FEATURE_1B("SBitPerm bit [25]" , smfr0, 25, Ext::kSSVE_BITPERM); + MERGE_FEATURE_1B("SF8DP2 bit [28]" , smfr0, 28, Ext::kSSVE_FP8DOT2); + MERGE_FEATURE_1B("SF8DP4 bit [29]" , smfr0, 29, Ext::kSSVE_FP8DOT4); + MERGE_FEATURE_1B("SF8FMA bit [30]" , smfr0, 30, Ext::kSSVE_FP8FMA); + MERGE_FEATURE_1B("F32F32 bit [32]" , smfr0, 32, Ext::kSME_F32F32); + MERGE_FEATURE_1B("BI32I32 bit [33]" , smfr0, 33, Ext::kSME_BI32I32); + MERGE_FEATURE_1B("B16F32 bit [34]" , smfr0, 34, Ext::kSME_B16F32); + MERGE_FEATURE_1B("F16F32 bit [35]" , smfr0, 35, Ext::kSME_F16F32); + MERGE_FEATURE_4S("I8I32 bits [39:36]" , smfr0, 36, 0xF, Ext::kSME_I8I32); + MERGE_FEATURE_1B("F8F32 bit [40]" , smfr0, 40, Ext::kSME_F8F32); + MERGE_FEATURE_1B("F8F16 bit [41]" , smfr0, 41, Ext::kSME_F8F16); + MERGE_FEATURE_1B("F16F16 bit [42]" , smfr0, 42, Ext::kSME_F16F16); + MERGE_FEATURE_1B("B16B16 bit [43]" , smfr0, 43, Ext::kSME_B16B16); + MERGE_FEATURE_4S("I16I32 bits [47:44]" , smfr0, 44, 0x5, Ext::kSME_I16I32); + MERGE_FEATURE_1B("F64F64 bit [48]" , smfr0, 48, Ext::kSME_F64F64); + MERGE_FEATURE_4S("I16I64 bits [55:52]" , smfr0, 52, 0xF, Ext::kSME_I16I64); + MERGE_FEATURE_4B("SMEver bits [59:56]" , smfr0, 56, Ext::kSME2, Ext::kSME2_1, Ext::kSME2_2); + MERGE_FEATURE_1B("LUTv2 bit [60]" , smfr0, 60, Ext::kSME_LUTv2); + MERGE_FEATURE_1B("FA64 bit [63]" , smfr0, 63, Ext::kSME_FA64); +} + +#undef MERGE_FEATURE_4S +#undef MERGE_FEATURE_4B +#undef MERGE_FEATURE_2B +#undef MERGE_FEATURE_1B +#undef MERGE_FEATURE_NA // CpuInfo - Detect - ARM - CPU Vendor Features // ============================================ @@ -1026,22 +1144,30 @@ static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID_AA64ZFR0(CpuInfo& cp // CPU features detection based on Apple family ID. enum class AppleFamilyId : uint32_t { // Apple design. - kSWIFT = 0x1E2D6381u, // Apple A6/A6X (ARMv7s). - kCYCLONE = 0x37A09642u, // Apple A7 (ARMv8.0-A). - kTYPHOON = 0x2C91A47Eu, // Apple A8 (ARMv8.0-A). - kTWISTER = 0x92FB37C8u, // Apple A9 (ARMv8.0-A). - kHURRICANE = 0x67CEEE93u, // Apple A10 (ARMv8.1-A). - kMONSOON_MISTRAL = 0xE81E7EF6u, // Apple A11 (ARMv8.2-A). - kVORTEX_TEMPEST = 0x07D34B9Fu, // Apple A12 (ARMv8.3-A). - kLIGHTNING_THUNDER = 0x462504D2u, // Apple A13 (ARMv8.4-A). - kFIRESTORM_ICESTORM = 0x1B588BB3u, // Apple A14/M1 (ARMv8.5-A). - kAVALANCHE_BLIZZARD = 0XDA33D83Du, // Apple A15/M2. - kEVEREST_SAWTOOTH = 0X8765EDEAu // Apple A16. + kSWIFT = 0x1E2D6381u, // Apple A6/A6X (ARMv7s). + kCYCLONE = 0x37A09642u, // Apple A7 (ARMv8.0-A). + kTYPHOON = 0x2C91A47Eu, // Apple A8 (ARMv8.0-A). + kTWISTER = 0x92FB37C8u, // Apple A9 (ARMv8.0-A). + kHURRICANE = 0x67CEEE93u, // Apple A10 (ARMv8.1-A). + kMONSOON_MISTRAL = 0xE81E7EF6u, // Apple A11 (ARMv8.2-A). + kVORTEX_TEMPEST = 0x07D34B9Fu, // Apple A12 (ARMv8.3-A). + kLIGHTNING_THUNDER = 0x462504D2u, // Apple A13 (ARMv8.4-A). + kFIRESTORM_ICESTORM = 0x1B588BB3u, // Apple A14/M1 (ARMv8.5-A). + kAVALANCHE_BLIZZARD = 0XDA33D83Du, // Apple A15/M2 (ARMv8.6-A). + kEVEREST_SAWTOOTH = 0X8765EDEAu, // Apple A16 (ARMv8.6-A). + kIBIZA = 0xFA33415Eu, // Apple M3 (ARMv8.6-A). + kPALMA = 0x72015832u, // Apple M3 Max (ARMv8.6-A). + kLOBOS = 0x5F4DEA93u, // Apple M3 Pro (ARMv8.6-A). + kCOLL = 0x2876F5B5u, // Apple A17 Pro (ARMv8.7-A). + kDONAN = 0x6F5129ACu, // Apple M4 (ARMv8.7-A). + kBRAVA = 0x17D5B93Au, // Apple M4 Max (ARMv8.7-A). + kTUPAI = 0x204526D0u, // Apple A18 . + kTAHITI = 0x75D4ACB9u // Apple A18 Pro . }; -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE bool detectARMFeaturesViaAppleFamilyId(CpuInfo& cpu) noexcept { - typedef AppleFamilyId Id; + using Id = AppleFamilyId; CpuFeatures::ARM& features = cpu.features().arm(); switch (cpu.familyId()) { @@ -1050,56 +1176,93 @@ static ASMJIT_FAVOR_SIZE bool detectARMFeaturesViaAppleFamilyId(CpuInfo& cpu) no case uint32_t(Id::kTYPHOON): case uint32_t(Id::kTWISTER): populateARMv8AFeatures(features, 0); - features.add(Ext::kAES, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256 + ); return true; // Apple A10 (ARMv8.0-A). case uint32_t(Id::kHURRICANE): populateARMv8AFeatures(features, 0); - features.add(Ext::kAES, Ext::kCRC32, Ext::kLOR, Ext::kPAN, Ext::kPMU, Ext::kPMULL, Ext::kRDM, Ext::kSHA1, - Ext::kSHA256, Ext::kVHE); + features.add( + Ext::kLOR, Ext::kPAN, Ext::kPMU, Ext::kVHE, + Ext::kAES, Ext::kCRC32, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kRDM + ); return true; // Apple A11 (ARMv8.2-A). case uint32_t(Id::kMONSOON_MISTRAL): populateARMv8AFeatures(features, 2); - features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, + Ext::kFP16, Ext::kFP16CONV + ); return true; // Apple A12 (ARMv8.3-A). case uint32_t(Id::kVORTEX_TEMPEST): populateARMv8AFeatures(features, 3); - features.add(Ext::kAES, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256); + features.add( + Ext::kPMU, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, + Ext::kFP16, Ext::kFP16CONV + ); return true; // Apple A13 (ARMv8.4-A). case uint32_t(Id::kLIGHTNING_THUNDER): populateARMv8AFeatures(features, 4); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, - Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512 + ); return true; // Apple A14/M1 (ARMv8.5-A). case uint32_t(Id::kFIRESTORM_ICESTORM): - // Missing: CSV2, CSV3. populateARMv8AFeatures(features, 4); - features.add(Ext::kAES, Ext::kDPB2, Ext::kECV, Ext::kFHM, Ext::kFLAGM2, Ext::kFP16, Ext::kFP16CONV, - Ext::kFRINTTS, Ext::kPMU, Ext::kPMULL, Ext::kSB, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, - Ext::kSHA512, Ext::kSSBS); + features.add( + Ext::kCSV2, Ext::kCSV3, Ext::kDPB2, Ext::kECV, Ext::kFLAGM2, Ext::kPMU, Ext::kSB, Ext::kSSBS, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kFRINTTS, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); return true; // Apple A15/M2. case uint32_t(Id::kAVALANCHE_BLIZZARD): populateARMv8AFeatures(features, 6); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kPMU, Ext::kPMULL, Ext::kSHA1, - Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); return true; - // Apple A16. + // Apple A16/M3. case uint32_t(Id::kEVEREST_SAWTOOTH): + case uint32_t(Id::kIBIZA): + case uint32_t(Id::kPALMA): + case uint32_t(Id::kLOBOS): populateARMv8AFeatures(features, 6); - features.add(Ext::kAES, Ext::kFHM, Ext::kFP16, Ext::kFP16CONV, Ext::kHCX, Ext::kPMU, Ext::kPMULL, - Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512); + features.add( + Ext::kHCX, Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kPMULL, Ext::kSHA1, Ext::kSHA256, Ext::kSHA3, Ext::kSHA512 + ); + return true; + + // Apple A17/M4. + case uint32_t(Id::kCOLL): + case uint32_t(Id::kDONAN): + case uint32_t(Id::kBRAVA): + populateARMv8AFeatures(features, 7); + features.add( + Ext::kPMU, + Ext::kFP16, Ext::kFP16CONV, Ext::kFHM, + Ext::kAES, Ext::kSHA1, Ext::kSHA3, Ext::kSHA256, Ext::kSHA512, + Ext::kSME, Ext::kSME2, Ext::kSME_F64F64, Ext::kSME_I16I64); return true; default: @@ -1116,7 +1279,7 @@ static ASMJIT_FAVOR_SIZE bool detectARMFeaturesViaAppleFamilyId(CpuInfo& cpu) no // target it was compiled to. #if ASMJIT_ARCH_ARM == 32 -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void detectAArch32FeaturesViaCompilerFlags(CpuInfo& cpu) noexcept { DebugUtils::unused(cpu); @@ -1154,7 +1317,7 @@ static ASMJIT_FAVOR_SIZE void detectAArch32FeaturesViaCompilerFlags(CpuInfo& cpu #endif // ASMJIT_ARCH_ARM == 32 #if ASMJIT_ARCH_ARM == 64 -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCompilerFlags(CpuInfo& cpu) noexcept { DebugUtils::unused(cpu); @@ -1310,7 +1473,7 @@ static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCompilerFlags(CpuInfo& cpu } #endif // ASMJIT_ARCH_ARM == 64 -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void detectARMFeaturesViaCompilerFlags(CpuInfo& cpu) noexcept { #if ASMJIT_ARCH_ARM == 32 detectAArch32FeaturesViaCompilerFlags(cpu); @@ -1323,7 +1486,7 @@ static ASMJIT_FAVOR_SIZE void detectARMFeaturesViaCompilerFlags(CpuInfo& cpu) no // ===================================================== // Postprocesses AArch32 features. -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void postProcessAArch32Features(CpuFeatures::ARM& features) noexcept { DebugUtils::unused(features); } @@ -1331,22 +1494,26 @@ static ASMJIT_FAVOR_SIZE void postProcessAArch32Features(CpuFeatures::ARM& featu // Postprocesses AArch64 features. // // The only reason to use this function is to deduce some flags from others. -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void postProcessAArch64Features(CpuFeatures::ARM& features) noexcept { - if (features.hasFP16()) + if (features.hasFP16()) { features.add(Ext::kFP16CONV); + } - if (features.hasMTE3()) + if (features.hasMTE3()) { features.add(Ext::kMTE2); + } - if (features.hasMTE2()) + if (features.hasMTE2()) { features.add(Ext::kMTE); + } - if (features.hasSSBS2()) + if (features.hasSSBS2()) { features.add(Ext::kSSBS); + } } -ASMJIT_MAYBE_UNUSED +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void postProcessARMCpuInfo(CpuInfo& cpu) noexcept { #if ASMJIT_ARCH_ARM == 32 postProcessAArch32Features(cpu.features().arm()); @@ -1363,7 +1530,7 @@ static ASMJIT_FAVOR_SIZE void postProcessARMCpuInfo(CpuInfo& cpu) noexcept { // Since the register ID is encoded with the instruction we have to create a function for each register ID to read. #define ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(func, regId) \ -ASMJIT_MAYBE_UNUSED \ +[[maybe_unused]] \ static inline uint64_t func() noexcept { \ uint64_t output; \ __asm__ __volatile__("mrs %0, " #regId : "=r"(output)); \ @@ -1390,18 +1557,13 @@ ASMJIT_AARCH64_DEFINE_CPUID_READ_FN(aarch64ReadZFR0, S3_0_C0_C4_4) // ID_AA64ZFR // dependent is zeroed, only the bits that are used for CPU feature identification would be present. // // References: -// - https://docs.kernel.org/arm64/cpu-feature-registers.html -ASMJIT_MAYBE_UNUSED +// - https://docs.kernel.org/arch/arm64/cpu-feature-registers.html +[[maybe_unused]] static ASMJIT_FAVOR_SIZE void detectAArch64FeaturesViaCPUID(CpuInfo& cpu) noexcept { populateBaseARMFeatures(cpu); - detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu, - aarch64ReadPFR0(), - aarch64ReadPFR1()); - - detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu, - aarch64ReadISAR0(), - aarch64ReadISAR1()); + detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu, aarch64ReadPFR0(), aarch64ReadPFR1()); + detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu, aarch64ReadISAR0(), aarch64ReadISAR1()); // TODO: Fix this on FreeBSD - I don't know what kernel version allows to access the registers below... @@ -1499,7 +1661,7 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { #ifndef AT_HWCAP #define AT_HWCAP 16 -#endif // AT_HWCAP +#endif // !AT_HWCAP #ifndef AT_HWCAP2 #define AT_HWCAP2 26 @@ -1524,48 +1686,96 @@ static void getAuxValues(unsigned long* vals, const unsigned long* tags, size_t #error "[asmjit] getAuxValues() - Unsupported OS." #endif -struct HWCapMapping { - uint8_t featureId; - uint8_t hwCapBit; -}; - static const unsigned long hwCapTags[2] = { AT_HWCAP, AT_HWCAP2 }; -static ASMJIT_FAVOR_SIZE void mergeHWCaps(CpuInfo& cpu, unsigned long mask, const HWCapMapping* mapping, size_t size) noexcept { - for (size_t i = 0; i < size; i++) { - cpu.features().addIf(Support::bitTest(mask, mapping[i].hwCapBit), mapping[i].featureId); +struct HWCapMapping32 { uint8_t featureId[32]; }; +struct HWCapMapping64 { uint8_t featureId[64]; }; + +template +static ASMJIT_FAVOR_SIZE void mergeHWCaps(CpuInfo& cpu, const Mask& mask, const Map& map) noexcept { + static_assert(sizeof(Mask) * 8u == sizeof(Map)); + + Support::BitWordIterator it(mask); + while (it.hasNext()) { + uint32_t featureId = map.featureId[it.next()]; + cpu.features().add(featureId); } + cpu.features().remove(0xFFu); } #if ASMJIT_ARCH_ARM == 32 // Reference: // - https://github.com/torvalds/linux/blob/master/arch/arm/include/uapi/asm/hwcap.h -static const HWCapMapping hwCapMapping[] = { - { uint8_t(Ext::kEDSP) , 7 }, // HWCAP_EDSP - { uint8_t(Ext::kASIMD) , 12 }, // HWCAP_NEON - { uint8_t(Ext::kFP) , 13 }, // HWCAP_VFPv3 - { uint8_t(Ext::kFMAC) , 16 }, // HWCAP_VFPv4 - { uint8_t(Ext::kIDIVA) , 17 }, // HWCAP_IDIVA - { uint8_t(Ext::kIDIVT) , 18 }, // HWCAP_IDIVT - { uint8_t(Ext::kVFP_D32) , 19 }, // HWCAP_VFPD32 - { uint8_t(Ext::kFP16CONV) , 22 }, // HWCAP_FPHP - { uint8_t(Ext::kFP16) , 23 }, // HWCAP_ASIMDHP - { uint8_t(Ext::kDOTPROD) , 24 }, // HWCAP_ASIMDDP - { uint8_t(Ext::kFHM) , 25 }, // HWCAP_ASIMDFHM - { uint8_t(Ext::kBF16) , 26 }, // HWCAP_ASIMDBF16 - { uint8_t(Ext::kI8MM) , 27 } // HWCAP_I8MM -}; +static constexpr HWCapMapping32 hwCap1Mapping = {{ + uint8_t(0xFF) , // [ 0] + uint8_t(0xFF) , // [ 1] + uint8_t(0xFF) , // [ 2] + uint8_t(0xFF) , // [ 3] + uint8_t(0xFF) , // [ 4] + uint8_t(0xFF) , // [ 5] + uint8_t(0xFF) , // [ 6] + uint8_t(Ext::kEDSP) , // [ 7] HWCAP_EDSP + uint8_t(0xFF) , // [ 8] + uint8_t(0xFF) , // [ 9] + uint8_t(0xFF) , // [10] + uint8_t(0xFF) , // [11] + uint8_t(Ext::kASIMD) , // [12] HWCAP_NEON + uint8_t(Ext::kFP) , // [13] HWCAP_VFPv3 + uint8_t(0xFF) , // [14] + uint8_t(0xFF) , // [15] + uint8_t(Ext::kFMAC) , // [16] HWCAP_VFPv4 + uint8_t(Ext::kIDIVA) , // [17] HWCAP_IDIVA + uint8_t(Ext::kIDIVT) , // [18] HWCAP_IDIVT + uint8_t(Ext::kVFP_D32) , // [19] HWCAP_VFPD32 + uint8_t(0xFF) , // [20] + uint8_t(0xFF) , // [21] + uint8_t(Ext::kFP16CONV) , // [22] HWCAP_FPHP + uint8_t(Ext::kFP16) , // [23] HWCAP_ASIMDHP + uint8_t(Ext::kDOTPROD) , // [24] HWCAP_ASIMDDP + uint8_t(Ext::kFHM) , // [25] HWCAP_ASIMDFHM + uint8_t(Ext::kBF16) , // [26] HWCAP_ASIMDBF16 + uint8_t(Ext::kI8MM) , // [27] HWCAP_I8MM + uint8_t(0xFF) , // [28] + uint8_t(0xFF) , // [29] + uint8_t(0xFF) , // [30] + uint8_t(0xFF) // [31] +}}; -static const HWCapMapping hwCap2Mapping[] = { - { uint8_t(Ext::kAES) , 0 }, // HWCAP2_AES - { uint8_t(Ext::kPMULL) , 1 }, // HWCAP2_PMULL - { uint8_t(Ext::kSHA1) , 2 }, // HWCAP2_SHA1 - { uint8_t(Ext::kSHA256) , 3 }, // HWCAP2_SHA2 - { uint8_t(Ext::kCRC32) , 4 }, // HWCAP2_CRC32 - { uint8_t(Ext::kSB) , 5 }, // HWCAP2_SB - { uint8_t(Ext::kSSBS) , 6 } // HWCAP2_SSBS -}; +static constexpr HWCapMapping32 hwCap2Mapping = {{ + uint8_t(Ext::kAES) , // [ 0] HWCAP2_AES + uint8_t(Ext::kPMULL) , // [ 1] HWCAP2_PMULL + uint8_t(Ext::kSHA1) , // [ 2] HWCAP2_SHA1 + uint8_t(Ext::kSHA256) , // [ 3] HWCAP2_SHA2 + uint8_t(Ext::kCRC32) , // [ 4] HWCAP2_CRC32 + uint8_t(Ext::kSB) , // [ 5] HWCAP2_SB + uint8_t(Ext::kSSBS) , // [ 6] HWCAP2_SSBS + uint8_t(0xFF) , // [ 7] + uint8_t(0xFF) , // [ 8] + uint8_t(0xFF) , // [ 9] + uint8_t(0xFF) , // [10] + uint8_t(0xFF) , // [11] + uint8_t(0xFF) , // [12] + uint8_t(0xFF) , // [13] + uint8_t(0xFF) , // [14] + uint8_t(0xFF) , // [15] + uint8_t(0xFF) , // [16] + uint8_t(0xFF) , // [17] + uint8_t(0xFF) , // [18] + uint8_t(0xFF) , // [19] + uint8_t(0xFF) , // [20] + uint8_t(0xFF) , // [21] + uint8_t(0xFF) , // [22] + uint8_t(0xFF) , // [23] + uint8_t(0xFF) , // [24] + uint8_t(0xFF) , // [25] + uint8_t(0xFF) , // [26] + uint8_t(0xFF) , // [27] + uint8_t(0xFF) , // [28] + uint8_t(0xFF) , // [29] + uint8_t(0xFF) , // [30] + uint8_t(0xFF) // [31] +}}; static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { cpu._wasDetected = true; @@ -1574,18 +1784,20 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { unsigned long hwCapMasks[2] {}; getAuxValues(hwCapMasks, hwCapTags, 2u); - mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); - mergeHWCaps(cpu, hwCap2Masks[0], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping)); + mergeHWCaps(cpu, hwCapMasks[0], hwCap1Mapping); + mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping); CpuFeatures::ARM& features = cpu.features().arm(); // ARMv7 provides FP|ASIMD. - if (features.hasFP() || features.hasASIMD()) + if (features.hasFP() || features.hasASIMD()) { features.add(CpuFeatures::ARM::kARMv7); + } // ARMv8 provives AES, CRC32, PMULL, SHA1, and SHA256. - if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA256()) + if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA256()) { features.add(CpuFeatures::ARM::kARMv8a); + } postProcessARMCpuInfo(cpu); } @@ -1593,92 +1805,141 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { #else // Reference: -// - https://docs.kernel.org/arm64/elf_hwcaps.html +// - https://docs.kernel.org/arch/arm64/elf_hwcaps.html // - https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h -static const HWCapMapping hwCapMapping[] = { - { uint8_t(Ext::kFP) , 0 }, // HWCAP_FP - { uint8_t(Ext::kASIMD) , 1 }, // HWCAP_ASIMD - /* - { uint8_t(Ext::k) , 2 }, // HWCAP_EVTSTRM - */ - { uint8_t(Ext::kAES) , 3 }, // HWCAP_AES - { uint8_t(Ext::kPMULL) , 4 }, // HWCAP_PMULL - { uint8_t(Ext::kSHA1) , 5 }, // HWCAP_SHA1 - { uint8_t(Ext::kSHA256) , 6 }, // HWCAP_SHA2 - { uint8_t(Ext::kCRC32) , 7 }, // HWCAP_CRC32 - { uint8_t(Ext::kLSE) , 8 }, // HWCAP_ATOMICS - { uint8_t(Ext::kFP16CONV) , 9 }, // HWCAP_FPHP - { uint8_t(Ext::kFP16) , 10 }, // HWCAP_ASIMDHP - { uint8_t(Ext::kCPUID) , 11 }, // HWCAP_CPUID - { uint8_t(Ext::kRDM) , 12 }, // HWCAP_ASIMDRDM - { uint8_t(Ext::kJSCVT) , 13 }, // HWCAP_JSCVT - { uint8_t(Ext::kFCMA) , 14 }, // HWCAP_FCMA - { uint8_t(Ext::kLRCPC) , 15 }, // HWCAP_LRCPC - { uint8_t(Ext::kDPB) , 16 }, // HWCAP_DCPOP - { uint8_t(Ext::kSHA3) , 17 }, // HWCAP_SHA3 - { uint8_t(Ext::kSM3) , 18 }, // HWCAP_SM3 - { uint8_t(Ext::kSM4) , 19 }, // HWCAP_SM4 - { uint8_t(Ext::kDOTPROD) , 20 }, // HWCAP_ASIMDDP - { uint8_t(Ext::kSHA512) , 21 }, // HWCAP_SHA512 - { uint8_t(Ext::kSVE) , 22 }, // HWCAP_SVE - { uint8_t(Ext::kFHM) , 23 }, // HWCAP_ASIMDFHM - { uint8_t(Ext::kDIT) , 24 }, // HWCAP_DIT - { uint8_t(Ext::kLSE2) , 25 }, // HWCAP_USCAT - { uint8_t(Ext::kLRCPC2) , 26 }, // HWCAP_ILRCPC - { uint8_t(Ext::kFLAGM) , 27 }, // HWCAP_FLAGM - { uint8_t(Ext::kSSBS) , 28 }, // HWCAP_SSBS - { uint8_t(Ext::kSB) , 29 } // HWCAP_SB - /* - { uint8_t(Ext::k) , 30 }, // HWCAP_PACA - { uint8_t(Ext::k) , 31 } // HWCAP_PACG - */ -}; +static constexpr HWCapMapping64 hwCap1Mapping = {{ + uint8_t(Ext::kFP) , // [ 0] HWCAP_FP + uint8_t(Ext::kASIMD) , // [ 1] HWCAP_ASIMD + uint8_t(0xFF) , // [ 2] HWCAP_EVTSTRM + uint8_t(Ext::kAES) , // [ 3] HWCAP_AES + uint8_t(Ext::kPMULL) , // [ 4] HWCAP_PMULL + uint8_t(Ext::kSHA1) , // [ 5] HWCAP_SHA1 + uint8_t(Ext::kSHA256) , // [ 6] HWCAP_SHA2 + uint8_t(Ext::kCRC32) , // [ 7] HWCAP_CRC32 + uint8_t(Ext::kLSE) , // [ 8] HWCAP_ATOMICS + uint8_t(Ext::kFP16CONV) , // [ 9] HWCAP_FPHP + uint8_t(Ext::kFP16) , // [10] HWCAP_ASIMDHP + uint8_t(Ext::kCPUID) , // [11] HWCAP_CPUID + uint8_t(Ext::kRDM) , // [12] HWCAP_ASIMDRDM + uint8_t(Ext::kJSCVT) , // [13] HWCAP_JSCVT + uint8_t(Ext::kFCMA) , // [14] HWCAP_FCMA + uint8_t(Ext::kLRCPC) , // [15] HWCAP_LRCPC + uint8_t(Ext::kDPB) , // [16] HWCAP_DCPOP + uint8_t(Ext::kSHA3) , // [17] HWCAP_SHA3 + uint8_t(Ext::kSM3) , // [18] HWCAP_SM3 + uint8_t(Ext::kSM4) , // [19] HWCAP_SM4 + uint8_t(Ext::kDOTPROD) , // [20] HWCAP_ASIMDDP + uint8_t(Ext::kSHA512) , // [21] HWCAP_SHA512 + uint8_t(Ext::kSVE) , // [22] HWCAP_SVE + uint8_t(Ext::kFHM) , // [23] HWCAP_ASIMDFHM + uint8_t(Ext::kDIT) , // [24] HWCAP_DIT + uint8_t(Ext::kLSE2) , // [25] HWCAP_USCAT + uint8_t(Ext::kLRCPC2) , // [26] HWCAP_ILRCPC + uint8_t(Ext::kFLAGM) , // [27] HWCAP_FLAGM + uint8_t(Ext::kSSBS) , // [28] HWCAP_SSBS + uint8_t(Ext::kSB) , // [29] HWCAP_SB + uint8_t(0xFF) , // [30] HWCAP_PACA + uint8_t(0xFF) , // [31] HWCAP_PACG + uint8_t(Ext::kGCS) , // [32] HWCAP_GCS + uint8_t(Ext::kCMPBR) , // [33] HWCAP_CMPBR + uint8_t(Ext::kFPRCVT) , // [34] HWCAP_FPRCVT + uint8_t(Ext::kF8F32MM) , // [35] HWCAP_F8MM8 + uint8_t(Ext::kF8F16MM) , // [36] HWCAP_F8MM4 + uint8_t(Ext::kSVE_F16MM) , // [37] HWCAP_SVE_F16MM + uint8_t(Ext::kSVE_ELTPERM) , // [38] HWCAP_SVE_ELTPERM + uint8_t(Ext::kSVE_AES2) , // [39] HWCAP_SVE_AES2 + uint8_t(Ext::kSVE_BFSCALE) , // [40] HWCAP_SVE_BFSCALE + uint8_t(Ext::kSVE2_2) , // [41] HWCAP_SVE2P2 + uint8_t(Ext::kSME2_2) , // [42] HWCAP_SME2P2 + uint8_t(Ext::kSSVE_BITPERM) , // [43] HWCAP_SME_SBITPERM + uint8_t(Ext::kSME_AES) , // [44] HWCAP_SME_AES + uint8_t(Ext::kSSVE_FEXPA) , // [45] HWCAP_SME_SFEXPA + uint8_t(Ext::kSME_TMOP) , // [46] HWCAP_SME_STMOP + uint8_t(Ext::kSME_MOP4) , // [47] HWCAP_SME_SMOP4 + uint8_t(0xFF) , // [48] + uint8_t(0xFF) , // [49] + uint8_t(0xFF) , // [50] + uint8_t(0xFF) , // [51] + uint8_t(0xFF) , // [52] + uint8_t(0xFF) , // [53] + uint8_t(0xFF) , // [54] + uint8_t(0xFF) , // [55] + uint8_t(0xFF) , // [56] + uint8_t(0xFF) , // [57] + uint8_t(0xFF) , // [58] + uint8_t(0xFF) , // [59] + uint8_t(0xFF) , // [60] + uint8_t(0xFF) , // [61] + uint8_t(0xFF) , // [62] + uint8_t(0xFF) // [63] +}}; -static const HWCapMapping hwCap2Mapping[] = { - { uint8_t(Ext::kDPB2) , 0 }, // HWCAP2_DCPODP - { uint8_t(Ext::kSVE2) , 1 }, // HWCAP2_SVE2 - { uint8_t(Ext::kSVE_AES) , 2 }, // HWCAP2_SVEAES - { uint8_t(Ext::kSVE_PMULL128) , 3 }, // HWCAP2_SVEPMULL - { uint8_t(Ext::kSVE_BITPERM) , 4 }, // HWCAP2_SVEBITPERM - { uint8_t(Ext::kSVE_SHA3) , 5 }, // HWCAP2_SVESHA3 - { uint8_t(Ext::kSVE_SM4) , 6 }, // HWCAP2_SVESM4 - { uint8_t(Ext::kFLAGM2) , 7 }, // HWCAP2_FLAGM2 - { uint8_t(Ext::kFRINTTS) , 8 }, // HWCAP2_FRINT - { uint8_t(Ext::kSVE_I8MM) , 9 }, // HWCAP2_SVEI8MM - { uint8_t(Ext::kSVE_F32MM) , 10 }, // HWCAP2_SVEF32MM - { uint8_t(Ext::kSVE_F64MM) , 11 }, // HWCAP2_SVEF64MM - { uint8_t(Ext::kSVE_BF16) , 12 }, // HWCAP2_SVEBF16 - { uint8_t(Ext::kI8MM) , 13 }, // HWCAP2_I8MM - { uint8_t(Ext::kBF16) , 14 }, // HWCAP2_BF16 - { uint8_t(Ext::kDGH) , 15 }, // HWCAP2_DGH - { uint8_t(Ext::kRNG) , 16 }, // HWCAP2_RNG - { uint8_t(Ext::kBTI) , 17 }, // HWCAP2_BTI - { uint8_t(Ext::kMTE) , 18 }, // HWCAP2_MTE - { uint8_t(Ext::kECV) , 19 }, // HWCAP2_ECV - { uint8_t(Ext::kAFP) , 20 }, // HWCAP2_AFP - { uint8_t(Ext::kRPRES) , 21 }, // HWCAP2_RPRES - { uint8_t(Ext::kMTE3) , 22 }, // HWCAP2_MTE3 - { uint8_t(Ext::kSME) , 23 }, // HWCAP2_SME - { uint8_t(Ext::kSME_I16I64) , 24 }, // HWCAP2_SME_I16I64 - { uint8_t(Ext::kSME_F64F64) , 25 }, // HWCAP2_SME_F64F64 - { uint8_t(Ext::kSME_I8I32) , 26 }, // HWCAP2_SME_I8I32 - { uint8_t(Ext::kSME_F16F32) , 27 }, // HWCAP2_SME_F16F32 - { uint8_t(Ext::kSME_B16F32) , 28 }, // HWCAP2_SME_B16F32 - { uint8_t(Ext::kSME_F32F32) , 29 }, // HWCAP2_SME_F32F32 - { uint8_t(Ext::kSME_FA64) , 30 }, // HWCAP2_SME_FA64 - { uint8_t(Ext::kWFXT) , 31 }, // HWCAP2_WFXT - { uint8_t(Ext::kEBF16) , 32 }, // HWCAP2_EBF16 - { uint8_t(Ext::kSVE_EBF16) , 33 }, // HWCAP2_SVE_EBF16 - { uint8_t(Ext::kCSSC) , 34 }, // HWCAP2_CSSC - { uint8_t(Ext::kRPRFM) , 35 }, // HWCAP2_RPRFM - { uint8_t(Ext::kSVE2_1) , 36 }, // HWCAP2_SVE2P1 - { uint8_t(Ext::kSME2) , 37 }, // HWCAP2_SME2 - { uint8_t(Ext::kSME2_1) , 38 }, // HWCAP2_SME2P1 - { uint8_t(Ext::kSME_I16I32) , 39 }, // HWCAP2_SME_I16I32 - { uint8_t(Ext::kSME_BI32I32) , 40 }, // HWCAP2_SME_BI32I32 - { uint8_t(Ext::kSME_B16B16) , 41 }, // HWCAP2_SME_B16B16 - { uint8_t(Ext::kSME_F16F16) , 42 } // HWCAP2_SME_F16F16 -}; +static constexpr HWCapMapping64 hwCap2Mapping = {{ + uint8_t(Ext::kDPB2) , // [ 0] HWCAP2_DCPODP + uint8_t(Ext::kSVE2) , // [ 1] HWCAP2_SVE2 + uint8_t(Ext::kSVE_AES) , // [ 2] HWCAP2_SVEAES + uint8_t(Ext::kSVE_PMULL128) , // [ 3] HWCAP2_SVEPMULL + uint8_t(Ext::kSVE_BITPERM) , // [ 4] HWCAP2_SVEBITPERM + uint8_t(Ext::kSVE_SHA3) , // [ 5] HWCAP2_SVESHA3 + uint8_t(Ext::kSVE_SM4) , // [ 6] HWCAP2_SVESM4 + uint8_t(Ext::kFLAGM2) , // [ 7] HWCAP2_FLAGM2 + uint8_t(Ext::kFRINTTS) , // [ 8] HWCAP2_FRINT + uint8_t(Ext::kSVE_I8MM) , // [ 9] HWCAP2_SVEI8MM + uint8_t(Ext::kSVE_F32MM) , // [10] HWCAP2_SVEF32MM + uint8_t(Ext::kSVE_F64MM) , // [11] HWCAP2_SVEF64MM + uint8_t(Ext::kSVE_BF16) , // [12] HWCAP2_SVEBF16 + uint8_t(Ext::kI8MM) , // [13] HWCAP2_I8MM + uint8_t(Ext::kBF16) , // [14] HWCAP2_BF16 + uint8_t(Ext::kDGH) , // [15] HWCAP2_DGH + uint8_t(Ext::kRNG) , // [16] HWCAP2_RNG + uint8_t(Ext::kBTI) , // [17] HWCAP2_BTI + uint8_t(Ext::kMTE) , // [18] HWCAP2_MTE + uint8_t(Ext::kECV) , // [19] HWCAP2_ECV + uint8_t(Ext::kAFP) , // [20] HWCAP2_AFP + uint8_t(Ext::kRPRES) , // [21] HWCAP2_RPRES + uint8_t(Ext::kMTE3) , // [22] HWCAP2_MTE3 + uint8_t(Ext::kSME) , // [23] HWCAP2_SME + uint8_t(Ext::kSME_I16I64) , // [24] HWCAP2_SME_I16I64 + uint8_t(Ext::kSME_F64F64) , // [25] HWCAP2_SME_F64F64 + uint8_t(Ext::kSME_I8I32) , // [26] HWCAP2_SME_I8I32 + uint8_t(Ext::kSME_F16F32) , // [27] HWCAP2_SME_F16F32 + uint8_t(Ext::kSME_B16F32) , // [28] HWCAP2_SME_B16F32 + uint8_t(Ext::kSME_F32F32) , // [29] HWCAP2_SME_F32F32 + uint8_t(Ext::kSME_FA64) , // [30] HWCAP2_SME_FA64 + uint8_t(Ext::kWFXT) , // [31] HWCAP2_WFXT + uint8_t(Ext::kEBF16) , // [32] HWCAP2_EBF16 + uint8_t(Ext::kSVE_EBF16) , // [33] HWCAP2_SVE_EBF16 + uint8_t(Ext::kCSSC) , // [34] HWCAP2_CSSC + uint8_t(Ext::kRPRFM) , // [35] HWCAP2_RPRFM + uint8_t(Ext::kSVE2_1) , // [36] HWCAP2_SVE2P1 + uint8_t(Ext::kSME2) , // [37] HWCAP2_SME2 + uint8_t(Ext::kSME2_1) , // [38] HWCAP2_SME2P1 + uint8_t(Ext::kSME_I16I32) , // [39] HWCAP2_SME_I16I32 + uint8_t(Ext::kSME_BI32I32) , // [40] HWCAP2_SME_BI32I32 + uint8_t(Ext::kSME_B16B16) , // [41] HWCAP2_SME_B16B16 + uint8_t(Ext::kSME_F16F16) , // [42] HWCAP2_SME_F16F16 + uint8_t(Ext::kMOPS) , // [43] HWCAP2_MOPS + uint8_t(Ext::kHBC) , // [44] HWCAP2_HBC + uint8_t(Ext::kSVE_B16B16) , // [45] HWCAP2_SVE_B16B16 + uint8_t(Ext::kLRCPC3) , // [46] HWCAP2_LRCPC3 + uint8_t(Ext::kLSE128) , // [47] HWCAP2_LSE128 + uint8_t(Ext::kFPMR) , // [48] HWCAP2_FPMR + uint8_t(Ext::kLUT) , // [49] HWCAP2_LUT + uint8_t(Ext::kFAMINMAX) , // [50] HWCAP2_FAMINMAX + uint8_t(Ext::kFP8) , // [51] HWCAP2_F8CVT + uint8_t(Ext::kFP8FMA) , // [52] HWCAP2_F8FMA + uint8_t(Ext::kFP8DOT4) , // [53] HWCAP2_F8DP4 + uint8_t(Ext::kFP8DOT2) , // [54] HWCAP2_F8DP2 + uint8_t(Ext::kF8E4M3) , // [55] HWCAP2_F8E4M3 + uint8_t(Ext::kF8E5M2) , // [56] HWCAP2_F8E5M2 + uint8_t(Ext::kSME_LUTv2) , // [57] HWCAP2_SME_LUTV2 + uint8_t(Ext::kSME_F8F16) , // [58] HWCAP2_SME_F8F16 + uint8_t(Ext::kSME_F8F32) , // [59] HWCAP2_SME_F8F32 + uint8_t(Ext::kSSVE_FP8FMA) , // [60] HWCAP2_SME_SF8FMA + uint8_t(Ext::kSSVE_FP8DOT4) , // [61] HWCAP2_SME_SF8DP4 + uint8_t(Ext::kSSVE_FP8DOT2) , // [62] HWCAP2_SME_SF8DP2 + uint8_t(0xFF) // [63] HWCAP2_POE +}}; static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { cpu._wasDetected = true; @@ -1687,8 +1948,8 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { unsigned long hwCapMasks[2] {}; getAuxValues(hwCapMasks, hwCapTags, 2u); - mergeHWCaps(cpu, hwCapMasks[0], hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); - mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping)); + mergeHWCaps(cpu, hwCapMasks[0], hwCap1Mapping); + mergeHWCaps(cpu, hwCapMasks[1], hwCap2Mapping); #if defined(ASMJIT_ARM_DETECT_VIA_CPUID) if (cpu.features().arm().hasCPUID()) { @@ -1702,6 +1963,87 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { #endif // ASMJIT_ARCH_ARM +// CpuInfo - Detect - ARM - Detect by NetBSD API That Reads CPUID +// ============================================================== + +#elif defined(__NetBSD__) && ASMJIT_ARCH_ARM >= 64 + +//! Position of AArch64 registers in a`aarch64_sysctl_cpu_id` struct, which is filled by sysctl(). +struct NetBSDAArch64Regs { + enum ID : uint32_t { + k64_MIDR = 0, //!< Main ID Register. + k64_REVIDR = 8, //!< Revision ID Register. + k64_MPIDR = 16, //!< Multiprocessor Affinity Register. + k64_AA64DFR0 = 24, //!< A64 Debug Feature Register 0. + k64_AA64DFR1 = 32, //!< A64 Debug Feature Register 1. + k64_AA64ISAR0 = 40, //!< A64 Instruction Set Attribute Register 0. + k64_AA64ISAR1 = 48, //!< A64 Instruction Set Attribute Register 1. + k64_AA64MMFR0 = 56, //!< A64 Memory Model Feature Register 0. + k64_AA64MMFR1 = 64, //!< A64 Memory Model Feature Register 1. + k64_AA64MMFR2 = 72, //!< A64 Memory Model Feature Register 2. + k64_AA64PFR0 = 80, //!< A64 Processor Feature Register 0. + k64_AA64PFR1 = 88, //!< A64 Processor Feature Register 1. + k64_AA64ZFR0 = 96, //!< A64 SVE Feature ID Register 0. + k32_MVFR0 = 104, //!< Media and VFP Feature Register 0. + k32_MVFR1 = 108, //!< Media and VFP Feature Register 1. + k32_MVFR2 = 112, //!< Media and VFP Feature Register 2. + k32_PAD = 116, //!< Padding (not used). + k64_CLIDR = 120, //!< Cache Level ID Register. + k64_CTR = 128 //!< Cache Type Register. + }; + + enum Limits : uint32_t { + kBufferSize = 136 + }; + + uint64_t data[kBufferSize / 8u]; + + ASMJIT_INLINE_NODEBUG uint64_t r64(uint32_t index) const noexcept { + ASMJIT_ASSERT(index % 8u == 0u); + return data[index / 8u]; + } + + ASMJIT_INLINE_NODEBUG uint32_t r32(uint32_t index) const noexcept { + ASMJIT_ASSERT(index % 4u == 0u); + uint32_t shift = (index % 8) * 8; + return uint32_t((r64(index) >> shift) & 0xFFFFFFFFu); + } +}; + +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + using Regs = NetBSDAArch64Regs; + + populateBaseARMFeatures(cpu); + + Regs regs {}; + size_t len = sizeof(regs); + const char sysctlCpuPath[] = "machdep.cpu0.cpu_id"; + + if (sysctlbyname(sysctlCpuPath, ®s, &len, nullptr, 0) == 0) { + detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu, regs.r64(Regs::k64_AA64PFR0), regs.r64(Regs::k64_AA64PFR1)); + detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu, regs.r64(Regs::k64_AA64ISAR0), regs.r64(Regs::k64_AA64ISAR1)); + + // TODO: AA64ISAR2 should be added when it's provided by NetBSD. + // detectAArch64FeaturesViaCPUID_AA64ISAR2(cpu, regs.r64Regs::k64_AA64ISAR2)); + + detectAArch64FeaturesViaCPUID_AA64MMFR0(cpu, regs.r64(Regs::k64_AA64MMFR0)); + detectAArch64FeaturesViaCPUID_AA64MMFR1(cpu, regs.r64(Regs::k64_AA64MMFR1)); + detectAArch64FeaturesViaCPUID_AA64MMFR2(cpu, regs.r64(Regs::k64_AA64MMFR2)); + + // Only read CPU_ID_AA64ZFR0 when either SVE or SME is available. + if (cpu.features().arm().hasAny(Ext::kSVE, Ext::kSME)) { + detectAArch64FeaturesViaCPUID_AA64ZFR0(cpu, regs.r64(Regs::k64_AA64ZFR0)); + + // TODO: AA64SMFR0 should be added when it's provided by NetBSD. + // if (cpu.features().arm().hasSME()) { + // detectAArch64FeaturesViaCPUID_AA64SMFR0(cpu, regs.r64(Regs::k64_kAA64SMFR0)); + // } + } + } + + postProcessARMCpuInfo(cpu); +} + // CpuInfo - Detect - ARM - Detect by OpenBSD API That Reads CPUID // =============================================================== @@ -1731,18 +2073,12 @@ static uint64_t openbsdReadAArch64CPUID(OpenBSDAArch64CPUID id) noexcept { } static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { - typedef OpenBSDAArch64CPUID ID; + using ID = OpenBSDAArch64CPUID; populateBaseARMFeatures(cpu); - detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu, - openbsdReadAArch64CPUID(ID::kAA64PFR0), - openbsdReadAArch64CPUID(ID::kAA64PFR1)); - - detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu, - openbsdReadAArch64CPUID(ID::kAA64ISAR0), - openbsdReadAArch64CPUID(ID::kAA64ISAR1)); - + detectAArch64FeaturesViaCPUID_AA64PFR0_AA64PFR1(cpu, openbsdReadAArch64CPUID(ID::kAA64PFR0), openbsdReadAArch64CPUID(ID::kAA64PFR1)); + detectAArch64FeaturesViaCPUID_AA64ISAR0_AA64ISAR1(cpu, openbsdReadAArch64CPUID(ID::kAA64ISAR0), openbsdReadAArch64CPUID(ID::kAA64ISAR1)); detectAArch64FeaturesViaCPUID_AA64ISAR2(cpu, openbsdReadAArch64CPUID(ID::kAA64ISAR2)); detectAArch64FeaturesViaCPUID_AA64MMFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64MMFR0)); detectAArch64FeaturesViaCPUID_AA64MMFR1(cpu, openbsdReadAArch64CPUID(ID::kAA64MMFR1)); @@ -1751,6 +2087,10 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { // Only read CPU_ID_AA64ZFR0 when either SVE or SME is available. if (cpu.features().arm().hasAny(Ext::kSVE, Ext::kSME)) { detectAArch64FeaturesViaCPUID_AA64ZFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64ZFR0)); + + if (cpu.features().arm().hasSME()) { + detectAArch64FeaturesViaCPUID_AA64SMFR0(cpu, openbsdReadAArch64CPUID(ID::kAA64SMFR0)); + } } postProcessARMCpuInfo(cpu); @@ -1792,15 +2132,16 @@ static ASMJIT_FAVOR_SIZE long appleDetectARMFeatureViaSysctl(AppleFeatureType ty memcpy(sysctlName + prefixSize, featureName, featureNameSize + 1u); // Include NULL terminator. long val = 0; - if (appleSysctlByName(sysctlName, &val)) + if (appleSysctlByName(sysctlName, &val)) { return val; + } } return 0; } static ASMJIT_FAVOR_SIZE void appleDetectARMFeaturesViaSysctl(CpuInfo& cpu) noexcept { - typedef AppleFeatureType FT; + using FT = AppleFeatureType; // Based on: // - https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics @@ -1872,8 +2213,9 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { memcpy(cpu._vendor.str, "APPLE", 6); bool cpuFeaturesPopulated = detectARMFeaturesViaAppleFamilyId(cpu); - if (!cpuFeaturesPopulated) + if (!cpuFeaturesPopulated) { appleDetectARMFeaturesViaSysctl(cpu); + } postProcessARMCpuInfo(cpu); } @@ -1883,7 +2225,7 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { #else #if ASMJIT_ARCH_ARM == 32 - #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with ARM CPU)") + #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with AArch32 CPU)") #else #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with AArch64 CPU)") #endif @@ -1902,13 +2244,13 @@ static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { // CpuInfo - Detect - Host // ======================= -static uint32_t cpuInfoInitialized; -static CpuInfo cpuInfoGlobal(Globals::NoInit); - const CpuInfo& CpuInfo::host() noexcept { - // This should never cause a problem as the resulting information should always be the same. - // In the worst case it would just be overwritten non-atomically. - if (!cpuInfoInitialized) { + static std::atomic cpuInfoInitialized; + static CpuInfo cpuInfoGlobal(Globals::NoInit); + + // This should never cause a problem as the resulting information should always + // be the same. In the worst case it would just be overwritten non-atomically. + if (!cpuInfoInitialized.load(std::memory_order_relaxed)) { CpuInfo cpuInfoLocal; cpuInfoLocal._arch = Arch::kHost; @@ -1918,13 +2260,11 @@ const CpuInfo& CpuInfo::host() noexcept { x86::detectX86Cpu(cpuInfoLocal); #elif ASMJIT_ARCH_ARM arm::detectARMCpu(cpuInfoLocal); -#else - #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown CPU)") #endif cpuInfoLocal._hwThreadCount = detectHWThreadCount(); cpuInfoGlobal = cpuInfoLocal; - cpuInfoInitialized = 1; + cpuInfoInitialized.store(1, std::memory_order_seq_cst); } return cpuInfoGlobal; diff --git a/pe-packer/asmjit/core/cpuinfo.h b/pe-packer/asmjit/core/cpuinfo.h index 8ddc1e1..0ad1411 100644 --- a/pe-packer/asmjit/core/cpuinfo.h +++ b/pe-packer/asmjit/core/cpuinfo.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_CPUINFO_H_INCLUDED @@ -22,23 +22,28 @@ ASMJIT_BEGIN_NAMESPACE //! Each feature is represented by a single bit in an embedded bit array. class CpuFeatures { public: - //! A word that is used to represents feature bits. - typedef Support::BitWord BitWord; - //! Iterator that can iterate all CPU features set. - typedef Support::BitVectorIterator Iterator; - //! \name Constants //! \{ //! \cond INTERNAL - enum : uint32_t { - kMaxFeatures = 256, - kNumBitWords = kMaxFeatures / Support::kBitWordSizeInBits - }; + static inline constexpr uint32_t kMaxFeatures = 256; + static inline constexpr uint32_t kNumBitWords = kMaxFeatures / Support::kBitWordSizeInBits; //! \endcond //! \} + //! \name Types + //! \{ + + //! A word that is used to represents feature bits. + using BitWord = Support::BitWord; + //! Iterator that can iterate all CPU features set. + using Iterator = Support::BitVectorIterator; + + using Bits = Support::Array; + + //! \} + //! \name Data //! \{ @@ -48,15 +53,18 @@ public: //! \{ //! Data bits. - Support::Array _bits; + Bits _bits; //! \} //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_NODEBUG bool operator==(const Data& other) noexcept { return eq(other); } - ASMJIT_INLINE_NODEBUG bool operator!=(const Data& other) noexcept { return !eq(other); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool operator==(const Data& other) const noexcept { return equals(other); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool operator!=(const Data& other) const noexcept { return !equals(other); } //! \} @@ -64,21 +72,28 @@ public: //! \{ //! Returns true if there are no features set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { return _bits.aggregate(0) == 0; } //! Returns all features as array of bitwords (see \ref Support::BitWord). + [[nodiscard]] ASMJIT_INLINE_NODEBUG BitWord* bits() noexcept { return _bits.data(); } + //! Returns all features as array of bitwords (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const BitWord* bits() const noexcept { return _bits.data(); } //! Returns the number of BitWords returned by \ref bits(). + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t bitWordCount() const noexcept { return kNumBitWords; } //! Returns \ref Support::BitVectorIterator, that can be used to iterate over all features efficiently. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Iterator iterator() const noexcept { return Iterator(_bits.data(), kNumBitWords); } //! Tests whether the feature `featureId` is present. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool has(const FeatureId& featureId) const noexcept { ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); @@ -88,17 +103,25 @@ public: return bool((_bits[idx] >> bit) & 0x1); } + //! \cond NONE template + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAny(const FeatureId& featureId) const noexcept { return has(featureId); } + //! \endcond + //! Tests whether any feature given is present. + //! + //! \note This is a variadic function template that can be used with multiple features. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAny(const FeatureId& featureId, Args&&... otherFeatureIds) const noexcept { return bool(unsigned(has(featureId)) | unsigned(hasAny(std::forward(otherFeatureIds)...))); } //! Tests whether all features as defined by `other` are present. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAll(const Data& other) const noexcept { uint32_t result = 1; for (uint32_t i = 0; i < kNumBitWords; i++) @@ -111,11 +134,12 @@ public: //! \name Manipulation //! \{ + //! Clears all features set. ASMJIT_INLINE_NODEBUG void reset() noexcept { _bits.fill(0); } //! Adds the given CPU `featureId` to the list of features. template - ASMJIT_INLINE_NODEBUG void add(const FeatureId& featureId) noexcept { + inline void add(const FeatureId& featureId) noexcept { ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; @@ -125,13 +149,13 @@ public: } template - ASMJIT_INLINE_NODEBUG void add(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + inline void add(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { add(featureId); add(std::forward(otherFeatureIds)...); } template - ASMJIT_INLINE_NODEBUG void addIf(bool condition, const FeatureId& featureId) noexcept { + inline void addIf(bool condition, const FeatureId& featureId) noexcept { ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; @@ -141,14 +165,14 @@ public: } template - ASMJIT_INLINE_NODEBUG void addIf(bool condition, const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + inline void addIf(bool condition, const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { addIf(condition, featureId); addIf(condition, std::forward(otherFeatureIds)...); } //! Removes the given CPU `featureId` from the list of features. template - ASMJIT_INLINE_NODEBUG void remove(const FeatureId& featureId) noexcept { + inline void remove(const FeatureId& featureId) noexcept { ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; @@ -158,16 +182,15 @@ public: } template - ASMJIT_INLINE_NODEBUG void remove(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + inline void remove(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { remove(featureId); remove(std::forward(otherFeatureIds)...); } //! Tests whether this CPU features data matches `other`. - ASMJIT_INLINE_NODEBUG bool eq(const Data& other) const noexcept { return _bits == other._bits; } + ASMJIT_INLINE_NODEBUG bool equals(const Data& other) const noexcept { return _bits == other._bits; } //! \} - }; //! X86 specific features data. @@ -179,42 +202,9 @@ public: kMT, //!< CPU has multi-threading capabilities. kNX, //!< CPU has Not-Execute-Bit aka DEP (data-execution prevention). - k3DNOW, //!< CPU has 3DNOW (3DNOW base instructions) {AMD} (deprecated). - k3DNOW2, //!< CPU has 3DNOW2 (enhanced 3DNOW) {AMD} (deprecated). kADX, //!< CPU has ADX (multi-precision add-carry instruction extensions). - kAESNI, //!< CPU has AESNI (AES encode/decode instructions). kALTMOVCR8, //!< CPU has LOCK MOV R<->CR0 (supports `MOV R<->CR8` via `LOCK MOV R<->CR0` in 32-bit mode) {AMD}. - kAMX_BF16, //!< CPU has AMX_BF16 (AMX-BF16 instructions). - kAMX_COMPLEX, //!< CPU has AMX_COMPLEX (AMX-COMPLEX instructions). - kAMX_FP16, //!< CPU has AMX_FP16 (AMX-FP16 instructions). - kAMX_INT8, //!< CPU has AMX_INT8 (AMX-INT8 instructions). - kAMX_TILE, //!< CPU has AMX_TILE (advanced matrix extensions). kAPX_F, //!< CPU has APX_F (advanced performance extensions - 32 GP registers, REX2 prefix, ...) {X86_64}. - kAVX, //!< CPU has AVX (advanced vector extensions). - kAVX2, //!< CPU has AVX2 (advanced vector extensions 2). - kAVX512_4FMAPS, //!< CPU has AVX512_FMAPS (FMA packed single). - kAVX512_4VNNIW, //!< CPU has AVX512_VNNIW (vector NN instructions word variable precision). - kAVX512_BF16, //!< CPU has AVX512_BF16 (AVX512 BFLOAT16 support instructions). - kAVX512_BITALG, //!< CPU has AVX512_BITALG (AVX512 VPOPCNT[B|W] and VPSHUFBITQMB instructions). - kAVX512_BW, //!< CPU has AVX512_BW (AVX512 integer BYTE|WORD instructions). - kAVX512_CD, //!< CPU has AVX512_CD (AVX512 conflict detection DWORD|QWORD instructions). - kAVX512_DQ, //!< CPU has AVX512_DQ (AVX512 integer DWORD|QWORD instructions). - kAVX512_ER, //!< CPU has AVX512_ER (AVX512 exponential and reciprocal instructions). - kAVX512_F, //!< CPU has AVX512_F (AVX512 foundation). - kAVX512_FP16, //!< CPU has AVX512_FP16 (AVX512 FP16 instructions). - kAVX512_IFMA, //!< CPU has AVX512_IFMA (AVX512 integer fused-multiply-add using 52-bit precision). - kAVX512_PF, //!< CPU has AVX512_PF (AVX512 prefetch instructions). - kAVX512_VBMI, //!< CPU has AVX512_VBMI (AVX152 vector byte manipulation instructions). - kAVX512_VBMI2, //!< CPU has AVX512_VBMI2 (AVX512 vector byte manipulation instructions v2). - kAVX512_VL, //!< CPU has AVX512_VL (AVX512 vector length extensions). - kAVX512_VNNI, //!< CPU has AVX512_VNNI (AVX512 vector neural network instructions). - kAVX512_VP2INTERSECT, //!< CPU has AVX512_VP2INTERSECT - kAVX512_VPOPCNTDQ, //!< CPU has AVX512_VPOPCNTDQ (AVX512 VPOPCNT[D|Q] instructions). - kAVX_IFMA, //!< CPU has AVX_IFMA (AVX/VEX encoding of vpmadd52huq/vpmadd52luq). - kAVX_NE_CONVERT, //!< CPU has AVX_NE_CONVERT. - kAVX_VNNI, //!< CPU has AVX_VNNI (AVX/VEX encoding of vpdpbusd/vpdpbusds/vpdpwssd/vpdpwssds). - kAVX_VNNI_INT16, //!< CPU has AVX_VNNI_INT16. - kAVX_VNNI_INT8, //!< CPU has AVX_VNNI_INT8. kBMI, //!< CPU has BMI (bit manipulation instructions #1). kBMI2, //!< CPU has BMI2 (bit manipulation instructions #2). kCET_IBT, //!< CPU has CET-IBT (indirect branch tracking). @@ -232,10 +222,6 @@ public: kENCLV, //!< CPU has ENCLV. kENQCMD, //!< CPU has ENQCMD (enqueue stores). kERMS, //!< CPU has ERMS (enhanced REP MOVSB/STOSB). - kF16C, //!< CPU has F16C (AVX FP16 conversion instructions). - kFMA, //!< CPU has FMA (AVX fused-multiply-add - 3 operand form). - kFMA4, //!< CPU has FMA4 (AVX fused-multiply-add - 4 operand form) (deprecated). - kFPU, //!< CPU has FPU (FPU support). kFSGSBASE, //!< CPU has FSGSBASE. kFSRM, //!< CPU has FSRM (fast short REP MOVSB). kFSRC, //!< CPU has FSRC (fast short REP CMPSB|SCASB). @@ -243,9 +229,6 @@ public: kFXSR, //!< CPU has FXSR (FXSAVE/FXRSTOR instructions). kFXSROPT, //!< CPU has FXSROTP (FXSAVE/FXRSTOR is optimized). kFZRM, //!< CPU has FZRM (fast zero-length REP MOVSB). - kGEODE, //!< CPU has GEODE extensions (GEODE 3DNOW additions) (deprecated). - kGFNI, //!< CPU has GFNI (galois field instructions). - kHLE, //!< CPU has HLE. kHRESET, //!< CPU has HRESET. kI486, //!< CPU has I486 features (I486+ support). kINVLPGB, //!< CPU has INVLPGB. @@ -254,20 +237,19 @@ public: kLWP, //!< CPU has LWP (lightweight profiling) {AMD}. kLZCNT, //!< CPU has LZCNT (LZCNT instruction). kMCOMMIT, //!< CPU has MCOMMIT (MCOMMIT instruction). - kMMX, //!< CPU has MMX (MMX base instructions) (deprecated). - kMMX2, //!< CPU has MMX2 (MMX2 extensions or initial SSE extensions) (deprecated). kMONITOR, //!< CPU has MONITOR (MONITOR/MWAIT instructions). kMONITORX, //!< CPU has MONITORX (MONITORX/MWAITX instructions). kMOVBE, //!< CPU has MOVBE (move with byte-order swap). kMOVDIR64B, //!< CPU has MOVDIR64B (move 64 bytes as direct store). kMOVDIRI, //!< CPU has MOVDIRI (move dword/qword as direct store). + kMOVRS, //!< CPU has MOVRS (move from shared memory). kMPX, //!< CPU has MPX (memory protection extensions). kMSR, //!< CPU has MSR (RDMSR/WRMSR instructions). kMSRLIST, //!< CPU has MSRLIST. + kMSR_IMM, //!< CPU has MSR_IMM (RDMSR/WRMSR immediate encoding). kMSSE, //!< CPU has MSSE (misaligned SSE support). kOSXSAVE, //!< CPU has OSXSAVE (XSAVE enabled by OS). kOSPKE, //!< CPU has OSPKE (PKE enabled by OS). - kPCLMULQDQ, //!< CPU has PCLMULQDQ (packed carry-less multiplication). kPCONFIG, //!< CPU has PCONFIG (PCONFIG instruction). kPOPCNT, //!< CPU has POPCNT (POPCNT instruction). kPREFETCHI, //!< CPU has PREFETCHI. @@ -282,97 +264,114 @@ public: kRDSEED, //!< CPU has RDSEED (RDSEED instruction). kRDTSC, //!< CPU has RDTSC. kRDTSCP, //!< CPU has RDTSCP. - kRTM, //!< CPU has RTM. + kRTM, //!< CPU has RTM (RTM instructions - deprecated). kSEAM, //!< CPU has SEAM. kSERIALIZE, //!< CPU has SERIALIZE. kSEV, //!< CPU has SEV (secure encrypted virtualization). kSEV_ES, //!< CPU has SEV_ES (SEV encrypted state). kSEV_SNP, //!< CPU has SEV_SNP (SEV secure nested paging). - kSHA, //!< CPU has SHA (SHA-1 and SHA-256 instructions). - kSHA512, //!< CPU has SHA512 (SHA-512 instructions). kSKINIT, //!< CPU has SKINIT (SKINIT/STGI instructions) {AMD}. - kSM3, //!< CPU has SM3 (SM3 hash extensions). - kSM4, //!< CPU has SM4 (SM4 cipher extensions). kSMAP, //!< CPU has SMAP (supervisor-mode access prevention). - kSME , //!< CPU has SME (secure memory encryption). + kSME, //!< CPU has SME (secure memory encryption). kSMEP, //!< CPU has SMEP (supervisor-mode execution prevention). kSMX, //!< CPU has SMX (safer mode extensions). - kSSE, //!< CPU has SSE (SSE instructions). - kSSE2, //!< CPU has SSE2 (SSE2 instructions). - kSSE3, //!< CPU has SSE3 (SSE3 instructions). - kSSE4_1, //!< CPU has SSE4.1 (SSE4.1 instructions). - kSSE4_2, //!< CPU has SSE4.2 (SSE4.2 instructions). - kSSE4A, //!< CPU has SSE4A (SSE4.A instructions) {AMD} (deprecated). - kSSSE3, //!< CPU has SSSE3 (SSSE3 instructions). kSVM, //!< CPU has SVM (virtualization) {AMD}. kTBM, //!< CPU has TBM (trailing bit manipulation) {AMD}. kTSE, //!< CPU has TSE. - kTSX, //!< CPU has TSX. kTSXLDTRK, //!< CPU has TSXLDTRK. kUINTR, //!< CPU has UINTR (user interrupts). - kVAES, //!< CPU has VAES (vector AES 256|512 bit support). kVMX, //!< CPU has VMX (virtualization) {INTEL}. - kVPCLMULQDQ, //!< CPU has VPCLMULQDQ (vector PCLMULQDQ 256|512-bit support). kWAITPKG, //!< CPU has WAITPKG (UMONITOR, UMWAIT, TPAUSE). kWBNOINVD, //!< CPU has WBNOINVD. kWRMSRNS, //!< CPU has WRMSRNS. - kXOP, //!< CPU has XOP (XOP instructions) {AMD} (deprecated). kXSAVE, //!< CPU has XSAVE. kXSAVEC, //!< CPU has XSAVEC. kXSAVEOPT, //!< CPU has XSAVEOPT. kXSAVES, //!< CPU has XSAVES. + + kFPU, //!< CPU has FPU (FPU support). + kMMX, //!< CPU has MMX (MMX base instructions) (deprecated). + kMMX2, //!< CPU has MMX2 (MMX2 extensions or initial SSE extensions) (deprecated). + k3DNOW, //!< CPU has 3DNOW (3DNOW base instructions) {AMD} (deprecated). + k3DNOW2, //!< CPU has 3DNOW2 (enhanced 3DNOW) {AMD} (deprecated). + kGEODE, //!< CPU has GEODE extensions (GEODE 3DNOW additions) (deprecated). + + kSSE, //!< CPU has SSE (SSE instructions). + kSSE2, //!< CPU has SSE2 (SSE2 instructions). + kSSE3, //!< CPU has SSE3 (SSE3 instructions). + kSSSE3, //!< CPU has SSSE3 (SSSE3 instructions). + kSSE4_1, //!< CPU has SSE4.1 (SSE4.1 instructions). + kSSE4_2, //!< CPU has SSE4.2 (SSE4.2 instructions). + kSSE4A, //!< CPU has SSE4A (SSE4.A instructions) {AMD} (deprecated). + kPCLMULQDQ, //!< CPU has PCLMULQDQ (packed carry-less multiplication). + + kAVX, //!< CPU has AVX (advanced vector extensions). + kAVX2, //!< CPU has AVX2 (advanced vector extensions 2). + kAVX_IFMA, //!< CPU has AVX_IFMA (AVX/VEX encoding of vpmadd52huq/vpmadd52luq). + kAVX_NE_CONVERT, //!< CPU has AVX_NE_CONVERT. + kAVX_VNNI, //!< CPU has AVX_VNNI (AVX/VEX encoding of vpdpbusd/vpdpbusds/vpdpwssd/vpdpwssds). + kAVX_VNNI_INT16, //!< CPU has AVX_VNNI_INT16. + kAVX_VNNI_INT8, //!< CPU has AVX_VNNI_INT8. + kF16C, //!< CPU has F16C (AVX FP16 conversion instructions). + kFMA, //!< CPU has FMA (AVX fused-multiply-add - 3 operand form). + kFMA4, //!< CPU has FMA4 (AVX fused-multiply-add - 4 operand form) (deprecated). + kXOP, //!< CPU has XOP (XOP instructions) {AMD} (deprecated). + + kAVX512_BF16, //!< CPU has AVX512_BF16 (AVX512 BFLOAT16 support instructions). + kAVX512_BITALG, //!< CPU has AVX512_BITALG (AVX512 VPOPCNT[B|W] and VPSHUFBITQMB instructions). + kAVX512_BW, //!< CPU has AVX512_BW (AVX512 integer BYTE|WORD instructions). + kAVX512_CD, //!< CPU has AVX512_CD (AVX512 conflict detection DWORD|QWORD instructions). + kAVX512_DQ, //!< CPU has AVX512_DQ (AVX512 integer DWORD|QWORD instructions). + kAVX512_F, //!< CPU has AVX512_F (AVX512 foundation). + kAVX512_FP16, //!< CPU has AVX512_FP16 (AVX512 FP16 instructions). + kAVX512_IFMA, //!< CPU has AVX512_IFMA (AVX512 integer fused-multiply-add using 52-bit precision). + kAVX512_VBMI, //!< CPU has AVX512_VBMI (AVX512 vector byte manipulation instructions). + kAVX512_VBMI2, //!< CPU has AVX512_VBMI2 (AVX512 vector byte manipulation instructions v2). + kAVX512_VL, //!< CPU has AVX512_VL (AVX512 vector length extensions). + kAVX512_VNNI, //!< CPU has AVX512_VNNI (AVX512 vector neural network instructions). + kAVX512_VP2INTERSECT, //!< CPU has AVX512_VP2INTERSECT + kAVX512_VPOPCNTDQ, //!< CPU has AVX512_VPOPCNTDQ (AVX512 VPOPCNT[D|Q] instructions). + + kAESNI, //!< CPU has AESNI (AES encode/decode instructions). + kGFNI, //!< CPU has GFNI (galois field instructions). + kSHA, //!< CPU has SHA (SHA-1 and SHA-256 instructions). + kSHA512, //!< CPU has SHA512 (SHA-512 instructions). + kSM3, //!< CPU has SM3 (SM3 hash extensions). + kSM4, //!< CPU has SM4 (SM4 cipher extensions). + kVAES, //!< CPU has VAES (vector AES 256|512 bit support). + kVPCLMULQDQ, //!< CPU has VPCLMULQDQ (vector PCLMULQDQ 256|512-bit support). + + kKL, //!< CPU has KL (Key Locker). + kAESKLE, //!< CPU has AESKLE (AESKLE). + kAESKLEWIDE_KL, //!< CPU has AESKLE+WIDEKL+KL (AESKLE & WIDEKL instructions and KL enabled) + + kAVX10_1, //!< CPU has AVX10.1/512 (AVX10.1 with 512-bit vectors). + kAVX10_2, //!< CPU has AVX10.2/512 (AVX10.2 with 512-bit vectors). + + kAMX_AVX512, //!< CPU has AMX_AVX512 (AMX-AVX512 instructions). + kAMX_BF16, //!< CPU has AMX_BF16 (AMX-BF16 instructions). + kAMX_COMPLEX, //!< CPU has AMX_COMPLEX (AMX-COMPLEX instructions). + kAMX_FP16, //!< CPU has AMX_FP16 (AMX-FP16 instructions). + kAMX_FP8, //!< CPU has AMX_FP8 (AMX-FP8 instructions). + kAMX_INT8, //!< CPU has AMX_INT8 (AMX-INT8 instructions). + kAMX_MOVRS, //!< CPU has AMX_MOVRS (AMX-MOVRS instructions). + kAMX_TF32, //!< CPU has AMX_TF32 (AMX-TF32 instructions). + kAMX_TILE, //!< CPU has AMX_TILE (advanced matrix extensions). + kAMX_TRANSPOSE, //!< CPU has AMX_TRANSPOSE (AMX-TRANSPOSE instructions). // @EnumValuesEnd@ -#ifndef ASMJIT_NO_DEPRECATED - kAVX512_CDI = kAVX512_CD, - kAVX512_ERI = kAVX512_ER, - kAVX512_PFI = kAVX512_PF, -#endif - - kMaxValue = kXSAVES + kMaxValue = kAMX_TILE }; #define ASMJIT_X86_FEATURE(FEATURE) \ + /*! Tests whether FEATURE is present. */ \ ASMJIT_INLINE_NODEBUG bool has##FEATURE() const noexcept { return has(X86::k##FEATURE); } ASMJIT_X86_FEATURE(MT) ASMJIT_X86_FEATURE(NX) - ASMJIT_X86_FEATURE(3DNOW) - ASMJIT_X86_FEATURE(3DNOW2) ASMJIT_X86_FEATURE(ADX) - ASMJIT_X86_FEATURE(AESNI) ASMJIT_X86_FEATURE(ALTMOVCR8) - ASMJIT_X86_FEATURE(AMX_BF16) - ASMJIT_X86_FEATURE(AMX_COMPLEX) - ASMJIT_X86_FEATURE(AMX_FP16) - ASMJIT_X86_FEATURE(AMX_INT8) - ASMJIT_X86_FEATURE(AMX_TILE) ASMJIT_X86_FEATURE(APX_F) - ASMJIT_X86_FEATURE(AVX) - ASMJIT_X86_FEATURE(AVX2) - ASMJIT_X86_FEATURE(AVX512_4FMAPS) - ASMJIT_X86_FEATURE(AVX512_4VNNIW) - ASMJIT_X86_FEATURE(AVX512_BF16) - ASMJIT_X86_FEATURE(AVX512_BITALG) - ASMJIT_X86_FEATURE(AVX512_BW) - ASMJIT_X86_FEATURE(AVX512_CD) - ASMJIT_X86_FEATURE(AVX512_DQ) - ASMJIT_X86_FEATURE(AVX512_ER) - ASMJIT_X86_FEATURE(AVX512_F) - ASMJIT_X86_FEATURE(AVX512_FP16) - ASMJIT_X86_FEATURE(AVX512_IFMA) - ASMJIT_X86_FEATURE(AVX512_PF) - ASMJIT_X86_FEATURE(AVX512_VBMI) - ASMJIT_X86_FEATURE(AVX512_VBMI2) - ASMJIT_X86_FEATURE(AVX512_VL) - ASMJIT_X86_FEATURE(AVX512_VNNI) - ASMJIT_X86_FEATURE(AVX512_VP2INTERSECT) - ASMJIT_X86_FEATURE(AVX512_VPOPCNTDQ) - ASMJIT_X86_FEATURE(AVX_IFMA) - ASMJIT_X86_FEATURE(AVX_NE_CONVERT) - ASMJIT_X86_FEATURE(AVX_VNNI) - ASMJIT_X86_FEATURE(AVX_VNNI_INT16) - ASMJIT_X86_FEATURE(AVX_VNNI_INT8) ASMJIT_X86_FEATURE(BMI) ASMJIT_X86_FEATURE(BMI2) ASMJIT_X86_FEATURE(CET_IBT) @@ -389,10 +388,6 @@ public: ASMJIT_X86_FEATURE(ENCLV) ASMJIT_X86_FEATURE(ENQCMD) ASMJIT_X86_FEATURE(ERMS) - ASMJIT_X86_FEATURE(F16C) - ASMJIT_X86_FEATURE(FMA) - ASMJIT_X86_FEATURE(FMA4) - ASMJIT_X86_FEATURE(FPU) ASMJIT_X86_FEATURE(FSGSBASE) ASMJIT_X86_FEATURE(FSRM) ASMJIT_X86_FEATURE(FSRC) @@ -400,9 +395,6 @@ public: ASMJIT_X86_FEATURE(FXSR) ASMJIT_X86_FEATURE(FXSROPT) ASMJIT_X86_FEATURE(FZRM) - ASMJIT_X86_FEATURE(GEODE) - ASMJIT_X86_FEATURE(GFNI) - ASMJIT_X86_FEATURE(HLE) ASMJIT_X86_FEATURE(HRESET) ASMJIT_X86_FEATURE(I486) ASMJIT_X86_FEATURE(INVLPGB) @@ -411,20 +403,19 @@ public: ASMJIT_X86_FEATURE(LWP) ASMJIT_X86_FEATURE(LZCNT) ASMJIT_X86_FEATURE(MCOMMIT) - ASMJIT_X86_FEATURE(MMX) - ASMJIT_X86_FEATURE(MMX2) ASMJIT_X86_FEATURE(MONITOR) ASMJIT_X86_FEATURE(MONITORX) ASMJIT_X86_FEATURE(MOVBE) ASMJIT_X86_FEATURE(MOVDIR64B) ASMJIT_X86_FEATURE(MOVDIRI) + ASMJIT_X86_FEATURE(MOVRS) ASMJIT_X86_FEATURE(MPX) ASMJIT_X86_FEATURE(MSR) ASMJIT_X86_FEATURE(MSRLIST) + ASMJIT_X86_FEATURE(MSR_IMM) ASMJIT_X86_FEATURE(MSSE) ASMJIT_X86_FEATURE(OSXSAVE) ASMJIT_X86_FEATURE(OSPKE) - ASMJIT_X86_FEATURE(PCLMULQDQ) ASMJIT_X86_FEATURE(PCONFIG) ASMJIT_X86_FEATURE(POPCNT) ASMJIT_X86_FEATURE(PREFETCHI) @@ -445,46 +436,93 @@ public: ASMJIT_X86_FEATURE(SEV) ASMJIT_X86_FEATURE(SEV_ES) ASMJIT_X86_FEATURE(SEV_SNP) - ASMJIT_X86_FEATURE(SHA) ASMJIT_X86_FEATURE(SKINIT) ASMJIT_X86_FEATURE(SMAP) ASMJIT_X86_FEATURE(SMEP) ASMJIT_X86_FEATURE(SMX) - ASMJIT_X86_FEATURE(SSE) - ASMJIT_X86_FEATURE(SSE2) - ASMJIT_X86_FEATURE(SSE3) - ASMJIT_X86_FEATURE(SSE4_1) - ASMJIT_X86_FEATURE(SSE4_2) - ASMJIT_X86_FEATURE(SSE4A) - ASMJIT_X86_FEATURE(SSSE3) ASMJIT_X86_FEATURE(SVM) ASMJIT_X86_FEATURE(TBM) ASMJIT_X86_FEATURE(TSE) - ASMJIT_X86_FEATURE(TSX) ASMJIT_X86_FEATURE(TSXLDTRK) ASMJIT_X86_FEATURE(UINTR) - ASMJIT_X86_FEATURE(VAES) ASMJIT_X86_FEATURE(VMX) - ASMJIT_X86_FEATURE(VPCLMULQDQ) ASMJIT_X86_FEATURE(WAITPKG) ASMJIT_X86_FEATURE(WBNOINVD) ASMJIT_X86_FEATURE(WRMSRNS) - ASMJIT_X86_FEATURE(XOP) ASMJIT_X86_FEATURE(XSAVE) ASMJIT_X86_FEATURE(XSAVEC) ASMJIT_X86_FEATURE(XSAVEOPT) ASMJIT_X86_FEATURE(XSAVES) -#ifndef ASMJIT_NO_DEPRECATED - ASMJIT_DEPRECATED("Use hasAVX512_CD() instead") - ASMJIT_X86_FEATURE(AVX512_CDI) + ASMJIT_X86_FEATURE(FPU) + ASMJIT_X86_FEATURE(MMX) + ASMJIT_X86_FEATURE(MMX2) + ASMJIT_X86_FEATURE(3DNOW) + ASMJIT_X86_FEATURE(3DNOW2) + ASMJIT_X86_FEATURE(GEODE) - ASMJIT_DEPRECATED("Use hasAVX512_ER() instead") - ASMJIT_X86_FEATURE(AVX512_ERI) + ASMJIT_X86_FEATURE(SSE) + ASMJIT_X86_FEATURE(SSE2) + ASMJIT_X86_FEATURE(SSE3) + ASMJIT_X86_FEATURE(SSSE3) + ASMJIT_X86_FEATURE(SSE4_1) + ASMJIT_X86_FEATURE(SSE4_2) + ASMJIT_X86_FEATURE(SSE4A) + ASMJIT_X86_FEATURE(PCLMULQDQ) - ASMJIT_DEPRECATED("Use hasAVX512_PF() instead") - ASMJIT_X86_FEATURE(AVX512_PFI) -#endif + ASMJIT_X86_FEATURE(AVX) + ASMJIT_X86_FEATURE(AVX2) + ASMJIT_X86_FEATURE(AVX_IFMA) + ASMJIT_X86_FEATURE(AVX_NE_CONVERT) + ASMJIT_X86_FEATURE(AVX_VNNI) + ASMJIT_X86_FEATURE(AVX_VNNI_INT16) + ASMJIT_X86_FEATURE(AVX_VNNI_INT8) + ASMJIT_X86_FEATURE(F16C) + ASMJIT_X86_FEATURE(FMA) + ASMJIT_X86_FEATURE(FMA4) + ASMJIT_X86_FEATURE(XOP) + + ASMJIT_X86_FEATURE(AVX512_BF16) + ASMJIT_X86_FEATURE(AVX512_BITALG) + ASMJIT_X86_FEATURE(AVX512_BW) + ASMJIT_X86_FEATURE(AVX512_CD) + ASMJIT_X86_FEATURE(AVX512_DQ) + ASMJIT_X86_FEATURE(AVX512_F) + ASMJIT_X86_FEATURE(AVX512_FP16) + ASMJIT_X86_FEATURE(AVX512_IFMA) + ASMJIT_X86_FEATURE(AVX512_VBMI) + ASMJIT_X86_FEATURE(AVX512_VBMI2) + ASMJIT_X86_FEATURE(AVX512_VL) + ASMJIT_X86_FEATURE(AVX512_VNNI) + ASMJIT_X86_FEATURE(AVX512_VP2INTERSECT) + ASMJIT_X86_FEATURE(AVX512_VPOPCNTDQ) + + ASMJIT_X86_FEATURE(AESNI) + ASMJIT_X86_FEATURE(GFNI) + ASMJIT_X86_FEATURE(SHA) + ASMJIT_X86_FEATURE(SHA512) + ASMJIT_X86_FEATURE(SM3) + ASMJIT_X86_FEATURE(SM4) + ASMJIT_X86_FEATURE(VAES) + ASMJIT_X86_FEATURE(VPCLMULQDQ) + + ASMJIT_X86_FEATURE(KL) + ASMJIT_X86_FEATURE(AESKLE) + ASMJIT_X86_FEATURE(AESKLEWIDE_KL) + + ASMJIT_X86_FEATURE(AVX10_1) + ASMJIT_X86_FEATURE(AVX10_2) + + ASMJIT_X86_FEATURE(AMX_AVX512) + ASMJIT_X86_FEATURE(AMX_BF16) + ASMJIT_X86_FEATURE(AMX_COMPLEX) + ASMJIT_X86_FEATURE(AMX_FP16) + ASMJIT_X86_FEATURE(AMX_FP8) + ASMJIT_X86_FEATURE(AMX_INT8) + ASMJIT_X86_FEATURE(AMX_MOVRS) + ASMJIT_X86_FEATURE(AMX_TF32) + ASMJIT_X86_FEATURE(AMX_TILE) + ASMJIT_X86_FEATURE(AMX_TRANSPOSE) #undef ASMJIT_X86_FEATURE }; @@ -502,137 +540,235 @@ public: kARMv6, //!< CPU is at least ARMv6 {A32}. kARMv7, //!< CPU is at least ARMv7 {A32}. kARMv8a, //!< CPU is at least ARMv8A. - kTHUMB, //!< CPU has THUMB (16-bit THUMB encoding) {A32}. - kTHUMBv2, //!< CPU has THUMBv2 (32-bit THUMB encoding) {A32}. + kTHUMB, //!< CPU has THUMB (16-bit THUMB encoding) {A32}. + kTHUMBv2, //!< CPU has THUMBv2 (32-bit THUMB encoding) {A32}. - kAES, //!< CPU has AES (ASIMD AES instructions). - kAFP, //!< CPU has AFP (alternate floating-point behavior) {A64}. - kASIMD, //!< CPU has ASIMD (NEON on ARM/THUMB). - kBF16, //!< CPU has BF16 (BFloat16 instructions) {A64}. - kBTI, //!< CPU has BTI (branch target identification). - kCCIDX, //!< CPU has CCIDX (extend of the CCSIDR number of sets). - kCHK, //!< CPU has CHK (CHKFEAT instruction) {A64}. - kCLRBHB, //!< CPU has CLRBHB (clear BHB instruction). - kCPUID, //!< CPU has CPUID (CPUID registers accessible in user-space). - kCRC32, //!< CPU has CRC32 (CRC32 instructions). - kCSSC, //!< CPU has CSSC (common short sequence compression) {A64}. - kD128, //!< CPU has D128 (128-bit translation tables, 56 bit PA) {A64}. - kDGH, //!< CPU has DGH (data gathering hint) {A64}. - kDIT, //!< CPU has DIT (data independent timing of instructions). - kDOTPROD, //!< CPU has DOTPROD (ASIMD Int8 dot product instructions). - kDPB, //!< CPU has DPB (DC CVAP instruction) {A64}. - kDPB2, //!< CPU has DPB2 (DC CVADP instruction) {A64}. - kEBF16, //!< CPU has EBF16 (extended BFloat16 mode) {A64}. - kECV, //!< CPU has ECV (enhanced counter virtualization). - kEDSP, //!< CPU has EDSP (ARM/THUMB only). - kFCMA, //!< CPU has FCMA (FCADD/FCMLA). - kFGT, //!< CPU has FGT (fine-grained traps). - kFGT2, //!< CPU has FGT2 (fine-grained traps 2). - kFHM, //!< CPU has FHM (half-precision floating-point FMLAL instructions). - kFLAGM, //!< CPU has FLAGM (condition flag manipulation) {A64}. - kFLAGM2, //!< CPU has FLAGM2 (condition flag manipulation version v2) {A64}. - kFMAC, //!< CPU has FMAC (ARM/THUMB only). - kFP, //!< CPU has FP (floating-point) (on 32-bit ARM this means VFPv3). - kFP16, //!< CPU has FP16 (half-precision floating-point data processing). - kFP16CONV, //!< CPU has FP16CONV (half-precision float conversion). - kFRINTTS, //!< CPU has FRINTTS (FRINT[32|64][X|Z] instructions) {A64}. - kGCS, //!< CPU has GCS (guarded control stack extension) {A64}. - kHBC, //!< CPU has HBC (hinted conditional branches) {A64} - kHCX, //!< CPU has HCX (support for the HCRX_EL2 register) {A64}. - kI8MM, //!< CPU has I8MM (int8 matrix multiplication) {A64}. - kIDIVA, //!< CPU has IDIV (hardware SDIV and UDIV in ARM mode). - kIDIVT, //!< CPU has IDIV (hardware SDIV and UDIV in THUMB mode). - kJSCVT, //!< CPU has JSCVT (JavaScript FJCVTS conversion instruction) {A64}. - kLOR, //!< CPU has LOR (limited ordering regions extension). - kLRCPC, //!< CPU has LRCPC (load-acquire RCpc instructions) {A64}. - kLRCPC2, //!< CPU has LRCPC2 (load-acquire RCpc instructions v2) {A64}. - kLRCPC3, //!< CPU has LRCPC3 (load-Acquire RCpc instructions v3) {A64}. - kLS64, //!< CPU has LS64 (64 byte loads/stores without return) {A64}. - kLS64_ACCDATA, //!< CPU has LS64_ACCDATA (64-byte EL0 stores with return) {A64}. - kLS64_V, //!< CPU has LS64_V (64-byte stores with return) {A64}. - kLSE, //!< CPU has LSE (large system extensions) {A64}. - kLSE128, //!< CPU has LSE128 (128-bit atomics) {A64}. - kLSE2, //!< CPU has LSE2 (large system extensions v2) {A64}. - kMOPS, //!< CPU has MOPS (memcpy and memset acceleration instructions) {A64}. - kMPAM, //!< CPU has MPAM (memory system partitioning and monitoring extension) {A64}. - kMTE, //!< CPU has MTE (instruction-only memory tagging extension) {A64}. - kMTE2, //!< CPU has MTE2 (full memory tagging extension) {A64}. - kMTE3, //!< CPU has MTE3 (MTE asymmetric fault handling) {A64}. - kMTE4, //!< CPU has MTE4 (MTE v4) {A64}. - kNMI, //!< CPU has NMI (non-maskable Interrupt) {A64}. - kNV, //!< CPU has NV (nested virtualization enchancement) {A64}. - kNV2, //!< CPU has NV2 (enhanced support for nested virtualization) {A64}. - kPAN, //!< CPU has PAN (privileged access-never extension) {A64}. - kPAN2, //!< CPU has PAN2 (PAN s1e1R and s1e1W variants) {A64}. - kPAN3, //!< CPU has PAN3 (support for SCTLR_ELx.EPAN) {A64}. - kPAUTH, //!< CPU has PAUTH (pointer authentication extension) {A64}. - kPMU, //!< CPU has PMU {A64}. - kPMULL, //!< CPU has PMULL {A64}. - kPRFMSLC, //!< CPU has PRFMSLC (PRFM instructions support the SLC target) {A64} - kRAS, //!< CPU has RAS (reliability, availability and serviceability extensions). - kRAS1_1, //!< CPU has RASv1p1 (RAS v1.1). - kRAS2, //!< CPU has RASv2 (RAS v2). - kRDM, //!< CPU has RDM (rounding double multiply accumulate) {A64}. - kRME, //!< CPU has RME (memory encryption contexts extension) {A64}. - kRNG, //!< CPU has RNG (random number generation). - kRNG_TRAP, //!< CPU has RNG_TRAP (random number trap to EL3 field) {A64}. - kRPRES, //!< CPU has RPRES (increased precision of reciprocal estimate and RSQRT estimate) {A64}. - kRPRFM, //!! CPU has RPRFM (range prefetch hint instruction). - kSB, //!< CPU has SB (speculative barrier). - kSHA1, //!< CPU has SHA1 (ASIMD SHA1 instructions). - kSHA256, //!< CPU has SHA256 (ASIMD SHA256 instructions). - kSHA3, //!< CPU has SHA3 (ASIMD EOR3, RAX1, XAR, and BCAX instructions). - kSHA512, //!< CPU has SHA512 (ASIMD SHA512 instructions). - kSM3, //!< CPU has SM3 (ASIMD SM3 instructions). - kSM4, //!< CPU has SM4 (ASIMD SM4 instructions). - kSME, //!< CPU has SME (SME v1 - scalable matrix extension) {A64}. - kSME2, //!< CPU has SME2 (SME v2) {A64}. - kSME2_1, //!< CPU has SME2p1 (SME v2.1) {A64}. - kSME_B16B16, //!< CPU has SME_B16B16 (SME non-widening BFloat16 to BFloat16 arithmetic) {A64}. - kSME_B16F32, //!< CPU has SME_B16F32 {A64}. - kSME_BI32I32, //!< CPU has SME_BI32I32 {A64}. - kSME_F16F16, //!< CPU has SME_F16F16 (SME2.1 non-widening half-precision FP16 to FP16 arithmetic) {A64}. - kSME_F16F32, //!< CPU has SME_F16F32 {A64}. - kSME_F32F32, //!< CPU has SME_F32F32 {A64}. - kSME_F64F64, //!< CPU has SME_F64F64 {A64}. - kSME_FA64, //!< CPU has SME_FA64 {A64}. - kSME_I16I32, //!< CPU has SME_I16I32 {A64}. - kSME_I16I64, //!< CPU has SME_I16I64 {A64}. - kSME_I8I32, //!< CPU has SME_I8I32 {A64}. - kSPECRES, //!< CPU has SPECRES (speculation restriction instructions). - kSPECRES2, //!< CPU has SPECRES2 (clear other speculative predictions). - kSSBS, //!< CPU has SSBS (speculative store bypass safe instruction). - kSSBS2, //!< CPU has SSBS2 (MRS and MSR instructions for SSBS). - kSVE, //!< CPU has SVE (SVE v1 - scalable vector extension) {A64}. - kSVE2, //!< CPU has SVE2 (SVE v2) {A64}. - kSVE2_1, //!< CPU has SVE2p1 (SVE v2.1) {A64}. - kSVE_AES, //!< CPU has SVE_AES (SVE AES instructions) {A64}. - kSVE_B16B16, //!< CPU has SVE_B16B16 (SVE non-widening BFloat16 to BFloat16 arithmetic) {A64}. - kSVE_BF16, //!< CPU has SVE_BF16 (SVE BF16 instructions) {A64}. - kSVE_BITPERM, //!< CPU has SVE_BITPERM (SVE bit permute) {A64}. - kSVE_EBF16, //!< CPU has SVE_EBF16 (SVE extended BFloat16 mode) {A64}. - kSVE_F32MM, //!< CPU has SVE_F32MM (SVE single-precision floating-point matrix multiply instruction) {A64}. - kSVE_F64MM, //!< CPU has SVE_F64MM (SVE double-precision floating-point matrix multiply instruction) {A64}. - kSVE_I8MM, //!< CPU has SVE_I8MM (SVE int8 matrix multiplication) {A64}. - kSVE_PMULL128, //!< CPU has SVE_PMULL128 (SVE PMULL instructions) {A64}. - kSVE_SHA3, //!< CPU has SVE_SHA3 (SVE SHA-3 instructions) {A64}. - kSVE_SM4, //!< CPU has SVE_SM4 (SVE SM4 instructions {A64}. - kSYSINSTR128, //!< CPU has SYSINSTR128 (128-bit system instructions) {A64}. - kSYSREG128, //!< CPU has SYSREG128 (128-bit system registers) {A64}. - kTHE, //!< CPU has THE (translation hardening extension). - kTME, //!< CPU has TME (transactional memory extensions). - kTRF, //!< CPU has TRF (trace extension). - kUAO, //!< CPU has UAO (AArch64 v8.2 UAO PState) {A64}. - kVFP_D32, //!< CPU has VFP_D32 (32 VFP-D registers) (ARM/THUMB only). - kVHE, //!< CPU has VHE (virtual host extension). - kWFXT, //!< CPU has WFxT (WFE and WFI instructions with timeout) {A64}. - kXS, //!< CPU has XS (XS attribute in TLBI and DSB instructions) {A64}. + kABLE, //!< CPU has ABLE (address breakpoint linking extension) {A64}. + kADERR, //!< CPU has ADERR (asynchronous device error exceptions) {A64}. + kAES, //!< CPU has AES (ASIMD AES instructions). + kAFP, //!< CPU has AFP (alternate floating-point behavior) {A64}. + kAIE, //!< CPU has AIE (memory attribute index enhancement) {A64}. + kAMU1, //!< CPU has AMUv1 (activity monitors extension version 1) {A64}. + kAMU1_1, //!< CPU has AMUv1p1 (activity monitors extension version 1.1) {A64}. + kANERR, //!< CPU has ANERR (asynchronous normal error exception) {A64}. + kASIMD, //!< CPU has ASIMD (NEON on ARM/THUMB). + kBF16, //!< CPU has BF16 (BFloat16 instructions) {A64}. + kBRBE, //!< CPU has BRBE (branch record buffer extension) {A64}. + kBTI, //!< CPU has BTI (branch target identification). + kBWE, //!< CPU has BWE (breakpoint mismatch and range extension) {A64}. + kCCIDX, //!< CPU has CCIDX (extend of the CCSIDR number of sets). + kCHK, //!< CPU has CHK (check feature status - CHKFEAT instruction) {A64}. + kCLRBHB, //!< CPU has CLRBHB (clear BHB instruction). + kCMOW, //!< CPU has CMOW (control for cache maintenance permission) {A64}. + kCMPBR, //!< CPU has CMPBR (Compare and branch instructions) {A64}. + kCONSTPACFIELD, //!< CPU has CONSTPACFIELD (PAC algorithm enhancement) {A64}. + kCPA, //!< CPU has CPA (instruction-only Checked Pointer Arithmetic) {A64}. + kCPA2, //!< CPU has CPA2 (checked Pointer Arithmetic) {A64}. + kCPUID, //!< CPU has CPUID (CPUID registers accessible in user-space). + kCRC32, //!< CPU has CRC32 (CRC32 instructions). + kCSSC, //!< CPU has CSSC (common short sequence compression) {A64}. + kCSV2, //!< CPU has CSV2 (cache speculation variant 2 version 2.1) {A64}. + kCSV2_3, //!< CPU has CSV2_3 (cache speculation variant 2 version 3) {A64}. + kCSV3, //!< CPU has CSV3 (cache speculation Variant 3) {A64}. + kD128, //!< CPU has D128 (128-bit translation tables, 56 bit PA) {A64}. + kDGH, //!< CPU has DGH (data gathering hint) {A64}. + kDIT, //!< CPU has DIT (data independent timing of instructions). + kDOTPROD, //!< CPU has DOTPROD (ASIMD Int8 dot product instructions). + kDPB, //!< CPU has DPB (DC CVAP instruction) {A64}. + kDPB2, //!< CPU has DPB2 (DC CVADP instruction) {A64}. + kEBEP, //!< CPU has EBEP (exception-based event profiling) {A64}. + kEBF16, //!< CPU has EBF16 (extended BFloat16 mode) {A64}. + kECBHB, //!< CPU has ECBHB (exploitative control using branch history information) {A64}. + kECV, //!< CPU has ECV (enhanced counter virtualization). + kEDHSR, //!< CPU has EDHSR (support for EDHSR) {A64}. + kEDSP, //!< CPU has EDSP (ARM/THUMB only). + kF8E4M3, //!< CPU has F8E4M3 {A64}. + kF8E5M2, //!< CPU has F8E5M2 {A64}. + kF8F16MM, //!< CPU has F8F16MM (8-bit floating-point matrix multiply-accumulate to half-precision) {A64} + kF8F32MM, //!< CPU has F8F32MM (8-bit floating-point matrix multiply-accumulate to single-precision) {A64} + kFAMINMAX, //!< CPU has FAMINMAX (floating-point maximum and minimum absolute value instructions) {A64}. + kFCMA, //!< CPU has FCMA (FCADD/FCMLA). + kFGT, //!< CPU has FGT (fine-grained traps). + kFGT2, //!< CPU has FGT2 (fine-grained traps 2). + kFHM, //!< CPU has FHM (half-precision floating-point FMLAL instructions). + kFLAGM, //!< CPU has FLAGM (condition flag manipulation) {A64}. + kFLAGM2, //!< CPU has FLAGM2 (condition flag manipulation version v2) {A64}. + kFMAC, //!< CPU has FMAC (ARM/THUMB only). + kFP, //!< CPU has FP (floating-point) (on 32-bit ARM this means VFPv3). + kFP16, //!< CPU has FP16 (half-precision floating-point data processing). + kFP16CONV, //!< CPU has FP16CONV (half-precision float conversion). + kFP8, //!< CPU has FP8 (FP8 convert instructions) {A64}. + kFP8DOT2, //!< CPU has FP8DOT2 (FP8 2-way dot product to half-precision instructions) {A64}. + kFP8DOT4, //!< CPU has FP8DOT4 (FP8 4-way dot product to single-precision instructions) {A64}. + kFP8FMA, //!< CPU has FP8FMA (FP8 multiply-accumulate to half-precision and single-precision instructions) {A64}. + kFPMR, //!< CPU has FPMR (floating-point Mode Register) {A64}. + kFPRCVT, //!< CPU has FPRCVT (floating-point to/from integer in scalar FP register) {A64}. + kFRINTTS, //!< CPU has FRINTTS (FRINT[32|64][X|Z] instructions) {A64}. + kGCS, //!< CPU has GCS (guarded control stack extension) {A64}. + kHACDBS, //!< CPU has HACDBS (hardware accelerator for cleaning Dirty state) {A64}. + kHAFDBS, //!< CPU has HAFDBS (hardware management of the access flag and dirty state) {A64}. + kHAFT, //!< CPU has HAFT (hardware managed access flag for table descriptors) {A64}. + kHDBSS, //!< CPU has HDBSS (hardware Dirty state tracking Structure) {A64}. + kHBC, //!< CPU has HBC (hinted conditional branches) {A64}. + kHCX, //!< CPU has HCX (support for the HCRX_EL2 register) {A64}. + kHPDS, //!< CPU has HPDS (hierarchical permission disables in translation tables ) {A64}. + kHPDS2, //!< CPU has HPDS2 (hierarchical permission disables) {A64}. + kI8MM, //!< CPU has I8MM (int8 matrix multiplication) {A64}. + kIDIVA, //!< CPU has IDIV (hardware SDIV and UDIV in ARM mode). + kIDIVT, //!< CPU has IDIV (hardware SDIV and UDIV in THUMB mode). + kITE, //!< CPU has ITE (instrumentation extension) {A64}. + kJSCVT, //!< CPU has JSCVT (JavaScript FJCVTS conversion instruction) {A64}. + kLOR, //!< CPU has LOR (limited ordering regions extension). + kLRCPC, //!< CPU has LRCPC (load-acquire RCpc instructions) {A64}. + kLRCPC2, //!< CPU has LRCPC2 (load-acquire RCpc instructions v2) {A64}. + kLRCPC3, //!< CPU has LRCPC3 (load-Acquire RCpc instructions v3) {A64}. + kLS64, //!< CPU has LS64 (64 byte loads/stores without return) {A64}. + kLS64_ACCDATA, //!< CPU has LS64_ACCDATA (64-byte EL0 stores with return) {A64}. + kLS64_V, //!< CPU has LS64_V (64-byte stores with return) {A64}. + kLS64WB, //!< CPU has LS64WB (LS64 for Write-back cacheable memory) {A64} + kLSE, //!< CPU has LSE (large system extensions) {A64}. + kLSE128, //!< CPU has LSE128 (128-bit atomics) {A64}. + kLSE2, //!< CPU has LSE2 (large system extensions v2) {A64}. + kLSFE, //!< CPU has LSFE (large system float extension) {A64}. + kLSUI, //!< CPU has LSUI (unprivileged load store) {A64}. + kLUT, //!< CPU has LUT (lookup table instructions with 2-bit and 4-bit indices) {A64}. + kLVA, //!< CPU has LVA (large VA support) {A64}. + kLVA3, //!< CPU has LVA3 (56-bit VA) {A64}. + kMEC, //!< CPU has MEC (memory encryption contexts) {A64}. + kMOPS, //!< CPU has MOPS (memcpy and memset acceleration instructions) {A64}. + kMPAM, //!< CPU has MPAM (memory system partitioning and monitoring extension) {A64}. + kMTE, //!< CPU has MTE (instruction-only memory tagging extension) {A64}. + kMTE2, //!< CPU has MTE2 (full memory tagging extension) {A64}. + kMTE3, //!< CPU has MTE3 (MTE asymmetric fault handling) {A64}. + kMTE4, //!< CPU has MTE4 (MTE v4) {A64}. + kMTE_ASYM_FAULT, //!< CPU has MTE_ASYM_FAULT (memory tagging asymmetric faults) {A64}. + kMTE_ASYNC, //!< CPU has MTE_ASYNC (memory tagging asynchronous faulting) {A64}. + kMTE_CANONICAL_TAGS, //!< CPU has MTE_CANONICAL_TAGS (canonical tag checking for untagged memory) {A64}. + kMTE_NO_ADDRESS_TAGS, //!< CPU has MTE_NO_ADDRESS_TAGS (memory tagging with address tagging disabled) {A64}. + kMTE_PERM_S1, //!< CPU has MTE_PERM_S1 (allocation tag access permission) {A64}. + kMTE_STORE_ONLY, //!< CPU has MTE_STORE_ONLY (store-only tag checking) {A64}. + kMTE_TAGGED_FAR, //!< CPU has MTE_TAGGED_FAR (FAR_ELx on a tag check fault) {A64}. + kMTPMU, //!< CPU has MTPMU (multi-threaded PMU extensions) {A64}. + kNMI, //!< CPU has NMI (non-maskable Interrupt) {A64}. + kNV, //!< CPU has NV (nested virtualization enchancement) {A64}. + kNV2, //!< CPU has NV2 (enhanced support for nested virtualization) {A64}. + kOCCMO, //!< CPU has OCCMO (outer cacheable cache maintenance operation) {A64}. + kPAN, //!< CPU has PAN (privileged access-never extension) {A64}. + kPAN2, //!< CPU has PAN2 (PAN s1e1R and s1e1W variants) {A64}. + kPAN3, //!< CPU has PAN3 (support for SCTLR_ELx.EPAN) {A64}. + kPAUTH, //!< CPU has PAUTH (pointer authentication extension) {A64}. + kPFAR, //!< CPU has PFAR (physical fault address registers) {A64}. + kPMU, //!< CPU has PMU {A64}. + kPMULL, //!< CPU has PMULL (ASIMD PMULL instructions) {A64}. + kPRFMSLC, //!< CPU has PRFMSLC (PRFM instructions support the SLC target) {A64}. + kRAS, //!< CPU has RAS (reliability, availability and serviceability extensions). + kRAS1_1, //!< CPU has RASv1p1 (RAS v1.1). + kRAS2, //!< CPU has RASv2 (RAS v2). + kRASSA2, //!< CPU has RASSAv2 (RAS v2 system architecture). + kRDM, //!< CPU has RDM (rounding double multiply accumulate) {A64}. + kRME, //!< CPU has RME (memory encryption contexts extension) {A64}. + kRNG, //!< CPU has RNG (random number generation). + kRNG_TRAP, //!< CPU has RNG_TRAP (random number trap to EL3 field) {A64}. + kRPRES, //!< CPU has RPRES (increased precision of reciprocal estimate and RSQRT estimate) {A64}. + kRPRFM, //!< CPU has RPRFM (range prefetch hint instruction). + kS1PIE, //!< CPU has S1PIE (permission model enhancements) {A64}. + kS1POE, //!< CPU has S1POE (permission model enhancements) {A64}. + kS2PIE, //!< CPU has S2PIE (permission model enhancements) {A64}. + kS2POE, //!< CPU has S2POE (permission model enhancements) {A64}. + kSB, //!< CPU has SB (speculative barrier). + kSCTLR2, //!< CPU has SCTLR2 (extension to SCTLR_ELx) {A64}. + kSEBEP, //!< CPU has SEBEP (synchronous exception-based event profiling) {A64}. + kSEL2, //!< CPU has SEL2 (secure EL2) {A64}. + kSHA1, //!< CPU has SHA1 (ASIMD SHA1 instructions). + kSHA256, //!< CPU has SHA256 (ASIMD SHA256 instructions). + kSHA3, //!< CPU has SHA3 (ASIMD EOR3, RAX1, XAR, and BCAX instructions). + kSHA512, //!< CPU has SHA512 (ASIMD SHA512 instructions). + kSM3, //!< CPU has SM3 (ASIMD SM3 instructions). + kSM4, //!< CPU has SM4 (ASIMD SM4 instructions). + kSME, //!< CPU has SME (SME v1 - scalable matrix extension) {A64}. + kSME2, //!< CPU has SME2 (SME v2) {A64}. + kSME2_1, //!< CPU has SME2p1 (SME v2.1) {A64}. + kSME2_2, //!< CPU has SME2p1 (SME v2.2) {A64}. + kSME_AES, //!< CPU has SME_AES {A64}. + kSME_B16B16, //!< CPU has SME_B16B16 (SME non-widening BFloat16 to BFloat16 arithmetic) {A64}. + kSME_B16F32, //!< CPU has SME_B16F32 (BFMOPA and BFMOPS instructions that accumulate BFloat16 outer products into single-precision tiles) {A64}. + kSME_BI32I32, //!< CPU has SME_BI32I32 (BMOPA and BMOPS instructions that accumulate 1-bit binary outer products into 32-bit integer tiles) {A64}. + kSME_F16F16, //!< CPU has SME_F16F16 (SME2.1 non-widening half-precision FP16 to FP16 arithmetic) {A64}. + kSME_F16F32, //!< CPU has SME_F16F32 {A64}. + kSME_F32F32, //!< CPU has SME_F32F32 {A64}. + kSME_F64F64, //!< CPU has SME_F64F64 {A64}. + kSME_F8F16, //!< CPU has SME_F8F16 (SME2 ZA-targeting FP8 multiply-accumulate, dot product, and outer product to half-precision instructions) {A64}. + kSME_F8F32, //!< CPU has SME_F8F32 (SME2 ZA-targeting FP8 multiply-accumulate, dot product, and outer product to single-precision instructions) {A64}. + kSME_FA64, //!< CPU has SME_FA64 {A64}. + kSME_I16I32, //!< CPU has SME_I16I32 {A64}. + kSME_I16I64, //!< CPU has SME_I16I64 {A64}. + kSME_I8I32, //!< CPU has SME_I8I32 {A64}. + kSME_LUTv2, //!< CPU has SME_LUTv2 (lookup table instructions with 4-bit indices and 8-bit elements) {A64}. + kSME_MOP4, //!< CPU has SME_MOP4 (quarter-tile outer product instructions) {A64}. + kSME_TMOP, //!< CPU has SME_TMOP {A64}. + kSPE, //!< CPU has SPE (statistical profiling extension) {A64}. + kSPE1_1, //!< CPU has SPEv1p1 (statistical profiling extensions version 1.1) {A64}. + kSPE1_2, //!< CPU has SPEv1p2 (statistical profiling extensions version 1.2) {A64}. + kSPE1_3, //!< CPU has SPEv1p3 (statistical profiling extensions version 1.3) {A64}. + kSPE1_4, //!< CPU has SPEv1p4 (statistical profiling extensions version 1.4) {A64}. + kSPE_ALTCLK, //!< CPU has SPE_ALTCLK (statistical profiling alternate clock domain extension) {A64}. + kSPE_CRR, //!< CPU has SPE_CRR (statistical profiling call return branch records) {A64}. + kSPE_EFT, //!< CPU has SPE_EFT (statistical profiling extended filtering by type) {A64}. + kSPE_FDS, //!< CPU has SPE_FDS (statistical profiling data source filtering) {A64}. + kSPE_FPF, //!< CPU has SPE_FPF (statistical profiling floating-point flag extension) {A64}. + kSPE_SME, //!< CPU has SPE_SME (statistical profiling extensions for SME) {A64}. + kSPECRES, //!< CPU has SPECRES (speculation restriction instructions). + kSPECRES2, //!< CPU has SPECRES2 (clear other speculative predictions). + kSPMU, //!< CPU has SPMU (system performance monitors extension) {A64}. + kSSBS, //!< CPU has SSBS (speculative store bypass safe instruction). + kSSBS2, //!< CPU has SSBS2 (MRS and MSR instructions for SSBS). + kSSVE_AES, //!< CPU has SSVE_AES {A64}. + kSSVE_BITPERM, //!< CPU has SSVE_BITPERM {A64}. + kSSVE_FEXPA, //!< CPU has SSVE_FEXPA {A64}. + kSSVE_FP8DOT2, //!< CPU has SSVE_FP8DOT2 (SVE2 FP8 2-way dot product to half-precision instructions in Streaming SVE mode) {A64}. + kSSVE_FP8DOT4, //!< CPU has SSVE_FP8DOT4 (SVE2 FP8 4-way dot product to single-precision instructions in Streaming SVE mode) {A64}. + kSSVE_FP8FMA, //!< CPU has SSVE_FP8FMA (SVE2 FP8 multiply-accumulate to half-precision and single-precision instructions in Streaming SVE mode) {A64}. + kSVE, //!< CPU has SVE (SVE v1 - scalable vector extension) {A64}. + kSVE2, //!< CPU has SVE2 (SVE v2) {A64}. + kSVE2_1, //!< CPU has SVE2p1 (SVE v2.1) {A64}. + kSVE2_2, //!< CPU has SVE2p2 (SVE v2.2) {A64}. + kSVE_AES, //!< CPU has SVE_AES (SVE AES instructions) {A64}. + kSVE_AES2, //!< CPU has SVE_AES2 {A64}. + kSVE_B16B16, //!< CPU has SVE_B16B16 (SVE non-widening BFloat16 to BFloat16 arithmetic) {A64}. + kSVE_BF16, //!< CPU has SVE_BF16 (SVE BF16 instructions) {A64}. + kSVE_BFSCALE, //!< CPU has SVE_BFSCALE {A64}. + kSVE_BITPERM, //!< CPU has SVE_BITPERM (SVE bit permute) {A64}. + kSVE_EBF16, //!< CPU has SVE_EBF16 (SVE extended BFloat16 mode) {A64}. + kSVE_ELTPERM, //!< CPU has SVE_ELTPERM {A64}. + kSVE_F16MM, //!< CPU has SVE_F16MM (SVE half-precision floating-point matrix multiply instruction) {A64}. + kSVE_F32MM, //!< CPU has SVE_F32MM (SVE single-precision floating-point matrix multiply instruction) {A64}. + kSVE_F64MM, //!< CPU has SVE_F64MM (SVE double-precision floating-point matrix multiply instruction) {A64}. + kSVE_I8MM, //!< CPU has SVE_I8MM (SVE int8 matrix multiplication) {A64}. + kSVE_PMULL128, //!< CPU has SVE_PMULL128 (SVE PMULL instructions) {A64}. + kSVE_SHA3, //!< CPU has SVE_SHA3 (SVE SHA-3 instructions) {A64}. + kSVE_SM4, //!< CPU has SVE_SM4 (SVE SM4 instructions {A64}. + kSYSINSTR128, //!< CPU has SYSINSTR128 (128-bit system instructions) {A64}. + kSYSREG128, //!< CPU has SYSREG128 (128-bit system registers) {A64}. + kTHE, //!< CPU has THE (translation hardening extension). + kTLBIOS, //!< CPU has TLBIOS (TLBI instructions in Outer Shareable domain) {A64}. + kTLBIRANGE, //!< CPU has TLBIRANGE (TLBI range instructions) {A64}. + kTLBIW, //!< CPU has TLBIW (TLBI VMALL for dirty state) {A64}. + kTME, //!< CPU has TME (transactional memory extensions). + kTRF, //!< CPU has TRF (self-hosted trace extensions). + kUAO, //!< CPU has UAO (AArch64 v8.2 UAO PState) {A64}. + kVFP_D32, //!< CPU has VFP_D32 (32 VFP-D registers) (ARM/THUMB only). + kVHE, //!< CPU has VHE (virtual host extension). + kVMID16, //!< CPU has VMID16 (16-bit VMID) {A64}. + kWFXT, //!< CPU has WFxT (WFE and WFI instructions with timeout) {A64}. + kXNX, //!< CPU has XNX (translation table stage 2 unprivileged execute-never) {A64}. + kXS, //!< CPU has XS (XS attribute in TLBI and DSB instructions) {A64}. // @EnumValuesEnd@ kMaxValue = kXS }; #define ASMJIT_ARM_FEATURE(FEATURE) \ + /*! Tests whether FEATURE is present. */ \ ASMJIT_INLINE_NODEBUG bool has##FEATURE() const noexcept { return has(ARM::k##FEATURE); } ASMJIT_ARM_FEATURE(THUMB) @@ -642,26 +778,50 @@ public: ASMJIT_ARM_FEATURE(ARMv7) ASMJIT_ARM_FEATURE(ARMv8a) + ASMJIT_ARM_FEATURE(ABLE) + ASMJIT_ARM_FEATURE(ADERR) ASMJIT_ARM_FEATURE(AES) ASMJIT_ARM_FEATURE(AFP) + ASMJIT_ARM_FEATURE(AIE) + ASMJIT_ARM_FEATURE(AMU1) + ASMJIT_ARM_FEATURE(AMU1_1) + ASMJIT_ARM_FEATURE(ANERR) ASMJIT_ARM_FEATURE(ASIMD) ASMJIT_ARM_FEATURE(BF16) + ASMJIT_ARM_FEATURE(BRBE) ASMJIT_ARM_FEATURE(BTI) + ASMJIT_ARM_FEATURE(BWE) ASMJIT_ARM_FEATURE(CCIDX) ASMJIT_ARM_FEATURE(CHK) ASMJIT_ARM_FEATURE(CLRBHB) + ASMJIT_ARM_FEATURE(CMOW) + ASMJIT_ARM_FEATURE(CMPBR) + ASMJIT_ARM_FEATURE(CONSTPACFIELD) + ASMJIT_ARM_FEATURE(CPA) + ASMJIT_ARM_FEATURE(CPA2) ASMJIT_ARM_FEATURE(CPUID) ASMJIT_ARM_FEATURE(CRC32) ASMJIT_ARM_FEATURE(CSSC) + ASMJIT_ARM_FEATURE(CSV2) + ASMJIT_ARM_FEATURE(CSV2_3) + ASMJIT_ARM_FEATURE(CSV3) ASMJIT_ARM_FEATURE(D128) ASMJIT_ARM_FEATURE(DGH) ASMJIT_ARM_FEATURE(DIT) ASMJIT_ARM_FEATURE(DOTPROD) ASMJIT_ARM_FEATURE(DPB) ASMJIT_ARM_FEATURE(DPB2) + ASMJIT_ARM_FEATURE(EBEP) ASMJIT_ARM_FEATURE(EBF16) + ASMJIT_ARM_FEATURE(ECBHB) ASMJIT_ARM_FEATURE(ECV) + ASMJIT_ARM_FEATURE(EDHSR) ASMJIT_ARM_FEATURE(EDSP) + ASMJIT_ARM_FEATURE(F8E4M3) + ASMJIT_ARM_FEATURE(F8E5M2) + ASMJIT_ARM_FEATURE(F8F16MM) + ASMJIT_ARM_FEATURE(F8F32MM) + ASMJIT_ARM_FEATURE(FAMINMAX) ASMJIT_ARM_FEATURE(FCMA) ASMJIT_ARM_FEATURE(FGT) ASMJIT_ARM_FEATURE(FGT2) @@ -672,13 +832,26 @@ public: ASMJIT_ARM_FEATURE(FP) ASMJIT_ARM_FEATURE(FP16) ASMJIT_ARM_FEATURE(FP16CONV) + ASMJIT_ARM_FEATURE(FP8) + ASMJIT_ARM_FEATURE(FP8DOT2) + ASMJIT_ARM_FEATURE(FP8DOT4) + ASMJIT_ARM_FEATURE(FP8FMA) + ASMJIT_ARM_FEATURE(FPMR) + ASMJIT_ARM_FEATURE(FPRCVT) ASMJIT_ARM_FEATURE(FRINTTS) ASMJIT_ARM_FEATURE(GCS) + ASMJIT_ARM_FEATURE(HACDBS) + ASMJIT_ARM_FEATURE(HAFDBS) + ASMJIT_ARM_FEATURE(HAFT) + ASMJIT_ARM_FEATURE(HDBSS) ASMJIT_ARM_FEATURE(HBC) ASMJIT_ARM_FEATURE(HCX) + ASMJIT_ARM_FEATURE(HPDS) + ASMJIT_ARM_FEATURE(HPDS2) ASMJIT_ARM_FEATURE(I8MM) ASMJIT_ARM_FEATURE(IDIVA) ASMJIT_ARM_FEATURE(IDIVT) + ASMJIT_ARM_FEATURE(ITE) ASMJIT_ARM_FEATURE(JSCVT) ASMJIT_ARM_FEATURE(LOR) ASMJIT_ARM_FEATURE(LRCPC) @@ -687,35 +860,60 @@ public: ASMJIT_ARM_FEATURE(LS64) ASMJIT_ARM_FEATURE(LS64_ACCDATA) ASMJIT_ARM_FEATURE(LS64_V) + ASMJIT_ARM_FEATURE(LS64WB) ASMJIT_ARM_FEATURE(LSE) ASMJIT_ARM_FEATURE(LSE128) ASMJIT_ARM_FEATURE(LSE2) + ASMJIT_ARM_FEATURE(LSFE) + ASMJIT_ARM_FEATURE(LSUI) + ASMJIT_ARM_FEATURE(LUT) + ASMJIT_ARM_FEATURE(LVA) + ASMJIT_ARM_FEATURE(LVA3) + ASMJIT_ARM_FEATURE(MEC) ASMJIT_ARM_FEATURE(MOPS) ASMJIT_ARM_FEATURE(MPAM) ASMJIT_ARM_FEATURE(MTE) ASMJIT_ARM_FEATURE(MTE2) ASMJIT_ARM_FEATURE(MTE3) ASMJIT_ARM_FEATURE(MTE4) + ASMJIT_ARM_FEATURE(MTE_ASYM_FAULT) + ASMJIT_ARM_FEATURE(MTE_ASYNC) + ASMJIT_ARM_FEATURE(MTE_CANONICAL_TAGS) + ASMJIT_ARM_FEATURE(MTE_NO_ADDRESS_TAGS) + ASMJIT_ARM_FEATURE(MTE_PERM_S1) + ASMJIT_ARM_FEATURE(MTE_STORE_ONLY) + ASMJIT_ARM_FEATURE(MTE_TAGGED_FAR) + ASMJIT_ARM_FEATURE(MTPMU) ASMJIT_ARM_FEATURE(NMI) ASMJIT_ARM_FEATURE(NV) ASMJIT_ARM_FEATURE(NV2) + ASMJIT_ARM_FEATURE(OCCMO) ASMJIT_ARM_FEATURE(PAN) ASMJIT_ARM_FEATURE(PAN2) ASMJIT_ARM_FEATURE(PAN3) ASMJIT_ARM_FEATURE(PAUTH) + ASMJIT_ARM_FEATURE(PFAR) ASMJIT_ARM_FEATURE(PMU) ASMJIT_ARM_FEATURE(PMULL) ASMJIT_ARM_FEATURE(PRFMSLC) ASMJIT_ARM_FEATURE(RAS) ASMJIT_ARM_FEATURE(RAS1_1) ASMJIT_ARM_FEATURE(RAS2) + ASMJIT_ARM_FEATURE(RASSA2) ASMJIT_ARM_FEATURE(RDM) ASMJIT_ARM_FEATURE(RME) ASMJIT_ARM_FEATURE(RNG) ASMJIT_ARM_FEATURE(RNG_TRAP) ASMJIT_ARM_FEATURE(RPRES) ASMJIT_ARM_FEATURE(RPRFM) + ASMJIT_ARM_FEATURE(S1PIE) + ASMJIT_ARM_FEATURE(S1POE) + ASMJIT_ARM_FEATURE(S2PIE) + ASMJIT_ARM_FEATURE(S2POE) ASMJIT_ARM_FEATURE(SB) + ASMJIT_ARM_FEATURE(SCTLR2) + ASMJIT_ARM_FEATURE(SEBEP) + ASMJIT_ARM_FEATURE(SEL2) ASMJIT_ARM_FEATURE(SHA1) ASMJIT_ARM_FEATURE(SHA256) ASMJIT_ARM_FEATURE(SHA3) @@ -725,6 +923,8 @@ public: ASMJIT_ARM_FEATURE(SME) ASMJIT_ARM_FEATURE(SME2) ASMJIT_ARM_FEATURE(SME2_1) + ASMJIT_ARM_FEATURE(SME2_2) + ASMJIT_ARM_FEATURE(SME_AES) ASMJIT_ARM_FEATURE(SME_B16B16) ASMJIT_ARM_FEATURE(SME_B16F32) ASMJIT_ARM_FEATURE(SME_BI32I32) @@ -732,22 +932,50 @@ public: ASMJIT_ARM_FEATURE(SME_F16F32) ASMJIT_ARM_FEATURE(SME_F32F32) ASMJIT_ARM_FEATURE(SME_F64F64) + ASMJIT_ARM_FEATURE(SME_F8F16) + ASMJIT_ARM_FEATURE(SME_F8F32) ASMJIT_ARM_FEATURE(SME_FA64) ASMJIT_ARM_FEATURE(SME_I16I32) ASMJIT_ARM_FEATURE(SME_I16I64) ASMJIT_ARM_FEATURE(SME_I8I32) + ASMJIT_ARM_FEATURE(SME_LUTv2) + ASMJIT_ARM_FEATURE(SME_MOP4) + ASMJIT_ARM_FEATURE(SME_TMOP) + ASMJIT_ARM_FEATURE(SPE) + ASMJIT_ARM_FEATURE(SPE1_1) + ASMJIT_ARM_FEATURE(SPE1_2) + ASMJIT_ARM_FEATURE(SPE1_3) + ASMJIT_ARM_FEATURE(SPE1_4) + ASMJIT_ARM_FEATURE(SPE_ALTCLK) + ASMJIT_ARM_FEATURE(SPE_CRR) + ASMJIT_ARM_FEATURE(SPE_EFT) + ASMJIT_ARM_FEATURE(SPE_FDS) + ASMJIT_ARM_FEATURE(SPE_FPF) + ASMJIT_ARM_FEATURE(SPE_SME) ASMJIT_ARM_FEATURE(SPECRES) ASMJIT_ARM_FEATURE(SPECRES2) + ASMJIT_ARM_FEATURE(SPMU) ASMJIT_ARM_FEATURE(SSBS) ASMJIT_ARM_FEATURE(SSBS2) + ASMJIT_ARM_FEATURE(SSVE_AES) + ASMJIT_ARM_FEATURE(SSVE_BITPERM) + ASMJIT_ARM_FEATURE(SSVE_FEXPA) + ASMJIT_ARM_FEATURE(SSVE_FP8DOT2) + ASMJIT_ARM_FEATURE(SSVE_FP8DOT4) + ASMJIT_ARM_FEATURE(SSVE_FP8FMA) ASMJIT_ARM_FEATURE(SVE) ASMJIT_ARM_FEATURE(SVE2) ASMJIT_ARM_FEATURE(SVE2_1) + ASMJIT_ARM_FEATURE(SVE2_2) ASMJIT_ARM_FEATURE(SVE_AES) + ASMJIT_ARM_FEATURE(SVE_AES2) ASMJIT_ARM_FEATURE(SVE_B16B16) ASMJIT_ARM_FEATURE(SVE_BF16) + ASMJIT_ARM_FEATURE(SVE_BFSCALE) ASMJIT_ARM_FEATURE(SVE_BITPERM) ASMJIT_ARM_FEATURE(SVE_EBF16) + ASMJIT_ARM_FEATURE(SVE_ELTPERM) + ASMJIT_ARM_FEATURE(SVE_F16MM) ASMJIT_ARM_FEATURE(SVE_F32MM) ASMJIT_ARM_FEATURE(SVE_F64MM) ASMJIT_ARM_FEATURE(SVE_I8MM) @@ -757,12 +985,17 @@ public: ASMJIT_ARM_FEATURE(SYSINSTR128) ASMJIT_ARM_FEATURE(SYSREG128) ASMJIT_ARM_FEATURE(THE) + ASMJIT_ARM_FEATURE(TLBIOS) + ASMJIT_ARM_FEATURE(TLBIRANGE) + ASMJIT_ARM_FEATURE(TLBIW) ASMJIT_ARM_FEATURE(TME) ASMJIT_ARM_FEATURE(TRF) ASMJIT_ARM_FEATURE(UAO) ASMJIT_ARM_FEATURE(VFP_D32) ASMJIT_ARM_FEATURE(VHE) + ASMJIT_ARM_FEATURE(VMID16) ASMJIT_ARM_FEATURE(WFXT) + ASMJIT_ARM_FEATURE(XNX) ASMJIT_ARM_FEATURE(XS) #undef ASMJIT_ARM_FEATURE @@ -785,6 +1018,7 @@ public: ASMJIT_INLINE_NODEBUG CpuFeatures() noexcept {} ASMJIT_INLINE_NODEBUG CpuFeatures(const CpuFeatures& other) noexcept = default; + ASMJIT_INLINE_NODEBUG explicit CpuFeatures(const Data& other) noexcept : _data{other._bits} {} ASMJIT_INLINE_NODEBUG explicit CpuFeatures(Globals::NoInit_) noexcept {} //! \} @@ -794,8 +1028,8 @@ public: ASMJIT_INLINE_NODEBUG CpuFeatures& operator=(const CpuFeatures& other) noexcept = default; - ASMJIT_INLINE_NODEBUG bool operator==(const CpuFeatures& other) noexcept { return eq(other); } - ASMJIT_INLINE_NODEBUG bool operator!=(const CpuFeatures& other) noexcept { return !eq(other); } + ASMJIT_INLINE_NODEBUG bool operator==(const CpuFeatures& other) const noexcept { return equals(other); } + ASMJIT_INLINE_NODEBUG bool operator!=(const CpuFeatures& other) const noexcept { return !equals(other); } //! \} @@ -849,6 +1083,7 @@ public: //! \name Manipulation //! \{ + //! Clears all features set. ASMJIT_INLINE_NODEBUG void reset() noexcept { _data.reset(); } //! Adds the given CPU `featureId` to the list of features. @@ -864,7 +1099,7 @@ public: ASMJIT_INLINE_NODEBUG void remove(Args&&... args) noexcept { return _data.remove(std::forward(args)...); } //! Tests whether this CPU features matches `other`. - ASMJIT_INLINE_NODEBUG bool eq(const CpuFeatures& other) const noexcept { return _data.eq(other._data); } + ASMJIT_INLINE_NODEBUG bool equals(const CpuFeatures& other) const noexcept { return _data.equals(other._data); } //! \} }; @@ -876,65 +1111,83 @@ public: //! \{ //! Architecture. - Arch _arch; + Arch _arch {}; //! Sub-architecture. - SubArch _subArch; + SubArch _subArch {}; //! True if the CPU was detected, false if the detection failed or it's not available. - bool _wasDetected; + bool _wasDetected {}; //! Reserved for future use. - uint8_t _reserved; + uint8_t _reserved {}; //! CPU family ID. - uint32_t _familyId; + uint32_t _familyId {}; //! CPU model ID. - uint32_t _modelId; + uint32_t _modelId {}; //! CPU brand ID. - uint32_t _brandId; + uint32_t _brandId {}; //! CPU stepping. - uint32_t _stepping; + uint32_t _stepping {}; //! Processor type. - uint32_t _processorType; + uint32_t _processorType {}; //! Maximum number of addressable IDs for logical processors. - uint32_t _maxLogicalProcessors; + uint32_t _maxLogicalProcessors {}; //! Cache line size (in bytes). - uint32_t _cacheLineSize; + uint32_t _cacheLineSize {}; //! Number of hardware threads. - uint32_t _hwThreadCount; + uint32_t _hwThreadCount {}; //! CPU vendor string. - FixedString<16> _vendor; + FixedString<16> _vendor {}; //! CPU brand string. - FixedString<64> _brand; + FixedString<64> _brand {}; //! CPU features. - CpuFeatures _features; + CpuFeatures _features {}; //! \} //! \name Construction & Destruction //! \{ - ASMJIT_INLINE_NODEBUG CpuInfo() noexcept { reset(); } + //! Creates a new CpuInfo instance. + ASMJIT_INLINE_NODEBUG CpuInfo() noexcept {} + //! Creates a copy of `other` instance. ASMJIT_INLINE_NODEBUG CpuInfo(const CpuInfo& other) noexcept = default; + //! Creates an unitialized `CpuInfo` instance. ASMJIT_INLINE_NODEBUG explicit CpuInfo(Globals::NoInit_) noexcept : _features(Globals::NoInit) {}; + //! \} + + //! \name CPU Information Detection + //! \{ + //! Returns the host CPU information. + //! + //! \note The returned reference is global - it's setup only once and then shared. + [[nodiscard]] ASMJIT_API static const CpuInfo& host() noexcept; + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Copy assignment. + ASMJIT_INLINE_NODEBUG CpuInfo& operator=(const CpuInfo& other) noexcept = default; + + //! \} + + //! \name Initialization & Reset + //! \{ + //! Initializes CpuInfo architecture and sub-architecture members to `arch` and `subArch`, respectively. ASMJIT_INLINE_NODEBUG void initArch(Arch arch, SubArch subArch = SubArch::kUnknown) noexcept { _arch = arch; _subArch = subArch; } - ASMJIT_INLINE_NODEBUG void reset() noexcept { memset(this, 0, sizeof(*this)); } - - //! \} - - //! \name Overloaded Operators - //! \{ - - ASMJIT_INLINE_NODEBUG CpuInfo& operator=(const CpuInfo& other) noexcept = default; + //! Resets this \ref CpuInfo to a default constructed state. + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = CpuInfo{}; } //! \} @@ -942,15 +1195,18 @@ public: //! \{ //! Returns the CPU architecture this information relates to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return _arch; } //! Returns the CPU sub-architecture this information relates to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SubArch subArch() const noexcept { return _subArch; } //! Returns whether the CPU was detected successfully. //! //! If the returned value is false it means that AsmJit either failed to detect the CPU or it doesn't have //! implementation targeting the host architecture and operating system. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool wasDetected() const noexcept { return _wasDetected; } //! Returns the CPU family ID. @@ -960,6 +1216,7 @@ public: //! - Family identifier matches the FamilyId read by using CPUID. //! - ARM: //! - Apple - returns Apple Family identifier returned by sysctlbyname("hw.cpufamily"). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t familyId() const noexcept { return _familyId; } //! Returns the CPU model ID. @@ -967,6 +1224,7 @@ public: //! The information provided depends on architecture and OS: //! - X86: //! - Model identifier matches the ModelId read by using CPUID. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t modelId() const noexcept { return _modelId; } //! Returns the CPU brand id. @@ -974,6 +1232,7 @@ public: //! The information provided depends on architecture and OS: //! - X86: //! - Brand identifier matches the BrandId read by using CPUID. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t brandId() const noexcept { return _brandId; } //! Returns the CPU stepping. @@ -981,6 +1240,7 @@ public: //! The information provided depends on architecture and OS: //! - X86: //! - Stepping identifier matches the Stepping information read by using CPUID. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t stepping() const noexcept { return _stepping; } //! Returns the processor type. @@ -988,34 +1248,46 @@ public: //! The information provided depends on architecture and OS: //! - X86: //! - Processor type identifier matches the ProcessorType read by using CPUID. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t processorType() const noexcept { return _processorType; } //! Returns the maximum number of logical processors. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t maxLogicalProcessors() const noexcept { return _maxLogicalProcessors; } //! Returns the size of a CPU cache line. //! //! On a multi-architecture system this should return the smallest cache line of all CPUs. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t cacheLineSize() const noexcept { return _cacheLineSize; } //! Returns number of hardware threads available. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t hwThreadCount() const noexcept { return _hwThreadCount; } //! Returns a CPU vendor string. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* vendor() const noexcept { return _vendor.str; } + //! Tests whether the CPU vendor string is equal to `s`. - ASMJIT_INLINE_NODEBUG bool isVendor(const char* s) const noexcept { return _vendor.eq(s); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isVendor(const char* s) const noexcept { return _vendor.equals(s); } //! Returns a CPU brand string. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* brand() const noexcept { return _brand.str; } //! Returns CPU features. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CpuFeatures& features() noexcept { return _features; } + //! Returns CPU features (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const CpuFeatures& features() const noexcept { return _features; } //! Tests whether the CPU has the given `feature`. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFeature(const FeatureId& featureId) const noexcept { return _features.has(featureId); } //! Adds the given CPU `featureId` to the list of features. diff --git a/pe-packer/asmjit/core/emithelper.cpp b/pe-packer/asmjit/core/emithelper.cpp index c06bbe2..51a1e6c 100644 --- a/pe-packer/asmjit/core/emithelper.cpp +++ b/pe-packer/asmjit/core/emithelper.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -23,22 +23,27 @@ static void dumpFuncValue(String& sb, Arch arch, const FuncValue& value) noexcep Formatter::formatTypeId(sb, value.typeId()); sb.append('@'); - if (value.isIndirect()) + if (value.isIndirect()) { sb.append('['); + } - if (value.isReg()) + if (value.isReg()) { Formatter::formatRegister(sb, 0, nullptr, arch, value.regType(), value.regId()); - else if (value.isStack()) + } + else if (value.isStack()) { sb.appendFormat("[%d]", value.stackOffset()); - else + } + else { sb.append(""); + } - if (value.isIndirect()) + if (value.isIndirect()) { sb.append(']'); + } } static void dumpAssignment(String& sb, const FuncArgsContext& ctx) noexcept { - typedef FuncArgsContext::Var Var; + using Var = FuncArgsContext::Var; Arch arch = ctx.arch(); uint32_t varCount = ctx.varCount(); @@ -53,8 +58,9 @@ static void dumpAssignment(String& sb, const FuncArgsContext& ctx) noexcept { sb.append(" <- "); dumpFuncValue(sb, arch, cur); - if (var.isDone()) + if (var.isDone()) { sb.append(" {Done}"); + } sb.append('\n'); } @@ -69,12 +75,12 @@ Error BaseEmitHelper::emitRegMove(const Operand_& dst_, const Operand_& src_, Ty return DebugUtils::errored(kErrorInvalidState); } -Error BaseEmitHelper::emitRegSwap(const BaseReg& a, const BaseReg& b, const char* comment) { +Error BaseEmitHelper::emitRegSwap(const Reg& a, const Reg& b, const char* comment) { DebugUtils::unused(a, b, comment); return DebugUtils::errored(kErrorInvalidState); } -Error BaseEmitHelper::emitArgMove(const BaseReg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { +Error BaseEmitHelper::emitArgMove(const Reg& dst_, TypeId dstTypeId, const Operand_& src_, TypeId srcTypeId, const char* comment) { DebugUtils::unused(dst_, dstTypeId, src_, srcTypeId, comment); return DebugUtils::errored(kErrorInvalidState); } @@ -83,8 +89,8 @@ Error BaseEmitHelper::emitArgMove(const BaseReg& dst_, TypeId dstTypeId, const O // =================================== ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args) { - typedef FuncArgsContext::Var Var; - typedef FuncArgsContext::WorkData WorkData; + using Var = FuncArgsContext::Var; + using WorkData = FuncArgsContext::WorkData; enum WorkFlags : uint32_t { kWorkNone = 0x00, @@ -114,14 +120,16 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram uint32_t varCount = ctx._varCount; uint32_t saVarId = ctx._saVarId; - BaseReg sp = BaseReg(_emitter->_gpSignature, archTraits.spRegId()); - BaseReg sa = sp; + Reg sp = Reg(_emitter->_gpSignature, archTraits.spRegId()); + Reg sa = sp; if (frame.hasDynamicAlignment()) { - if (frame.hasPreservedFP()) + if (frame.hasPreservedFP()) { sa.setId(archTraits.fpRegId()); - else + } + else { sa.setId(saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId()); + } } // Register to stack and stack to stack moves must be first as now we have @@ -135,14 +143,15 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram for (uint32_t varId = 0; varId < varCount; varId++) { Var& var = ctx._vars[varId]; - if (!var.out.isStack()) + if (!var.out.isStack()) { continue; + } FuncValue& cur = var.cur; FuncValue& out = var.out; ASMJIT_ASSERT(cur.isReg() || cur.isStack()); - BaseReg reg; + Reg reg; BaseMem dstStackPtr = baseStackPtr.cloneAdjusted(out.stackOffset()); BaseMem srcStackPtr = baseArgPtr.cloneAdjusted(cur.stackOffset()); @@ -158,10 +167,10 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram } if (cur.isReg() && !cur.isIndirect()) { - WorkData& wd = workData[archTraits.regTypeToGroup(cur.regType())]; + WorkData& wd = workData[RegUtils::groupOf(cur.regType())]; uint32_t regId = cur.regId(); - reg.setSignatureAndId(archTraits.regTypeToSignature(cur.regType()), regId); + reg.setSignatureAndId(RegUtils::signatureOf(cur.regType()), regId); wd.unassign(varId, regId); } else { @@ -169,13 +178,15 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram // we follow the rule that IntToInt moves will use GP regs with possibility to signature or zero extend, // and all other moves will either use GP or VEC regs depending on the size of the move. OperandSignature signature = getSuitableRegForMemToMemMove(arch, out.typeId(), cur.typeId()); - if (ASMJIT_UNLIKELY(!signature.isValid())) + if (ASMJIT_UNLIKELY(!signature.isValid())) { return DebugUtils::errored(kErrorInvalidState); + } WorkData& wd = workData[signature.regGroup()]; RegMask availableRegs = wd.availableRegs(); - if (ASMJIT_UNLIKELY(!availableRegs)) + if (ASMJIT_UNLIKELY(!availableRegs)) { return DebugUtils::errored(kErrorInvalidState); + } uint32_t availableId = Support::ctz(availableRegs); reg.setSignatureAndId(signature, availableId); @@ -183,8 +194,9 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram ASMJIT_PROPAGATE(emitArgMove(reg, out.typeId(), srcStackPtr, cur.typeId())); } - if (cur.isIndirect() && cur.isReg()) + if (cur.isIndirect() && cur.isReg()) { workData[RegGroup::kGp].unassign(varId, cur.regId()); + } // Register to stack move. ASMJIT_PROPAGATE(emitRegMove(dstStackPtr, reg, cur.typeId())); @@ -198,14 +210,15 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram for (;;) { for (uint32_t varId = 0; varId < varCount; varId++) { Var& var = ctx._vars[varId]; - if (var.isDone() || !var.cur.isReg()) + if (var.isDone() || !var.cur.isReg()) { continue; + } FuncValue& cur = var.cur; FuncValue& out = var.out; - RegGroup curGroup = archTraits.regTypeToGroup(cur.regType()); - RegGroup outGroup = archTraits.regTypeToGroup(out.regType()); + RegGroup curGroup = RegUtils::groupOf(cur.regType()); + RegGroup outGroup = RegUtils::groupOf(out.regType()); uint32_t curId = cur.regId(); uint32_t outId = out.regId(); @@ -216,18 +229,23 @@ ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& fram } else { WorkData& wd = workData[outGroup]; - if (!wd.isAssigned(outId)) { + if (!wd.isAssigned(outId) || curId == outId) { EmitMove: ASMJIT_PROPAGATE( emitArgMove( - BaseReg(archTraits.regTypeToSignature(out.regType()), outId), out.typeId(), - BaseReg(archTraits.regTypeToSignature(cur.regType()), curId), cur.typeId())); + Reg(RegUtils::signatureOf(out.regType()), outId), out.typeId(), + Reg(RegUtils::signatureOf(cur.regType()), curId), cur.typeId())); + + // Only reassign if this is not a sign/zero extension that happens on the same in/out register. + if (curId != outId) { + wd.reassign(varId, outId, curId); + } - wd.reassign(varId, outId, curId); cur.initReg(out.regType(), outId, out.typeId()); - if (outId == out.regId()) + if (outId == out.regId()) { var.markDone(); + } workFlags |= kWorkDidSome | kWorkPending; } else { @@ -238,20 +256,21 @@ EmitMove: // Only few architectures provide swap operations, and only for few register groups. if (archTraits.hasInstRegSwap(curGroup)) { RegType highestType = Support::max(cur.regType(), altVar.cur.regType()); - if (Support::isBetween(highestType, RegType::kGp8Lo, RegType::kGp16)) + if (Support::isBetween(highestType, RegType::kGp8Lo, RegType::kGp16)) { highestType = RegType::kGp32; + } - OperandSignature signature = archTraits.regTypeToSignature(highestType); - ASMJIT_PROPAGATE( - emitRegSwap(BaseReg(signature, outId), BaseReg(signature, curId))); + OperandSignature signature = RegUtils::signatureOf(highestType); + ASMJIT_PROPAGATE(emitRegSwap(Reg(signature, outId), Reg(signature, curId))); wd.swap(varId, curId, altId, outId); cur.setRegId(outId); var.markDone(); altVar.cur.setRegId(curId); - if (altVar.out.isInitialized()) + if (altVar.out.isInitialized()) { altVar.markDone(); + } workFlags |= kWorkDidSome; } else { @@ -259,8 +278,9 @@ EmitMove: RegMask availableRegs = wd.availableRegs(); if (availableRegs) { RegMask inOutRegs = wd.dstRegs(); - if (availableRegs & ~inOutRegs) + if (availableRegs & ~inOutRegs) { availableRegs &= ~inOutRegs; + } outId = Support::ctz(availableRegs); goto EmitMove; } @@ -276,12 +296,14 @@ EmitMove: } } - if (!(workFlags & kWorkPending)) + if (!(workFlags & kWorkPending)) { break; + } // If we did nothing twice it means that something is really broken. - if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed) + if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed) { return DebugUtils::errored(kErrorInvalidState); + } workFlags = (workFlags & kWorkDidSome) ? kWorkNone : kWorkPostponed; } @@ -291,8 +313,9 @@ EmitMove: if (ctx._hasStackSrc) { uint32_t iterCount = 1; - if (frame.hasDynamicAlignment() && !frame.hasPreservedFP()) + if (frame.hasDynamicAlignment() && !frame.hasPreservedFP()) { sa.setId(saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId()); + } // Base address of all arguments passed by stack. BaseMem baseArgPtr(sa, int32_t(frame.saOffset(sa.id()))); @@ -300,8 +323,9 @@ EmitMove: for (uint32_t iter = 0; iter < iterCount; iter++) { for (uint32_t varId = 0; varId < varCount; varId++) { Var& var = ctx._vars[varId]; - if (var.isDone()) + if (var.isDone()) { continue; + } if (var.cur.isStack()) { ASMJIT_ASSERT(var.out.isReg()); @@ -309,7 +333,7 @@ EmitMove: uint32_t outId = var.out.regId(); RegType outType = var.out.regType(); - RegGroup group = archTraits.regTypeToGroup(outType); + RegGroup group = RegUtils::groupOf(outType); WorkData& wd = workData[group]; if (outId == sa.id() && group == RegGroup::kGp) { @@ -321,7 +345,7 @@ EmitMove: wd.unassign(wd._physToVarId[outId], outId); } - BaseReg dstReg = BaseReg(archTraits.regTypeToSignature(outType), outId); + Reg dstReg = Reg(RegUtils::signatureOf(outType), outId); BaseMem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset()); ASMJIT_PROPAGATE(emitArgMove( diff --git a/pe-packer/asmjit/core/emithelper_p.h b/pe-packer/asmjit/core/emithelper_p.h index f597422..32add43 100644 --- a/pe-packer/asmjit/core/emithelper_p.h +++ b/pe-packer/asmjit/core/emithelper_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITHELPER_P_H_INCLUDED @@ -24,19 +24,23 @@ public: ASMJIT_INLINE_NODEBUG explicit BaseEmitHelper(BaseEmitter* emitter = nullptr) noexcept : _emitter(emitter) {} + ASMJIT_INLINE_NODEBUG virtual ~BaseEmitHelper() noexcept = default; + + [[nodiscard]] ASMJIT_INLINE_NODEBUG BaseEmitter* emitter() const noexcept { return _emitter; } + ASMJIT_INLINE_NODEBUG void setEmitter(BaseEmitter* emitter) noexcept { _emitter = emitter; } //! Emits a pure move operation between two registers or the same type or between a register and its home //! slot. This function does not handle register conversion. virtual Error emitRegMove( const Operand_& dst_, - const Operand_& src_, TypeId typeId, const char* comment = nullptr) = 0; + const Operand_& src_, TypeId typeId, const char* comment = nullptr); //! Emits swap between two registers. virtual Error emitRegSwap( - const BaseReg& a, - const BaseReg& b, const char* comment = nullptr) = 0; + const Reg& a, + const Reg& b, const char* comment = nullptr); //! Emits move from a function argument (either register or stack) to a register. //! @@ -44,8 +48,8 @@ public: //! to another, if it's possible. Any attempt of conversion that requires third register of a different group //! (for example conversion from K to MMX on X86/X64) will fail. virtual Error emitArgMove( - const BaseReg& dst_, TypeId dstTypeId, - const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) = 0; + const Reg& dst_, TypeId dstTypeId, + const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr); Error emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args); }; diff --git a/pe-packer/asmjit/core/emitter.cpp b/pe-packer/asmjit/core/emitter.cpp index fd036b1..3f76507 100644 --- a/pe-packer/asmjit/core/emitter.cpp +++ b/pe-packer/asmjit/core/emitter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -52,17 +52,21 @@ static ASMJIT_NOINLINE void BaseEmitter_updateForcedOptions(BaseEmitter* self) n hasDiagnosticOptions = self->hasDiagnosticOption(DiagnosticOptions::kValidateIntermediate); } - if (emitComments) + if (emitComments) { self->_addEmitterFlags(EmitterFlags::kLogComments); - else + } + else { self->_clearEmitterFlags(EmitterFlags::kLogComments); + } // The reserved option tells emitter (Assembler/Builder/Compiler) that there may be either a border // case (CodeHolder not attached, for example) or that logging or validation is required. - if (self->_code == nullptr || self->_logger || hasDiagnosticOptions) + if (self->_code == nullptr || self->_logger || hasDiagnosticOptions) { self->_forcedInstOptions |= InstOptions::kReserved; - else + } + else { self->_forcedInstOptions &= ~InstOptions::kReserved; + } } // BaseEmitter - Diagnostic Options @@ -90,8 +94,9 @@ void BaseEmitter::setLogger(Logger* logger) noexcept { else { _logger = nullptr; _clearEmitterFlags(EmitterFlags::kOwnLogger); - if (_code) + if (_code) { _logger = _code->logger(); + } } BaseEmitter_updateForcedOptions(this); #else @@ -110,18 +115,23 @@ void BaseEmitter::setErrorHandler(ErrorHandler* errorHandler) noexcept { else { _errorHandler = nullptr; _clearEmitterFlags(EmitterFlags::kOwnErrorHandler); - if (_code) + if (_code) { _errorHandler = _code->errorHandler(); + } } } -Error BaseEmitter::reportError(Error err, const char* message) { +Error BaseEmitter::_reportError(Error err, const char* message) { + ASMJIT_ASSERT(err != kErrorOk); + ErrorHandler* eh = _errorHandler; if (eh) { - if (!message) + if (!message) { message = DebugUtils::errorAsString(err); + } eh->handleError(err, message, this); } + return err; } @@ -289,6 +299,12 @@ Error BaseEmitter::embedDataArray(TypeId typeId, const void* data, size_t itemCo return DebugUtils::errored(kErrorInvalidState); } +// [[pure virtual]] +Error BaseEmitter::embedConstPool(const Label& label, const ConstPool& pool) { + DebugUtils::unused(label, pool); + return DebugUtils::errored(kErrorInvalidState); +} + // [[pure virtual]] Error BaseEmitter::embedLabel(const Label& label, size_t dataSize) { DebugUtils::unused(label, dataSize); @@ -305,15 +321,16 @@ Error BaseEmitter::embedLabelDelta(const Label& label, const Label& base, size_t // ===================== // [[pure virtual]] -Error comment(const char* data, size_t size = SIZE_MAX) { +Error BaseEmitter::comment(const char* data, size_t size) { DebugUtils::unused(data, size); return DebugUtils::errored(kErrorInvalidState); } Error BaseEmitter::commentf(const char* fmt, ...) { if (!hasEmitterFlag(EmitterFlags::kLogComments)) { - if (!hasEmitterFlag(EmitterFlags::kAttached)) + if (!hasEmitterFlag(EmitterFlags::kAttached)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } return kErrorOk; } @@ -335,8 +352,9 @@ Error BaseEmitter::commentf(const char* fmt, ...) { Error BaseEmitter::commentv(const char* fmt, va_list ap) { if (!hasEmitterFlag(EmitterFlags::kLogComments)) { - if (!hasEmitterFlag(EmitterFlags::kAttached)) + if (!hasEmitterFlag(EmitterFlags::kAttached)) { return reportError(DebugUtils::errored(kErrorNotInitialized)); + } return kErrorOk; } @@ -355,29 +373,34 @@ Error BaseEmitter::commentv(const char* fmt, va_list ap) { // BaseEmitter - Events // ==================== -Error BaseEmitter::onAttach(CodeHolder* code) noexcept { - _code = code; - _environment = code->environment(); +Error BaseEmitter::onAttach(CodeHolder& code) noexcept { + _code = &code; + _environment = code.environment(); _addEmitterFlags(EmitterFlags::kAttached); - const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); - RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; - _gpSignature = archTraits.regTypeToSignature(nativeRegType); + _gpSignature.setBits( + Environment::is32Bit(code.arch()) + ? RegTraits::kSignature + : RegTraits::kSignature + ); onSettingsUpdated(); return kErrorOk; } -Error BaseEmitter::onDetach(CodeHolder* code) noexcept { +Error BaseEmitter::onDetach(CodeHolder& code) noexcept { DebugUtils::unused(code); - if (!hasOwnLogger()) + if (!hasOwnLogger()) { _logger = nullptr; + } - if (!hasOwnErrorHandler()) + if (!hasOwnErrorHandler()) { _errorHandler = nullptr; + } _clearEmitterFlags(~kEmitterPreservedFlags); + _instructionAlignment = uint8_t(0); _forcedInstOptions = InstOptions::kReserved; _privateData = 0; @@ -391,15 +414,28 @@ Error BaseEmitter::onDetach(CodeHolder* code) noexcept { return kErrorOk; } +Error BaseEmitter::onReinit(CodeHolder& code) noexcept { + ASMJIT_ASSERT(_code == &code); + DebugUtils::unused(code); + + _instOptions = InstOptions::kNone; + _extraReg.reset(); + _inlineComment = nullptr; + + return kErrorOk; +} + void BaseEmitter::onSettingsUpdated() noexcept { // Only called when attached to CodeHolder by CodeHolder. ASMJIT_ASSERT(_code != nullptr); - if (!hasOwnLogger()) + if (!hasOwnLogger()) { _logger = _code->logger(); + } - if (!hasOwnErrorHandler()) + if (!hasOwnErrorHandler()) { _errorHandler = _code->errorHandler(); + } BaseEmitter_updateForcedOptions(this); } diff --git a/pe-packer/asmjit/core/emitter.h b/pe-packer/asmjit/core/emitter.h index e3cdd56..93f6369 100644 --- a/pe-packer/asmjit/core/emitter.h +++ b/pe-packer/asmjit/core/emitter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITTER_H_INCLUDED @@ -186,7 +186,7 @@ enum class DiagnosticOptions : uint32_t { kRADebugUnreachable = 0x00000800u, //! Enable all debug options (Compiler/RA). - kRADebugAll = 0x0000FF00u, + kRADebugAll = 0x0000FF00u }; ASMJIT_DEFINE_ENUM_FLAGS(DiagnosticOptions) @@ -194,45 +194,11 @@ ASMJIT_DEFINE_ENUM_FLAGS(DiagnosticOptions) class ASMJIT_VIRTAPI BaseEmitter { public: ASMJIT_BASE_CLASS(BaseEmitter) + ASMJIT_NONCOPYABLE(BaseEmitter) - //! \name Members + //! \name Types //! \{ - //! See \ref EmitterType. - EmitterType _emitterType = EmitterType::kNone; - //! See \ref EmitterFlags. - EmitterFlags _emitterFlags = EmitterFlags::kNone; - //! Validation flags in case validation is used. - //! - //! \note Validation flags are specific to the emitter and they are setup at construction time and then never - //! changed. - ValidationFlags _validationFlags = ValidationFlags::kNone; - //! Validation options. - DiagnosticOptions _diagnosticOptions = DiagnosticOptions::kNone; - - //! All supported architectures in a bit-mask, where LSB is the bit with a zero index. - uint64_t _archMask = 0; - - //! Encoding options. - EncodingOptions _encodingOptions = EncodingOptions::kNone; - - //! Forced instruction options, combined with \ref _instOptions by \ref emit(). - InstOptions _forcedInstOptions = InstOptions::kReserved; - //! Internal private data used freely by any emitter. - uint32_t _privateData = 0; - - //! CodeHolder the emitter is attached to. - CodeHolder* _code = nullptr; - //! Attached \ref Logger. - Logger* _logger = nullptr; - //! Attached \ref ErrorHandler. - ErrorHandler* _errorHandler = nullptr; - - //! Describes the target environment, matches \ref CodeHolder::environment(). - Environment _environment {}; - //! Native GP register signature and signature related information. - OperandSignature _gpSignature {}; - //! Emitter state that can be used to specify options and inline comment of a next node or instruction. struct State { InstOptions options; @@ -240,29 +206,22 @@ public: const char* comment; }; - //! Next instruction options (affects the next instruction). - InstOptions _instOptions = InstOptions::kNone; - //! Extra register (op-mask {k} on AVX-512) (affects the next instruction). - RegOnly _extraReg {}; - //! Inline comment of the next instruction (affects the next instruction). - const char* _inlineComment = nullptr; - - //! Function callbacks used by emitter implementation. + //! Functions used by backend-specific emitter implementation. //! //! These are typically shared between Assembler/Builder/Compiler of a single backend. struct Funcs { - typedef Error (ASMJIT_CDECL* EmitProlog)(BaseEmitter* emitter, const FuncFrame& frame); - typedef Error (ASMJIT_CDECL* EmitEpilog)(BaseEmitter* emitter, const FuncFrame& frame); - typedef Error (ASMJIT_CDECL* EmitArgsAssignment)(BaseEmitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args); + using EmitProlog = Error (ASMJIT_CDECL*)(BaseEmitter* emitter, const FuncFrame& frame); + using EmitEpilog = Error (ASMJIT_CDECL*)(BaseEmitter* emitter, const FuncFrame& frame); + using EmitArgsAssignment = Error (ASMJIT_CDECL*)(BaseEmitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args); - typedef Error (ASMJIT_CDECL* FormatInstruction)( + using FormatInstruction = Error (ASMJIT_CDECL*)( String& sb, FormatFlags formatFlags, const BaseEmitter* emitter, Arch arch, - const BaseInst& inst, const Operand_* operands, size_t opCount) ASMJIT_NOEXCEPT_TYPE; + const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept; - typedef Error (ASMJIT_CDECL* ValidateFunc)(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) ASMJIT_NOEXCEPT_TYPE; + using ValidateFunc = Error (ASMJIT_CDECL*)(const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept; //! Emit prolog implementation. EmitProlog emitProlog; @@ -276,16 +235,73 @@ public: ValidateFunc validate; //! Resets all functions to nullptr. - ASMJIT_INLINE_NODEBUG void reset() noexcept { - emitProlog = nullptr; - emitEpilog = nullptr; - emitArgsAssignment = nullptr; - validate = nullptr; - } + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = Funcs{}; } }; + //! \} + + //! \name Members + //! \{ + + //! See \ref EmitterType. + EmitterType _emitterType = EmitterType::kNone; + + //! See \ref EmitterFlags. + EmitterFlags _emitterFlags = EmitterFlags::kNone; + + //! Instruction alignment. + uint8_t _instructionAlignment = 0u; + + //! Validation flags in case validation is used. + //! + //! \note Validation flags are specific to the emitter and they are setup at construction time and then never + //! changed. + ValidationFlags _validationFlags = ValidationFlags::kNone; + + //! Validation options. + DiagnosticOptions _diagnosticOptions = DiagnosticOptions::kNone; + + //! Encoding options. + EncodingOptions _encodingOptions = EncodingOptions::kNone; + + //! Forced instruction options, combined with \ref _instOptions by \ref emit(). + InstOptions _forcedInstOptions = InstOptions::kReserved; + + //! All supported architectures in a bit-mask, where LSB is the bit with a zero index. + uint64_t _archMask = 0; + + //! CodeHolder the emitter is attached to. + CodeHolder* _code = nullptr; + + //! Attached \ref Logger. + Logger* _logger = nullptr; + + //! Attached \ref ErrorHandler. + ErrorHandler* _errorHandler = nullptr; + + //! Describes the target environment, matches \ref CodeHolder::environment(). + Environment _environment {}; + + //! Native GP register signature (either a 32-bit or 64-bit GP register signature). + OperandSignature _gpSignature {}; + //! Internal private data used freely by any emitter. + uint32_t _privateData = 0; + + //! Next instruction options (affects the next instruction). + InstOptions _instOptions = InstOptions::kNone; + //! Extra register (op-mask {k} on AVX-512) (affects the next instruction). + RegOnly _extraReg {}; + //! Inline comment of the next instruction (affects the next instruction). + const char* _inlineComment = nullptr; + + //! Pointer to functions used by backend-specific emitter implementation. Funcs _funcs {}; + //! Emitter attached before this emitter in \ref CodeHolder, otherwise nullptr if there is no emitter before. + BaseEmitter* _attachedPrev = nullptr; + //! Emitter attached after this emitter in \ref CodeHolder, otherwise nullptr if there is no emitter after. + BaseEmitter* _attachedNext = nullptr; + //! \} //! \name Construction & Destruction @@ -300,9 +316,11 @@ public: //! \{ template + [[nodiscard]] ASMJIT_INLINE_NODEBUG T* as() noexcept { return reinterpret_cast(this); } template + [[nodiscard]] ASMJIT_INLINE_NODEBUG const T* as() const noexcept { return reinterpret_cast(this); } //! \} @@ -311,61 +329,104 @@ public: //! \{ //! Returns the type of this emitter, see `EmitterType`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG EmitterType emitterType() const noexcept { return _emitterType; } + //! Returns emitter flags , see `Flags`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG EmitterFlags emitterFlags() const noexcept { return _emitterFlags; } //! Tests whether the emitter inherits from `BaseAssembler`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAssembler() const noexcept { return _emitterType == EmitterType::kAssembler; } + //! Tests whether the emitter inherits from `BaseBuilder`. //! //! \note Both Builder and Compiler emitters would return `true`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isBuilder() const noexcept { return uint32_t(_emitterType) >= uint32_t(EmitterType::kBuilder); } + //! Tests whether the emitter inherits from `BaseCompiler`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isCompiler() const noexcept { return _emitterType == EmitterType::kCompiler; } //! Tests whether the emitter has the given `flag` enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasEmitterFlag(EmitterFlags flag) const noexcept { return Support::test(_emitterFlags, flag); } + //! Tests whether the emitter is finalized. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFinalized() const noexcept { return hasEmitterFlag(EmitterFlags::kFinalized); } + //! Tests whether the emitter is destroyed (only used during destruction). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isDestroyed() const noexcept { return hasEmitterFlag(EmitterFlags::kDestroyed); } + //! \} + + //! \cond INTERNAL + //! \name Internal Functions + //! \{ + ASMJIT_INLINE_NODEBUG void _addEmitterFlags(EmitterFlags flags) noexcept { _emitterFlags |= flags; } ASMJIT_INLINE_NODEBUG void _clearEmitterFlags(EmitterFlags flags) noexcept { _emitterFlags &= _emitterFlags & ~flags; } //! \} + //! \endcond //! \name Target Information //! \{ //! Returns the CodeHolder this emitter is attached to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CodeHolder* code() const noexcept { return _code; } //! Returns the target environment. //! //! The returned \ref Environment reference matches \ref CodeHolder::environment(). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const Environment& environment() const noexcept { return _environment; } //! Tests whether the target architecture is 32-bit. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool is32Bit() const noexcept { return environment().is32Bit(); } + //! Tests whether the target architecture is 64-bit. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool is64Bit() const noexcept { return environment().is64Bit(); } //! Returns the target architecture type. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return environment().arch(); } + //! Returns the target architecture sub-type. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SubArch subArch() const noexcept { return environment().subArch(); } //! Returns the target architecture's GP register size (4 or 8 bytes). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t registerSize() const noexcept { return environment().registerSize(); } + //! Returns a signature of a native general purpose register (either 32-bit or 64-bit depending on the architecture). + [[nodiscard]] + ASMJIT_INLINE_NODEBUG OperandSignature gpSignature() const noexcept { return _gpSignature; } + + //! Returns instruction alignment. + //! + //! The following values are returned based on the target architecture: + //! - X86 and X86_64 - instruction alignment is 1 + //! - AArch32 - instruction alignment is 4 in A32 mode and 2 in THUMB mode. + //! - AArch64 - instruction alignment is 4 + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t instructionAlignment() const noexcept { return _instructionAlignment; } + //! \} //! \name Initialization & Finalization //! \{ //! Tests whether the emitter is initialized (i.e. attached to \ref CodeHolder). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInitialized() const noexcept { return _code != nullptr; } //! Finalizes this emitter. @@ -383,18 +444,21 @@ public: //! \{ //! Tests whether the emitter has a logger. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasLogger() const noexcept { return _logger != nullptr; } //! Tests whether the emitter has its own logger. //! //! Own logger means that it overrides the possible logger that may be used by \ref CodeHolder this emitter is //! attached to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOwnLogger() const noexcept { return hasEmitterFlag(EmitterFlags::kOwnLogger); } //! Returns the logger this emitter uses. //! //! The returned logger is either the emitter's own logger or it's logger used by \ref CodeHolder this emitter //! is attached to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Logger* logger() const noexcept { return _logger; } //! Sets or resets the logger of the emitter. @@ -416,18 +480,21 @@ public: //! \{ //! Tests whether the emitter has an error handler attached. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasErrorHandler() const noexcept { return _errorHandler != nullptr; } //! Tests whether the emitter has its own error handler. //! //! Own error handler means that it overrides the possible error handler that may be used by \ref CodeHolder this //! emitter is attached to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOwnErrorHandler() const noexcept { return hasEmitterFlag(EmitterFlags::kOwnErrorHandler); } //! Returns the error handler this emitter uses. //! //! The returned error handler is either the emitter's own error handler or it's error handler used by //! \ref CodeHolder this emitter is attached to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ErrorHandler* errorHandler() const noexcept { return _errorHandler; } //! Sets or resets the error handler of the emitter. @@ -436,11 +503,23 @@ public: //! Resets the error handler. ASMJIT_INLINE_NODEBUG void resetErrorHandler() noexcept { setErrorHandler(nullptr); } + //! \cond INTERNAL + ASMJIT_API Error _reportError(Error err, const char* message = nullptr); + //! \endcond + //! Handles the given error in the following way: //! 1. If the emitter has \ref ErrorHandler attached, it calls its \ref ErrorHandler::handleError() member function //! first, and then returns the error. The `handleError()` function may throw. //! 2. if the emitter doesn't have \ref ErrorHandler, the error is simply returned. - ASMJIT_API Error reportError(Error err, const char* message = nullptr); + ASMJIT_INLINE Error reportError(Error err, const char* message = nullptr) { + Error e = _reportError(err, message); + + // Static analysis is not working properly without these assumptions. + ASMJIT_ASSUME(e == err); + ASMJIT_ASSUME(e != kErrorOk); + + return e; + } //! \} @@ -448,8 +527,11 @@ public: //! \{ //! Returns encoding options. + [[nodiscard]] ASMJIT_INLINE_NODEBUG EncodingOptions encodingOptions() const noexcept { return _encodingOptions; } + //! Tests whether the encoding `option` is set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasEncodingOption(EncodingOptions option) const noexcept { return Support::test(_encodingOptions, option); } //! Enables the given encoding `options`. @@ -463,9 +545,11 @@ public: //! \{ //! Returns the emitter's diagnostic options. + [[nodiscard]] ASMJIT_INLINE_NODEBUG DiagnosticOptions diagnosticOptions() const noexcept { return _diagnosticOptions; } //! Tests whether the given `option` is present in the emitter's diagnostic options. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasDiagnosticOption(DiagnosticOptions option) const noexcept { return Support::test(_diagnosticOptions, option); } //! Activates the given diagnostic `options`. @@ -503,35 +587,49 @@ public: //! Forced instruction options are merged with next instruction options before the instruction is encoded. These //! options have some bits reserved that are used by error handling, logging, and instruction validation purposes. //! Other options are globals that affect each instruction. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstOptions forcedInstOptions() const noexcept { return _forcedInstOptions; } //! Returns options of the next instruction. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstOptions instOptions() const noexcept { return _instOptions; } + //! Returns options of the next instruction. ASMJIT_INLINE_NODEBUG void setInstOptions(InstOptions options) noexcept { _instOptions = options; } + //! Adds options of the next instruction. ASMJIT_INLINE_NODEBUG void addInstOptions(InstOptions options) noexcept { _instOptions |= options; } + //! Resets options of the next instruction. ASMJIT_INLINE_NODEBUG void resetInstOptions() noexcept { _instOptions = InstOptions::kNone; } //! Tests whether the extra register operand is valid. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasExtraReg() const noexcept { return _extraReg.isReg(); } + //! Returns an extra operand that will be used by the next instruction (architecture specific). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _extraReg; } + //! Sets an extra operand that will be used by the next instruction (architecture specific). - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _extraReg.init(reg); } + //! Sets an extra operand that will be used by the next instruction (architecture specific). ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } + //! Resets an extra operand that will be used by the next instruction (architecture specific). ASMJIT_INLINE_NODEBUG void resetExtraReg() noexcept { _extraReg.reset(); } //! Returns comment/annotation of the next instruction. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* inlineComment() const noexcept { return _inlineComment; } + //! Sets comment/annotation of the next instruction. //! //! \note This string is set back to null by `_emit()`, but until that it has to remain valid as the Emitter is not //! required to make a copy of it (and it would be slow to do that for each instruction). ASMJIT_INLINE_NODEBUG void setInlineComment(const char* s) noexcept { _inlineComment = s; } + //! Resets the comment/annotation to nullptr. ASMJIT_INLINE_NODEBUG void resetInlineComment() noexcept { _inlineComment = nullptr; } @@ -540,24 +638,40 @@ public: //! \name Emitter State //! \{ + //! Resets the emitter state, which contains instruction options, extra register, and inline comment. + //! + //! Emitter can have a state that describes instruction options and extra register used by the instruction. Most + //! instructions don't need nor use the state, however, if an instruction uses a prefix such as REX or REP prefix, + //! which is set explicitly, then the state would contain it. This allows to mimic the syntax of assemblers such + //! as X86. For example `rep().movs(...)` would map to a `REP MOVS` instuction on X86. The same applies to various + //! hints and the use of a mask register in AVX-512 mode. ASMJIT_INLINE_NODEBUG void resetState() noexcept { resetInstOptions(); resetExtraReg(); resetInlineComment(); } + //! \cond INTERNAL + + //! Grabs the current emitter state and resets the emitter state at the same time, returning the state the emitter + //! had before the state was reset. + [[nodiscard]] ASMJIT_INLINE_NODEBUG State _grabState() noexcept { State s{_instOptions | _forcedInstOptions, _extraReg, _inlineComment}; resetState(); return s; } + //! \endcond //! \} //! \name Sections //! \{ - ASMJIT_API virtual Error section(Section* section) = 0; + //! Switches the given `section`. + //! + //! Once switched, everything is added to the given `section`. + ASMJIT_API virtual Error section(Section* section); //! \} @@ -565,13 +679,19 @@ public: //! \{ //! Creates a new label. - ASMJIT_API virtual Label newLabel() = 0; + [[nodiscard]] + ASMJIT_API virtual Label newLabel(); + //! Creates a new named label. - ASMJIT_API virtual Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId) = 0; + [[nodiscard]] + ASMJIT_API virtual Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId); //! Creates a new anonymous label with a name, which can only be used for debugging purposes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label newAnonymousLabel(const char* name, size_t nameSize = SIZE_MAX) { return newNamedLabel(name, nameSize, LabelType::kAnonymous); } + //! Creates a new external label. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Label newExternalLabel(const char* name, size_t nameSize = SIZE_MAX) { return newNamedLabel(name, nameSize, LabelType::kExternal); } //! Returns `Label` by `name`. @@ -580,16 +700,20 @@ public: //! //! \note This function doesn't trigger ErrorHandler in case the name is invalid or no such label exist. You must //! always check the validity of the `Label` returned. + [[nodiscard]] ASMJIT_API Label labelByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept; //! Binds the `label` to the current position of the current section. //! //! \note Attempt to bind the same label multiple times will return an error. - ASMJIT_API virtual Error bind(const Label& label) = 0; + ASMJIT_API virtual Error bind(const Label& label); //! Tests whether the label `id` is valid (i.e. registered). + [[nodiscard]] ASMJIT_API bool isLabelValid(uint32_t labelId) const noexcept; + //! Tests whether the `label` is valid (i.e. registered). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLabelValid(const Label& label) const noexcept { return isLabelValid(label.id()); } //! \} @@ -620,35 +744,51 @@ public: ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5); //! Emits an instruction `instId` with the given `operands`. + //! + //! This is the most universal way of emitting code, which accepts an instruction identifier and instruction + //! operands. This is called an "unchecked" API as emit doesn't provide any type checks at compile-time. This + //! allows to emit instruction with just \ref Operand instances, which could be handy in some cases - for + //! example emitting generic code where you don't know whether some operand is register, memory, or immediate. template ASMJIT_INLINE_NODEBUG Error emit(InstId instId, Args&&... operands) { return _emitI(instId, Support::ForwardOp::forward(operands)...); } + //! Similar to \ref emit(), but uses array of `operands` instead. ASMJIT_INLINE_NODEBUG Error emitOpArray(InstId instId, const Operand_* operands, size_t opCount) { return _emitOpArray(instId, operands, opCount); } - ASMJIT_FORCE_INLINE Error emitInst(const BaseInst& inst, const Operand_* operands, size_t opCount) { + //! Similar to \ref emit(), but emits instruction with both instruction options and extra register, followed + //! by an array of `operands`. + ASMJIT_INLINE Error emitInst(const BaseInst& inst, const Operand_* operands, size_t opCount) { setInstOptions(inst.options()); setExtraReg(inst.extraReg()); return _emitOpArray(inst.id(), operands, opCount); } + //! \} + //! \cond INTERNAL + //! \name Emit Internals + //! \{ + //! Emits an instruction - all 6 operands must be defined. - ASMJIT_API virtual Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* oExt) = 0; + ASMJIT_API virtual Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* oExt); //! Emits instruction having operands stored in array. ASMJIT_API virtual Error _emitOpArray(InstId instId, const Operand_* operands, size_t opCount); - //! \endcond //! \} + //! \endcond //! \name Emit Utilities //! \{ + //! Emits a function prolog described by the given function `frame`. ASMJIT_API Error emitProlog(const FuncFrame& frame); + //! Emits a function epilog described by the given function `frame`. ASMJIT_API Error emitEpilog(const FuncFrame& frame); + //! Emits code that reassigns function `frame` arguments to the given `args`. ASMJIT_API Error emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args); //! \} @@ -661,7 +801,7 @@ public: //! The sequence that is used to fill the gap between the aligned location and the current location depends on the //! align `mode`, see \ref AlignMode. The `alignment` argument specifies alignment in bytes, so for example when //! it's `32` it means that the code buffer will be aligned to `32` bytes. - ASMJIT_API virtual Error align(AlignMode alignMode, uint32_t alignment) = 0; + ASMJIT_API virtual Error align(AlignMode alignMode, uint32_t alignment); //! \} @@ -669,7 +809,7 @@ public: //! \{ //! Embeds raw data into the \ref CodeBuffer. - ASMJIT_API virtual Error embed(const void* data, size_t dataSize) = 0; + ASMJIT_API virtual Error embed(const void* data, size_t dataSize); //! Embeds a typed data array. //! @@ -680,7 +820,7 @@ public: //! //! - Repeat the given data `repeatCount` times, so the data can be used as a fill pattern for example, or as a //! pattern used by SIMD instructions. - ASMJIT_API virtual Error embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1) = 0; + ASMJIT_API virtual Error embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1); //! Embeds int8_t `value` repeated by `repeatCount`. ASMJIT_INLINE_NODEBUG Error embedInt8(int8_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kInt8, &value, 1, repeatCount); } @@ -707,17 +847,17 @@ public: //! 1. Aligns by using AlignMode::kData to the minimum `pool` alignment. //! 2. Binds the ConstPool label so it's bound to an aligned location. //! 3. Emits ConstPool content. - ASMJIT_API virtual Error embedConstPool(const Label& label, const ConstPool& pool) = 0; + ASMJIT_API virtual Error embedConstPool(const Label& label, const ConstPool& pool); //! Embeds an absolute `label` address as data. //! //! The `dataSize` is an optional argument that can be used to specify the size of the address data. If it's zero //! (default) the address size is deduced from the target architecture (either 4 or 8 bytes). - ASMJIT_API virtual Error embedLabel(const Label& label, size_t dataSize = 0) = 0; + ASMJIT_API virtual Error embedLabel(const Label& label, size_t dataSize = 0); //! Embeds a delta (distance) between the `label` and `base` calculating it as `label - base`. This function was //! designed to make it easier to embed lookup tables where each index is a relative distance of two labels. - ASMJIT_API virtual Error embedLabelDelta(const Label& label, const Label& base, size_t dataSize = 0) = 0; + ASMJIT_API virtual Error embedLabelDelta(const Label& label, const Label& base, size_t dataSize = 0); //! \} @@ -725,7 +865,7 @@ public: //! \{ //! Emits a comment stored in `data` with an optional `size` parameter. - ASMJIT_API virtual Error comment(const char* data, size_t size = SIZE_MAX) = 0; + ASMJIT_API virtual Error comment(const char* data, size_t size = SIZE_MAX); //! Emits a formatted comment specified by `fmt` and variable number of arguments. ASMJIT_API Error commentf(const char* fmt, ...); @@ -738,9 +878,13 @@ public: //! \{ //! Called after the emitter was attached to `CodeHolder`. - ASMJIT_API virtual Error onAttach(CodeHolder* ASMJIT_NONNULL(code)) noexcept; + ASMJIT_API virtual Error onAttach(CodeHolder& code) noexcept; + //! Called after the emitter was detached from `CodeHolder`. - ASMJIT_API virtual Error onDetach(CodeHolder* ASMJIT_NONNULL(code)) noexcept; + ASMJIT_API virtual Error onDetach(CodeHolder& code) noexcept; + + //! Called when CodeHolder is reinitialized when the emitter is attached. + ASMJIT_API virtual Error onReinit(CodeHolder& code) noexcept; //! Called when \ref CodeHolder has updated an important setting, which involves the following: //! diff --git a/pe-packer/asmjit/core/emitterutils.cpp b/pe-packer/asmjit/core/emitterutils.cpp index d0a6872..c6fc891 100644 --- a/pe-packer/asmjit/core/emitterutils.cpp +++ b/pe-packer/asmjit/core/emitterutils.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -88,10 +88,12 @@ void logInstructionEmitted( sb.appendChars(' ', logger->indentation(FormatIndentationGroup::kCode)); self->_funcs.formatInstruction(sb, formatFlags, self, self->arch(), BaseInst(instId, options, self->extraReg()), opArray, Globals::kMaxOpCount); - if (Support::test(formatFlags, FormatFlags::kMachineCode)) + if (Support::test(formatFlags, FormatFlags::kMachineCode)) { finishFormattedLine(sb, logger->options(), self->bufferPtr(), size_t(emittedSize), relSize, immSize, self->inlineComment()); - else + } + else { finishFormattedLine(sb, logger->options(), nullptr, SIZE_MAX, 0, 0, self->inlineComment()); + } logger->log(sb); } diff --git a/pe-packer/asmjit/core/emitterutils_p.h b/pe-packer/asmjit/core/emitterutils_p.h index 8b6e1e0..77e93df 100644 --- a/pe-packer/asmjit/core/emitterutils_p.h +++ b/pe-packer/asmjit/core/emitterutils_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_EMITTERUTILS_P_H_INCLUDED @@ -31,7 +31,8 @@ enum kOpIndex : uint32_t { kOp5 = 2 }; -static ASMJIT_FORCE_INLINE uint32_t opCountFromEmitArgs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { +[[nodiscard]] +static ASMJIT_INLINE uint32_t opCountFromEmitArgs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { uint32_t opCount = 0; if (opExt[kOp3].isNone()) { @@ -49,7 +50,7 @@ static ASMJIT_FORCE_INLINE uint32_t opCountFromEmitArgs(const Operand_& o0, cons return opCount; } -static ASMJIT_FORCE_INLINE void opArrayFromEmitArgs(Operand_ dst[Globals::kMaxOpCount], const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { +static ASMJIT_INLINE void opArrayFromEmitArgs(Operand_ dst[Globals::kMaxOpCount], const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { dst[0].copyFrom(o0); dst[1].copyFrom(o1); dst[2].copyFrom(o2); diff --git a/pe-packer/asmjit/core/environment.cpp b/pe-packer/asmjit/core/environment.cpp index 9a694af..f0e1f52 100644 --- a/pe-packer/asmjit/core/environment.cpp +++ b/pe-packer/asmjit/core/environment.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -35,8 +35,9 @@ uint32_t Environment::stackAlignment() const noexcept { return 16u; } - if (isFamilyARM()) + if (isFamilyARM()) { return 8; + } // Bail to 4-byte alignment if we don't know. return 4; diff --git a/pe-packer/asmjit/core/environment.h b/pe-packer/asmjit/core/environment.h index 3095405..9d1992f 100644 --- a/pe-packer/asmjit/core/environment.h +++ b/pe-packer/asmjit/core/environment.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ENVIRONMENT_H_INCLUDED @@ -24,7 +24,7 @@ enum class Vendor : uint8_t { //! Unknown or uninitialized platform vendor. kUnknown = 0, - //! Maximum value of `PlatformVendor`. + //! Maximum value of `Vendor`. kMaxValue = kUnknown, //! Platform vendor detected at compile-time. @@ -116,7 +116,7 @@ enum class Platform : uint8_t { //! Platform ABI (application binary interface). enum class PlatformABI : uint8_t { - //! Unknown or uninitialied environment. + //! Unknown or uninitialized environment. kUnknown = 0, //! Microsoft ABI. kMSVC, @@ -126,6 +126,8 @@ enum class PlatformABI : uint8_t { kAndroid, //! Cygwin ABI. kCygwin, + //! Darwin ABI. + kDarwin, //! Maximum value of `PlatformABI`. kMaxValue, @@ -142,11 +144,26 @@ enum class PlatformABI : uint8_t { kGNU #elif defined(__ANDROID__) kAndroid +#elif defined(__APPLE__) + kDarwin #else kUnknown #endif }; +//! Floating point ABI (ARM). +enum class FloatABI : uint8_t { + kHardFloat = 0, + kSoftFloat, + + kHost = +#if ASMJIT_ARCH_ARM == 32 && defined(__SOFTFP__) + kSoftFloat +#else + kHardFloat +#endif +}; + //! Object format. //! //! \note AsmJit doesn't really use anything except \ref ObjectFormat::kUnknown and \ref ObjectFormat::kJIT at @@ -186,53 +203,56 @@ public: //! \{ //! Architecture. - Arch _arch; + Arch _arch = Arch::kUnknown; //! Sub-architecture type. - SubArch _subArch; + SubArch _subArch = SubArch::kUnknown; //! Vendor type. - Vendor _vendor; + Vendor _vendor = Vendor::kUnknown; //! Platform. - Platform _platform; + Platform _platform = Platform::kUnknown; //! Platform ABI. - PlatformABI _platformABI; + PlatformABI _platformABI = PlatformABI::kUnknown; //! Object format. - ObjectFormat _objectFormat; + ObjectFormat _objectFormat = ObjectFormat::kUnknown; + //! Floating point ABI. + FloatABI _floatABI = FloatABI::kHardFloat; //! Reserved for future use, must be zero. - uint8_t _reserved[2]; + uint8_t _reserved = 0; //! \} //! \name Construction & Destruction //! \{ - ASMJIT_INLINE_NODEBUG Environment() noexcept : - _arch(Arch::kUnknown), - _subArch(SubArch::kUnknown), - _vendor(Vendor::kUnknown), - _platform(Platform::kUnknown), - _platformABI(PlatformABI::kUnknown), - _objectFormat(ObjectFormat::kUnknown), - _reserved { 0, 0 } {} + //! Creates a default initialized environment (all values either unknown or set to safe defaults). + ASMJIT_INLINE_CONSTEXPR Environment() noexcept = default; + //! Creates a copy of `other` instance. + ASMJIT_INLINE_CONSTEXPR Environment(const Environment& other) noexcept = default; - ASMJIT_INLINE_NODEBUG explicit Environment( + //! Creates \ref Environment initialized to `arch`, `subArch`, `vendor`, `platform`, `platformABI`, `objectFormat`, + //! and `floatABI`. + ASMJIT_INLINE_CONSTEXPR explicit Environment( Arch arch, SubArch subArch = SubArch::kUnknown, Vendor vendor = Vendor::kUnknown, Platform platform = Platform::kUnknown, - PlatformABI abi = PlatformABI::kUnknown, - ObjectFormat objectFormat = ObjectFormat::kUnknown) noexcept { - - init(arch, subArch, vendor, platform, abi, objectFormat); - } - - ASMJIT_INLINE_NODEBUG Environment(const Environment& other) noexcept = default; + PlatformABI platformABI = PlatformABI::kUnknown, + ObjectFormat objectFormat = ObjectFormat::kUnknown, + FloatABI floatABI = FloatABI::kHardFloat) noexcept + : _arch(arch), + _subArch(subArch), + _vendor(vendor), + _platform(platform), + _platformABI(platformABI), + _objectFormat(objectFormat), + _floatABI(floatABI) {} //! Returns the host environment constructed from preprocessor macros defined by the compiler. //! //! The returned environment should precisely match the target host architecture, sub-architecture, platform, //! and ABI. - static ASMJIT_INLINE_NODEBUG Environment host() noexcept { - return Environment(Arch::kHost, SubArch::kHost, Vendor::kHost, Platform::kHost, PlatformABI::kHost, ObjectFormat::kUnknown); + static ASMJIT_INLINE_CONSTEXPR Environment host() noexcept { + return Environment(Arch::kHost, SubArch::kHost, Vendor::kHost, Platform::kHost, PlatformABI::kHost, ObjectFormat::kUnknown, FloatABI::kHost); } //! \} @@ -242,7 +262,10 @@ public: ASMJIT_INLINE_NODEBUG Environment& operator=(const Environment& other) noexcept = default; - ASMJIT_INLINE_NODEBUG bool operator==(const Environment& other) const noexcept { return equals(other); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool operator==(const Environment& other) const noexcept { return equals(other); } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool operator!=(const Environment& other) const noexcept { return !equals(other); } //! \} @@ -253,6 +276,7 @@ public: //! Tests whether the environment is not set up. //! //! Returns true if all members are zero, and thus unknown. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool empty() const noexcept { // Unfortunately compilers won't optimize fields are checked one by one... return _packed() == 0; @@ -260,10 +284,12 @@ public: //! Tests whether the environment is initialized, which means it must have //! a valid architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInitialized() const noexcept { return _arch != Arch::kUnknown; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t _packed() const noexcept { uint64_t x; memcpy(&x, this, 8); @@ -271,41 +297,50 @@ public: } //! Resets all members of the environment to zero / unknown. - ASMJIT_INLINE_NODEBUG void reset() noexcept { - _arch = Arch::kUnknown; - _subArch = SubArch::kUnknown; - _vendor = Vendor::kUnknown; - _platform = Platform::kUnknown; - _platformABI = PlatformABI::kUnknown; - _objectFormat = ObjectFormat::kUnknown; - _reserved[0] = 0; - _reserved[1] = 0; - } + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = Environment{}; } - ASMJIT_INLINE_NODEBUG bool equals(const Environment& other) const noexcept { - return _packed() == other._packed(); - } + //! Tests whether this environment is equal to `other`. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool equals(const Environment& other) const noexcept { return _packed() == other._packed(); } //! Returns the architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return _arch; } + //! Returns the sub-architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG SubArch subArch() const noexcept { return _subArch; } + //! Returns vendor. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Vendor vendor() const noexcept { return _vendor; } + //! Returns target's platform or operating system. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Platform platform() const noexcept { return _platform; } + //! Returns target's ABI. + [[nodiscard]] ASMJIT_INLINE_NODEBUG PlatformABI platformABI() const noexcept { return _platformABI; } + //! Returns target's object format. + [[nodiscard]] ASMJIT_INLINE_NODEBUG ObjectFormat objectFormat() const noexcept { return _objectFormat; } + //! Returns floating point ABI. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG FloatABI floatABI() const noexcept { return _floatABI; } + + //! Initializes \ref Environment to `arch`, `subArch`, `vendor`, `platform`, `platformABI`, `objectFormat`, + //! and `floatABI`. inline void init( Arch arch, SubArch subArch = SubArch::kUnknown, Vendor vendor = Vendor::kUnknown, Platform platform = Platform::kUnknown, PlatformABI platformABI = PlatformABI::kUnknown, - ObjectFormat objectFormat = ObjectFormat::kUnknown) noexcept { + ObjectFormat objectFormat = ObjectFormat::kUnknown, + FloatABI floatABI = FloatABI::kHardFloat) noexcept { _arch = arch; _subArch = subArch; @@ -313,56 +348,104 @@ public: _platform = platform; _platformABI = platformABI; _objectFormat = objectFormat; - _reserved[0] = 0; - _reserved[1] = 0; + _floatABI = floatABI; + _reserved = 0; } + //! Tests whether this environment describes a 32-bit X86. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchX86() const noexcept { return _arch == Arch::kX86; } + + //! Tests whether this environment describes a 64-bit X86. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchX64() const noexcept { return _arch == Arch::kX64; } + + //! Tests whether this environment describes a 32-bit ARM. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchARM() const noexcept { return isArchARM(_arch); } + + //! Tests whether this environment describes a 32-bit ARM in THUMB mode. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchThumb() const noexcept { return isArchThumb(_arch); } + + //! Tests whether this environment describes a 64-bit X86. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchAArch64() const noexcept { return isArchAArch64(_arch); } + + //! Tests whether this environment describes a 32-bit MIPS. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchMIPS32() const noexcept { return isArchMIPS32(_arch); } + + //! Tests whether this environment describes a 64-bit MIPS. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchMIPS64() const noexcept { return isArchMIPS64(_arch); } + + //! Tests whether this environment describes a 32-bit RISC-V. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchRISCV32() const noexcept { return _arch == Arch::kRISCV32; } + + //! Tests whether this environment describes a 64-bit RISC-V. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isArchRISCV64() const noexcept { return _arch == Arch::kRISCV64; } //! Tests whether the architecture is 32-bit. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool is32Bit() const noexcept { return is32Bit(_arch); } + //! Tests whether the architecture is 64-bit. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool is64Bit() const noexcept { return is64Bit(_arch); } //! Tests whether the architecture is little endian. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isLittleEndian() const noexcept { return isLittleEndian(_arch); } + //! Tests whether the architecture is big endian. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isBigEndian() const noexcept { return isBigEndian(_arch); } //! Tests whether this architecture is of X86 family. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyX86() const noexcept { return isFamilyX86(_arch); } + //! Tests whether this architecture family is ARM, THUMB, or AArch64. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyARM() const noexcept { return isFamilyARM(_arch); } + //! Tests whether this architecture family is AArch32 (ARM or THUMB). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyAArch32() const noexcept { return isFamilyAArch32(_arch); } + //! Tests whether this architecture family is AArch64. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyAArch64() const noexcept { return isFamilyAArch64(_arch); } + //! Tests whether this architecture family is MISP or MIPS64. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyMIPS() const noexcept { return isFamilyMIPS(_arch); } + //! Tests whether this architecture family is RISC-V (both 32-bit and 64-bit). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isFamilyRISCV() const noexcept { return isFamilyRISCV(_arch); } //! Tests whether the environment platform is Windows. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformWindows() const noexcept { return _platform == Platform::kWindows; } //! Tests whether the environment platform is Linux. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformLinux() const noexcept { return _platform == Platform::kLinux; } //! Tests whether the environment platform is Hurd. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformHurd() const noexcept { return _platform == Platform::kHurd; } //! Tests whether the environment platform is Haiku. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformHaiku() const noexcept { return _platform == Platform::kHaiku; } //! Tests whether the environment platform is any BSD. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformBSD() const noexcept { return _platform == Platform::kFreeBSD || _platform == Platform::kOpenBSD || @@ -371,6 +454,7 @@ public: } //! Tests whether the environment platform is any Apple platform (OSX, iOS, TVOS, WatchOS). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isPlatformApple() const noexcept { return _platform == Platform::kOSX || _platform == Platform::kIOS || @@ -379,14 +463,23 @@ public: } //! Tests whether the ABI is MSVC. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMSVC() const noexcept { return _platformABI == PlatformABI::kMSVC; } + //! Tests whether the ABI is GNU. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isGNU() const noexcept { return _platformABI == PlatformABI::kGNU; } + //! Tests whether the ABI is GNU. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isDarwin() const noexcept { return _platformABI == PlatformABI::kDarwin; } + //! Returns a calculated stack alignment for this environment. + [[nodiscard]] ASMJIT_API uint32_t stackAlignment() const noexcept; //! Returns a native register size of this architecture. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t registerSize() const noexcept { return registerSizeFromArch(_arch); } //! Sets the architecture to `arch`. @@ -402,95 +495,116 @@ public: //! Sets the object format to `objectFormat`. ASMJIT_INLINE_NODEBUG void setObjectFormat(ObjectFormat objectFormat) noexcept { _objectFormat = objectFormat; } + //! Sets floating point ABI to `floatABI`. + ASMJIT_INLINE_NODEBUG void setFloatABI(FloatABI floatABI) noexcept { _floatABI = floatABI; } + //! \} //! \name Static Utilities //! \{ + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isDefinedArch(Arch arch) noexcept { return uint32_t(arch) <= uint32_t(Arch::kMaxValue); } + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isValidArch(Arch arch) noexcept { return arch != Arch::kUnknown && uint32_t(arch) <= uint32_t(Arch::kMaxValue); } //! Tests whether the given architecture `arch` is 32-bit. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool is32Bit(Arch arch) noexcept { return (uint32_t(arch) & uint32_t(Arch::k32BitMask)) == uint32_t(Arch::k32BitMask); } //! Tests whether the given architecture `arch` is 64-bit. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool is64Bit(Arch arch) noexcept { return (uint32_t(arch) & uint32_t(Arch::k32BitMask)) == 0; } //! Tests whether the given architecture `arch` is little endian. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isLittleEndian(Arch arch) noexcept { return uint32_t(arch) < uint32_t(Arch::kBigEndian); } //! Tests whether the given architecture `arch` is big endian. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isBigEndian(Arch arch) noexcept { return uint32_t(arch) >= uint32_t(Arch::kBigEndian); } //! Tests whether the given architecture is Thumb or Thumb_BE. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isArchThumb(Arch arch) noexcept { return arch == Arch::kThumb || arch == Arch::kThumb_BE; } //! Tests whether the given architecture is ARM or ARM_BE. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isArchARM(Arch arch) noexcept { return arch == Arch::kARM || arch == Arch::kARM_BE; } //! Tests whether the given architecture is AArch64 or AArch64_BE. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isArchAArch64(Arch arch) noexcept { return arch == Arch::kAArch64 || arch == Arch::kAArch64_BE; } //! Tests whether the given architecture is MIPS32_LE or MIPS32_BE. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isArchMIPS32(Arch arch) noexcept { return arch == Arch::kMIPS32_LE || arch == Arch::kMIPS32_BE; } //! Tests whether the given architecture is MIPS64_LE or MIPS64_BE. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isArchMIPS64(Arch arch) noexcept { return arch == Arch::kMIPS64_LE || arch == Arch::kMIPS64_BE; } //! Tests whether the given architecture family is X86 or X64. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isFamilyX86(Arch arch) noexcept { return arch == Arch::kX86 || arch == Arch::kX64; } - //! Tests whether the given architecture family is ARM, THUMB, or AArch64. - static ASMJIT_INLINE_NODEBUG bool isFamilyARM(Arch arch) noexcept { - return isArchARM(arch) || isArchAArch64(arch) || isArchThumb(arch); - } - //! Tests whether the given architecture family is AArch32 (ARM or THUMB). + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isFamilyAArch32(Arch arch) noexcept { return isArchARM(arch) || isArchThumb(arch); } //! Tests whether the given architecture family is AArch64. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isFamilyAArch64(Arch arch) noexcept { return isArchAArch64(arch); } - //! Tests whether the given architecture family is MISP or MIPS64. + //! Tests whether the given architecture family is ARM, THUMB, or AArch64. + [[nodiscard]] + static ASMJIT_INLINE_NODEBUG bool isFamilyARM(Arch arch) noexcept { + return isFamilyAArch32(arch) || isFamilyAArch64(arch); + } + + //! Tests whether the given architecture family is MIPS or MIPS64. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isFamilyMIPS(Arch arch) noexcept { return isArchMIPS32(arch) || isArchMIPS64(arch); } //! Tests whether the given architecture family is RISC-V (both 32-bit and 64-bit). + [[nodiscard]] static ASMJIT_INLINE_NODEBUG bool isFamilyRISCV(Arch arch) noexcept { return arch == Arch::kRISCV32 || arch == Arch::kRISCV64; } //! Returns a native general purpose register size from the given architecture. + [[nodiscard]] static ASMJIT_INLINE_NODEBUG uint32_t registerSizeFromArch(Arch arch) noexcept { return is32Bit(arch) ? 4u : 8u; } diff --git a/pe-packer/asmjit/core/errorhandler.cpp b/pe-packer/asmjit/core/errorhandler.cpp index ce2ad01..d0efa9d 100644 --- a/pe-packer/asmjit/core/errorhandler.cpp +++ b/pe-packer/asmjit/core/errorhandler.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" diff --git a/pe-packer/asmjit/core/errorhandler.h b/pe-packer/asmjit/core/errorhandler.h index d2c37b3..9b5c8c2 100644 --- a/pe-packer/asmjit/core/errorhandler.h +++ b/pe-packer/asmjit/core/errorhandler.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_ERRORHANDLER_H_INCLUDED @@ -215,7 +215,7 @@ public: //! calling `handleError()` so `longjmp()` can be used without any issues to cancel the code generation if an //! error occurred. There is no difference between exceptions and `longjmp()` from AsmJit's perspective, however, //! never jump outside of `CodeHolder` and `BaseEmitter` scope as you would leak memory. - ASMJIT_API virtual void handleError(Error err, const char* message, BaseEmitter* origin) = 0; + ASMJIT_API virtual void handleError(Error err, const char* message, BaseEmitter* origin); //! \} }; diff --git a/pe-packer/asmjit/core/fixup.h b/pe-packer/asmjit/core/fixup.h new file mode 100644 index 0000000..7abb529 --- /dev/null +++ b/pe-packer/asmjit/core/fixup.h @@ -0,0 +1,282 @@ +// This file is part of AsmJit project +// +// See or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FIXUP_H_INCLUDED +#define ASMJIT_CORE_FIXUP_H_INCLUDED + +#include "../core/globals.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! Offset format type, used by \ref OffsetFormat. +enum class OffsetType : uint8_t { + // Common Offset Formats + // --------------------- + + //! A value having `_immBitCount` bits and shifted by `_immBitShift`. + //! + //! This offset type is sufficient for many targets that store offset as a continuous set bits within an + //! instruction word / sequence of bytes. + kSignedOffset, + + //! An unsigned value having `_immBitCount` bits and shifted by `_immBitShift`. + kUnsignedOffset, + + // AArch64 Specific Offset Formats + // ------------------------------- + + //! AArch64 ADR format of `[.|immlo:2|.....|immhi:19|.....]`. + kAArch64_ADR, + + //! AArch64 ADRP format of `[.|immlo:2|.....|immhi:19|.....]` (4kB pages). + kAArch64_ADRP, + + // AArch32 Specific Offset Formats (T16 & T32) + // ------------------------------------------- + + //! AArch32 THUMBv2 immediate encoding of 'ADR' instruction (12-bit payload and sign bit): + //! + //! `|.....|imm:1|..N.N|......|imm:3|....|imm:8|` + //! + //! Where `N` is one if the offset is negative. The immediate is encoded as absolute value of the offset if negative. + kThumb32_ADR, + + //! AArch32 THUMBv2 immediate encoding of 'BLX' instruction (23-bit immediate payload, multiplied by 4): + //! + //! `|.....|imm[22]|imm[19:10]|..|ja|1|jb|imm[9:0]|0` + //! + //! Where: + //! + //! - `ja` is calculated as imm[22] ^ imm[21] ^ 1. + //! - `jb` is calculated as imm[22] ^ imm[20] ^ 1. + kThumb32_BLX, + + //! AArch32 THUMBv2 immediate encoding of 'B' instruction without `` (24-bit immediate payload, multiplied by 2): + //! + //! `|.....|imm[23]|imm[20:11]|..|ja|1|jb|imm[10:0]` + //! + //! Where: + //! + //! - `ja` is calculated as imm[23] ^ imm[22] ^ 1. + //! - `jb` is calculated as imm[23] ^ imm[21] ^ 1. + kThumb32_B, + + //! AArch32 THUMBv2 immediate encoding of 'B' instruction with `` (20-bit immediate payload, multiplied by 2). + //! + //! `|.....|imm[19]|....|imm[16:11]|..|ja|1|jb|imm[10:0]` + //! + //! Where: + //! + //! - `ja` is calculated as imm[19] ^ imm[18] ^ 1. + //! - `jb` is calculated as imm[19] ^ imm[17] ^ 1. + kThumb32_BCond, + + // AArch32 Specific Offset Formats (A32) + // ------------------------------------- + + //! AArch32 ADR instruction, which uses a standard 12-bit immediate encoding that is used by other ARM instructions. + kAArch32_ADR, + + //! AArch32 signed offset that is similar to `kSignedOffset`, however it uses absolute value of the offset and its + //! sign is encoded in 23rd bit of the opcode. + //! + //! `|........|U.......|........|........|` + //! + kAArch32_U23_SignedOffset, + + //! AArch32 offset format that encodes 8-bit offset as: + //! + //! `|........|U.......|....|imm[7:4]|....|imm[3:0]|` + //! + //! in a 32-bit word, where U is a sign of the displacement and the displacement itself is encoded as its absolute + //! value. + kAArch32_U23_0To3At0_4To7At8, + + //! AArch32 offset format that encodes a signed 25-bit offset as: + //! + //! `|.......|imm[0]|imm[24:1]|` + //! + //! in a 32-bit word. + kAArch32_1To24At0_0At24, + + //! Maximum value of `OffsetFormatType`. + kMaxValue = kAArch32_1To24At0_0At24 +}; + +//! Provides information about formatting offsets, absolute addresses, or their parts. Offset format is used by both +//! \ref RelocEntry and \ref Fixup. The illustration below describes the relation of region size and offset size. +//! Region size is the size of the whole unit whereas offset size is the size of the unit that will be patched. +//! +//! ``` +//! +-> Code buffer | The subject of the relocation (region) | +//! | | (Word-Offset) (Word-Size) | +//! |xxxxxxxxxxxxxxx|................|*PATCHED*|................|xxxxxxxxxxxx-> +//! | | +//! [Word Offset points here]----+ +--- [WordOffset + WordSize] +//! ``` +//! +//! Once the offset word has been located it can be patched like this: +//! +//! ``` +//! |ImmDiscardLSB (discard LSB bits). +//! |.. +//! [0000000000000iiiiiiiiiiiiiiiiiDD] - Offset value (32-bit) +//! [000000000000000iiiiiiiiiiiiiiiii] - Offset value after discard LSB. +//! [00000000000iiiiiiiiiiiiiiiii0000] - Offset value shifted by ImmBitShift. +//! [xxxxxxxxxxxiiiiiiiiiiiiiiiiixxxx] - Patched word (32-bit) +//! |...............| +//! (ImmBitCount) +- ImmBitShift +//! ``` +struct OffsetFormat { + //! \name Members + //! \{ + + //! Type of the offset. + OffsetType _type; + //! Encoding flags. + uint8_t _flags; + //! Size of the region (in bytes) containing the offset value, if the offset value is part of an instruction, + //! otherwise it would be the same as `_valueSize`. + uint8_t _regionSize; + //! Size of the offset value, in bytes (1, 2, 4, or 8). + uint8_t _valueSize; + //! Offset of the offset value, in bytes, relative to the start of the region or data. Value offset would be + //! zero if both region size and value size are equal. + uint8_t _valueOffset; + //! Size of the offset immediate value in bits. + uint8_t _immBitCount; + //! Shift of the offset immediate value in bits in the target word. + uint8_t _immBitShift; + //! Number of least significant bits to discard before writing the immediate to the destination. All discarded + //! bits must be zero otherwise the value is invalid. + uint8_t _immDiscardLsb; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the type of the offset. + ASMJIT_INLINE_NODEBUG OffsetType type() const noexcept { return _type; } + + //! Returns whether the offset is encoded as an absolute value of the offset with additional field(s) that represent + //! the sign (AArch32 U/N fields in the opcode). + //! + //! If true, the offset itself is always positive and a separate U/N field is used to indicate the sign of the offset + //! (usually `U==1` means ADD, but sometimes `N==1` means negative offset, which implies SUB). + ASMJIT_INLINE_NODEBUG bool hasSignBit() const noexcept { + return _type == OffsetType::kThumb32_ADR || + _type == OffsetType::kAArch32_ADR || + _type == OffsetType::kAArch32_U23_SignedOffset || + _type == OffsetType::kAArch32_U23_0To3At0_4To7At8; + } + + //! Returns flags. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t flags() const noexcept { return _flags; } + + //! Returns the size of the region/instruction where the offset is encoded. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t regionSize() const noexcept { return _regionSize; } + + //! Returns the offset of the word relative to the start of the region where the offset is. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t valueOffset() const noexcept { return _valueOffset; } + + //! Returns the size of the data-type (word) that contains the offset, in bytes. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t valueSize() const noexcept { return _valueSize; } + + //! Returns the count of bits of the offset value in the data it's stored in. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immBitCount() const noexcept { return _immBitCount; } + + //! Returns the bit-shift of the offset value in the data it's stored in. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immBitShift() const noexcept { return _immBitShift; } + + //! Returns the number of least significant bits of the offset value, that must be zero and that are not part of + //! the encoded data. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t immDiscardLsb() const noexcept { return _immDiscardLsb; } + + //! Resets this offset format to a simple data value of `dataSize` bytes. + //! + //! The region will be the same size as data and immediate bits would correspond to `dataSize * 8`. There will be + //! no immediate bit shift or discarded bits. + inline void resetToSimpleValue(OffsetType type, size_t valueSize) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(valueSize * 8u); + _immBitShift = uint8_t(0); + _immDiscardLsb = uint8_t(0); + } + + inline void resetToImmValue(OffsetType type, size_t valueSize, uint32_t immBitShift, uint32_t immBitCount, uint32_t immDiscardLsb) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + ASMJIT_ASSERT(immBitShift < valueSize * 8u); + ASMJIT_ASSERT(immBitCount <= 64u); + ASMJIT_ASSERT(immDiscardLsb <= 64u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(immBitCount); + _immBitShift = uint8_t(immBitShift); + _immDiscardLsb = uint8_t(immDiscardLsb); + } + + inline void setRegion(size_t regionSize, size_t valueOffset) noexcept { + _regionSize = uint8_t(regionSize); + _valueOffset = uint8_t(valueOffset); + } + + inline void setLeadingAndTrailingSize(size_t leadingSize, size_t trailingSize) noexcept { + _regionSize = uint8_t(leadingSize + trailingSize + _valueSize); + _valueOffset = uint8_t(leadingSize); + } + + //! \} +}; + +//! Data structure used to mark where a fixup in code or data is necessary. +//! +//! Fixups are generally resolved during machine code generation. For example if a branch instruction is used to +//! jump to a label, which hasn't been bound yet, a fixup is created. However, when such label is bound, the fixup +//! is processed and removed from a list of fixups. +struct Fixup { + //! Next fixup in a single-linked list. + Fixup* next; + //! Section where the fixup comes from. + uint32_t sectionId; + //! Label id, relocation id, or \ref Globals::kInvalidId. + //! + //! \note Fixup that is used with a LabelEntry always uses relocation id here, however, when a fixup is turned + //! into unresolved and generally detached from LabelEntry, this field becomes a label identifier as unresolved + //! fixups won't reference a relocation. This is just a space optimization. + uint32_t labelOrRelocId; + //! Label offset relative to the start of the section where the unresolved link comes from. + size_t offset; + //! Inlined rel8/rel32. + intptr_t rel; + //! Offset format information. + OffsetFormat format; +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FIXUP_H_INCLUDED diff --git a/pe-packer/asmjit/core/formatter.cpp b/pe-packer/asmjit/core/formatter.cpp index 3fd5137..10660fa 100644 --- a/pe-packer/asmjit/core/formatter.cpp +++ b/pe-packer/asmjit/core/formatter.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -32,6 +32,26 @@ class VirtReg; namespace Formatter { +Error formatVirtRegName(String& sb, const VirtReg* vReg) noexcept { + if (vReg->nameSize()) { + return sb.append(vReg->name(), vReg->nameSize()); + } + else { + return sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(vReg->id()))); + } +} + +Error formatVirtRegNameWithPrefix(String& sb, const char* prefix, size_t prefixSize, const VirtReg* vReg) noexcept { + ASMJIT_PROPAGATE(sb.append(prefix, prefixSize)); + + if (vReg->nameSize()) { + return sb.append(vReg->name(), vReg->nameSize()); + } + else { + return sb.appendFormat("%%%u", unsigned(Operand::virtIdToIndex(vReg->id()))); + } +} + static const char wordNameTable[][8] = { "db", "dw", @@ -51,13 +71,15 @@ static const char wordNameTable[][8] = { Error formatTypeId(String& sb, TypeId typeId) noexcept { - if (typeId == TypeId::kVoid) + if (typeId == TypeId::kVoid) { return sb.append("void"); + } - if (!TypeUtils::isValid(typeId)) + if (!TypeUtils::isValid(typeId)) { return sb.append("unknown"); + } - const char* typeName = "unknown"; + const char* typeName = nullptr; uint32_t typeSize = TypeUtils::sizeOf(typeId); TypeId scalarType = TypeUtils::scalarOf(typeId); @@ -103,13 +125,15 @@ Error formatFeature( uint32_t featureId) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::FormatterInternal::formatFeature(sb, featureId); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyARM(arch)) + if (Environment::isFamilyARM(arch)) { return arm::FormatterInternal::formatFeature(sb, featureId); + } #endif return kErrorInvalidArch; @@ -124,28 +148,31 @@ Error formatLabel( DebugUtils::unused(formatFlags); if (emitter && emitter->code()) { - const LabelEntry* le = emitter->code()->labelEntry(labelId); - if (ASMJIT_UNLIKELY(!le)) + CodeHolder* code = emitter->code(); + if (ASMJIT_UNLIKELY(!code->isLabelValid(labelId))) { return sb.appendFormat("", labelId); + } - if (le->hasName()) { - if (le->hasParent()) { - uint32_t parentId = le->parentId(); - const LabelEntry* pe = emitter->code()->labelEntry(parentId); + const LabelEntry& le = code->labelEntry(labelId); + if (le.hasName()) { + if (le.hasParent()) { + uint32_t parentId = le.parentId(); + const LabelEntry& pe = code->labelEntry(parentId); - if (ASMJIT_UNLIKELY(!pe)) - ASMJIT_PROPAGATE(sb.appendFormat("", labelId)); - else if (ASMJIT_UNLIKELY(!pe->hasName())) + if (pe.hasName()) { + ASMJIT_PROPAGATE(sb.append(pe.name())); + } + else { ASMJIT_PROPAGATE(sb.appendFormat("L%u", parentId)); - else - ASMJIT_PROPAGATE(sb.append(pe->name())); + } ASMJIT_PROPAGATE(sb.append('.')); } - if (le->type() == LabelType::kAnonymous) + if (le.labelType() == LabelType::kAnonymous) { ASMJIT_PROPAGATE(sb.appendFormat("L%u@", labelId)); - return sb.append(le->name()); + } + return sb.append(le.name()); } } @@ -161,13 +188,15 @@ Error formatRegister( uint32_t regId) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::FormatterInternal::formatRegister(sb, formatFlags, emitter, arch, regType, regId); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::FormatterInternal::formatRegister(sb, formatFlags, emitter, arch, regType, regId); + if (Environment::isFamilyARM(arch)) { + return arm::FormatterInternal::formatRegister(sb, formatFlags, emitter, arch, regType, regId); + } #endif return kErrorInvalidArch; @@ -181,13 +210,15 @@ Error formatOperand( const Operand_& op) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::FormatterInternal::formatOperand(sb, formatFlags, emitter, arch, op); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::FormatterInternal::formatOperand(sb, formatFlags, emitter, arch, op); + if (Environment::isFamilyARM(arch)) { + return arm::FormatterInternal::formatOperand(sb, formatFlags, emitter, arch, op); + } #endif return kErrorInvalidArch; @@ -201,12 +232,14 @@ ASMJIT_API Error formatDataType( { DebugUtils::unused(formatFlags); - if (ASMJIT_UNLIKELY(uint32_t(arch) > uint32_t(Arch::kMaxValue))) + if (ASMJIT_UNLIKELY(uint32_t(arch) > uint32_t(Arch::kMaxValue))) { return DebugUtils::errored(kErrorInvalidArch); + } uint32_t typeSize = TypeUtils::sizeOf(typeId); - if (typeSize == 0 || typeSize > 8) + if (typeSize == 0 || typeSize > 8) { return DebugUtils::errored(kErrorInvalidState); + } uint32_t typeSizeLog2 = Support::ctz(typeSize); return sb.append(wordNameTable[size_t(ArchTraits::byArch(arch).typeNameIdByIndex(typeSizeLog2))]); @@ -220,14 +253,15 @@ static Error formatDataHelper(String& sb, const char* typeName, uint32_t typeSiz for (size_t i = 0; i < itemCount; i++) { uint64_t v = 0; - if (i != 0) + if (i != 0) { ASMJIT_PROPAGATE(sb.append(", ", 2)); + } switch (typeSize) { case 1: v = data[0]; break; - case 2: v = Support::readU16u(data); break; - case 4: v = Support::readU32u(data); break; - case 8: v = Support::readU64u(data); break; + case 2: v = Support::loadu_u16(data); break; + case 4: v = Support::loadu_u32(data); break; + case 8: v = Support::loadu_u64(data); break; } ASMJIT_PROPAGATE(sb.appendUInt(v, 16, typeSize * 2, StringFormatFlags::kAlternate)); @@ -241,16 +275,18 @@ Error formatData( String& sb, FormatFlags formatFlags, Arch arch, - TypeId typeId, const void* data, size_t itemCount, size_t repeatCount) noexcept -{ + TypeId typeId, const void* data, size_t itemCount, size_t repeatCount +) noexcept { DebugUtils::unused(formatFlags); - if (ASMJIT_UNLIKELY(!Environment::isDefinedArch(arch))) + if (ASMJIT_UNLIKELY(!Environment::isDefinedArch(arch))) { return DebugUtils::errored(kErrorInvalidArch); + } uint32_t typeSize = TypeUtils::sizeOf(typeId); - if (typeSize == 0) + if (typeSize == 0) { return DebugUtils::errored(kErrorInvalidState); + } if (!Support::isPowerOf2(typeSize)) { itemCount *= typeSize; @@ -265,8 +301,9 @@ Error formatData( uint32_t typeSizeLog2 = Support::ctz(typeSize); const char* wordName = wordNameTable[size_t(ArchTraits::byArch(arch).typeNameIdByIndex(typeSizeLog2))]; - if (repeatCount > 1) + if (repeatCount > 1) { ASMJIT_PROPAGATE(sb.appendFormat(".repeat %zu ", repeatCount)); + } return formatDataHelper(sb, wordName, typeSize, static_cast(data), itemCount); } @@ -279,13 +316,15 @@ Error formatInstruction( const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::FormatterInternal::formatInstruction(sb, formatFlags, emitter, arch, inst, operands, opCount); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyARM(arch)) + if (Environment::isFamilyAArch64(arch)) { return a64::FormatterInternal::formatInstruction(sb, formatFlags, emitter, arch, inst, operands, opCount); + } #endif return kErrorInvalidArch; @@ -301,8 +340,9 @@ static Error formatFuncValue(String& sb, FormatFlags formatFlags, const BaseEmit if (value.isAssigned()) { ASMJIT_PROPAGATE(sb.append('@')); - if (value.isIndirect()) + if (value.isIndirect()) { ASMJIT_PROPAGATE(sb.append('[')); + } // NOTE: It should be either reg or stack, but never both. We // use two IFs on purpose so if the FuncValue is both it would @@ -315,8 +355,9 @@ static Error formatFuncValue(String& sb, FormatFlags formatFlags, const BaseEmit ASMJIT_PROPAGATE(sb.appendFormat("[%d]", int(value.stackOffset()))); } - if (value.isIndirect()) + if (value.isIndirect()) { ASMJIT_PROPAGATE(sb.append(']')); + } } return kErrorOk; @@ -330,19 +371,23 @@ static Error formatFuncValuePack( const RegOnly* vRegs) noexcept { size_t count = pack.count(); - if (!count) + if (!count) { return sb.append("void"); + } - if (count > 1) - sb.append('['); + if (count > 1) { + ASMJIT_PROPAGATE(sb.append('[')); + } for (uint32_t valueIndex = 0; valueIndex < count; valueIndex++) { const FuncValue& value = pack[valueIndex]; - if (!value) + if (!value) { break; + } - if (valueIndex) + if (valueIndex) { ASMJIT_PROPAGATE(sb.append(", ")); + } ASMJIT_PROPAGATE(formatFuncValue(sb, formatFlags, cc, value)); @@ -350,15 +395,24 @@ static Error formatFuncValuePack( const VirtReg* virtReg = nullptr; static const char nullReg[] = ""; - if (vRegs[valueIndex].isReg() && cc->isVirtIdValid(vRegs[valueIndex].id())) + if (vRegs[valueIndex].isReg() && cc->isVirtIdValid(vRegs[valueIndex].id())) { virtReg = cc->virtRegById(vRegs[valueIndex].id()); + } - ASMJIT_PROPAGATE(sb.appendFormat(" %s", virtReg ? virtReg->name() : nullReg)); + ASMJIT_PROPAGATE(sb.append(' ')); + + if (virtReg) { + ASMJIT_PROPAGATE(Formatter::formatVirtRegName(sb, virtReg)); + } + else { + ASMJIT_PROPAGATE(sb.append(nullReg, sizeof(nullReg) - 1)); + } } } - if (count > 1) - sb.append(']'); + if (count > 1) { + ASMJIT_PROPAGATE(sb.append(']')); + } return kErrorOk; } @@ -380,13 +434,14 @@ static Error formatFuncArgs( const FuncNode::ArgPack* argPacks) noexcept { uint32_t argCount = fd.argCount(); - if (!argCount) + if (!argCount) { return sb.append("void"); + } for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) { - if (argIndex) + if (argIndex) { ASMJIT_PROPAGATE(sb.append(", ")); - + } ASMJIT_PROPAGATE(formatFuncValuePack(sb, formatFlags, cc, fd.argPack(argIndex), argPacks[argIndex]._data)); } @@ -400,8 +455,9 @@ Error formatNode( const BaseBuilder* builder, const BaseNode* node) noexcept { - if (node->hasPosition() && formatOptions.hasFlag(FormatFlags::kPositions)) + if (node->hasPosition() && formatOptions.hasFlag(FormatFlags::kPositions)) { ASMJIT_PROPAGATE(sb.appendFormat("<%05u> ", node->position())); + } size_t startLineIndex = sb.size(); @@ -417,8 +473,8 @@ Error formatNode( case NodeType::kSection: { const SectionNode* sectionNode = node->as(); - if (builder->_code->isSectionValid(sectionNode->id())) { - const Section* section = builder->_code->sectionById(sectionNode->id()); + if (builder->_code->isSectionValid(sectionNode->sectionId())) { + const Section* section = builder->_code->sectionById(sectionNode->sectionId()); ASMJIT_PROPAGATE(sb.appendFormat(".section %s", section->name())); } break; @@ -514,7 +570,7 @@ Error formatNode( ASMJIT_PROPAGATE(sb.append("[FuncRet]")); for (uint32_t i = 0; i < 2; i++) { - const Operand_& op = retNode->_opArray[i]; + const Operand_& op = retNode->op(i); if (!op.isNone()) { ASMJIT_PROPAGATE(sb.append(i == 0 ? " " : ", ")); ASMJIT_PROPAGATE(formatOperand(sb, formatOptions.flags(), builder, builder->arch(), op)); @@ -542,8 +598,9 @@ Error formatNode( size_t requiredPadding = paddingFromOptions(formatOptions, FormatPaddingGroup::kRegularLine); size_t currentPadding = sb.size() - startLineIndex; - if (currentPadding < requiredPadding) + if (currentPadding < requiredPadding) { ASMJIT_PROPAGATE(sb.appendChars(' ', requiredPadding - currentPadding)); + } ASMJIT_PROPAGATE(sb.append("; ")); ASMJIT_PROPAGATE(sb.append(node->inlineComment())); diff --git a/pe-packer/asmjit/core/formatter.h b/pe-packer/asmjit/core/formatter.h index 392e478..840d0c1 100644 --- a/pe-packer/asmjit/core/formatter.h +++ b/pe-packer/asmjit/core/formatter.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FORMATTER_H_INCLUDED @@ -26,20 +26,25 @@ enum class FormatFlags : uint32_t { //! No formatting flags. kNone = 0u, - //! Show also binary form of each logged instruction (Assembler). + //! Show also a binary representation of each logged instruction (Assembler). kMachineCode = 0x00000001u, + //! Show aliases of some instructions that have them. + //! + //! This option is now mostly for x86/x64 to show aliases of instructions such as `cmov`, `j`, `set`, + //! etc... + kShowAliases = 0x00000008u, //! Show a text explanation of some immediate values. - kExplainImms = 0x00000002u, + kExplainImms = 0x00000010u, //! Use hexadecimal notation of immediate values. - kHexImms = 0x00000004u, + kHexImms = 0x00000020u, //! Use hexadecimal notation of addresses and offsets in addresses. - kHexOffsets = 0x00000008u, - //! Show casts between virtual register types (Compiler output). - kRegCasts = 0x00000010u, - //! Show positions associated with nodes (Compiler output). - kPositions = 0x00000020u, - //! Always format a register type (Compiler output). - kRegType = 0x00000040u + kHexOffsets = 0x00000040u, + //! Show casts between virtual register types (Compiler). + kRegCasts = 0x00000100u, + //! Show positions associated with nodes (Compiler). + kPositions = 0x00000200u, + //! Always format a register type (Compiler). + kRegType = 0x00000400u }; ASMJIT_DEFINE_ENUM_FLAGS(FormatFlags) @@ -103,28 +108,39 @@ public: //! \{ //! Returns format flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FormatFlags flags() const noexcept { return _flags; } + //! Tests whether the given `flag` is set in format flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(FormatFlags flag) const noexcept { return Support::test(_flags, flag); } //! Resets all format flags to `flags`. ASMJIT_INLINE_NODEBUG void setFlags(FormatFlags flags) noexcept { _flags = flags; } + //! Adds `flags` to format flags. ASMJIT_INLINE_NODEBUG void addFlags(FormatFlags flags) noexcept { _flags |= flags; } + //! Removes `flags` from format flags. ASMJIT_INLINE_NODEBUG void clearFlags(FormatFlags flags) noexcept { _flags &= ~flags; } //! Returns indentation for the given indentation `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint8_t indentation(FormatIndentationGroup group) const noexcept { return _indentation[group]; } + //! Sets indentation for the given indentation `group`. ASMJIT_INLINE_NODEBUG void setIndentation(FormatIndentationGroup group, uint32_t n) noexcept { _indentation[group] = uint8_t(n); } + //! Resets indentation for the given indentation `group` to zero. ASMJIT_INLINE_NODEBUG void resetIndentation(FormatIndentationGroup group) noexcept { _indentation[group] = uint8_t(0); } //! Returns padding for the given padding `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t padding(FormatPaddingGroup group) const noexcept { return _padding[group]; } + //! Sets padding for the given padding `group`. ASMJIT_INLINE_NODEBUG void setPadding(FormatPaddingGroup group, size_t n) noexcept { _padding[group] = uint16_t(n); } + //! Resets padding for the given padding `group` to zero, which means that a default padding will be used //! based on the target architecture properties. ASMJIT_INLINE_NODEBUG void resetPadding(FormatPaddingGroup group) noexcept { _padding[group] = uint16_t(0); } diff --git a/pe-packer/asmjit/core/formatter_p.h b/pe-packer/asmjit/core/formatter_p.h index 6070fd7..7cb0d45 100644 --- a/pe-packer/asmjit/core/formatter_p.h +++ b/pe-packer/asmjit/core/formatter_p.h @@ -1,12 +1,14 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FORMATTER_P_H_INCLUDED #define ASMJIT_CORE_FORMATTER_P_H_INCLUDED +#include "../core/compilerdefs.h" #include "../core/formatter.h" +#include "../core/operand.h" ASMJIT_BEGIN_NAMESPACE @@ -16,7 +18,8 @@ ASMJIT_BEGIN_NAMESPACE namespace Formatter { -static ASMJIT_FORCE_INLINE size_t paddingFromOptions(const FormatOptions& formatOptions, FormatPaddingGroup group) noexcept { +[[maybe_unused]] +static ASMJIT_INLINE size_t paddingFromOptions(const FormatOptions& formatOptions, FormatPaddingGroup group) noexcept { static constexpr uint16_t _defaultPaddingTable[uint32_t(FormatPaddingGroup::kMaxValue) + 1] = { 44, 26 }; static_assert(uint32_t(FormatPaddingGroup::kMaxValue) + 1 == 2, "If a new group is defined it must be added here"); @@ -24,6 +27,9 @@ static ASMJIT_FORCE_INLINE size_t paddingFromOptions(const FormatOptions& format return padding ? padding : size_t(_defaultPaddingTable[uint32_t(group)]); } +Error formatVirtRegName(String& sb, const VirtReg* vReg) noexcept; +Error formatVirtRegNameWithPrefix(String& sb, const char* prefix, size_t prefixSize, const VirtReg* vReg) noexcept; + } // {Formatter} //! \} diff --git a/pe-packer/asmjit/core/func.cpp b/pe-packer/asmjit/core/func.cpp index 04dc2aa..843d228 100644 --- a/pe-packer/asmjit/core/func.cpp +++ b/pe-packer/asmjit/core/func.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -20,20 +20,22 @@ ASMJIT_BEGIN_NAMESPACE -// CallConv - Init & Reset -// ======================= +// CallConv - Initialization & Reset +// ================================= ASMJIT_FAVOR_SIZE Error CallConv::init(CallConvId ccId, const Environment& environment) noexcept { reset(); #if !defined(ASMJIT_NO_X86) - if (environment.isFamilyX86()) + if (environment.isFamilyX86()) { return x86::FuncInternal::initCallConv(*this, ccId, environment); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (environment.isFamilyAArch64()) + if (environment.isFamilyAArch64()) { return a64::FuncInternal::initCallConv(*this, ccId, environment); + } #endif return DebugUtils::errored(kErrorInvalidArgument); @@ -46,8 +48,9 @@ ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& signature, const E CallConvId ccId = signature.callConvId(); uint32_t argCount = signature.argCount(); - if (ASMJIT_UNLIKELY(argCount > Globals::kMaxFuncArgs)) + if (ASMJIT_UNLIKELY(argCount > Globals::kMaxFuncArgs)) { return DebugUtils::errored(kErrorInvalidArgument); + } CallConv& cc = _callConv; ASMJIT_PROPAGATE(cc.init(ccId, environment)); @@ -65,17 +68,20 @@ ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& signature, const E _vaIndex = uint8_t(signature.vaIndex()); TypeId ret = signature.ret(); - if (ret != TypeId::kVoid) + if (ret != TypeId::kVoid) { _rets[0].initTypeId(TypeUtils::deabstract(ret, deabstractDelta)); + } #if !defined(ASMJIT_NO_X86) - if (environment.isFamilyX86()) + if (environment.isFamilyX86()) { return x86::FuncInternal::initFuncDetail(*this, signature, registerSize); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (environment.isFamilyAArch64()) - return a64::FuncInternal::initFuncDetail(*this, signature, registerSize); + if (environment.isFamilyAArch64()) { + return a64::FuncInternal::initFuncDetail(*this, signature); + } #endif // We should never bubble here as if `cc.init()` succeeded then there has to be an implementation for the current @@ -88,8 +94,9 @@ ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& signature, const E ASMJIT_FAVOR_SIZE Error FuncFrame::init(const FuncDetail& func) noexcept { Arch arch = func.callConv().arch(); - if (!Environment::isValidArch(arch)) + if (!Environment::isValidArch(arch)) { return DebugUtils::errored(kErrorInvalidArch); + } const ArchTraits& archTraits = ArchTraits::byArch(arch); @@ -99,13 +106,14 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::init(const FuncDetail& func) noexcept { _arch = arch; _spRegId = uint8_t(archTraits.spRegId()); - _saRegId = uint8_t(BaseReg::kIdBad); + _saRegId = uint8_t(Reg::kIdBad); uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment(); uint32_t minDynamicAlignment = Support::max(naturalStackAlignment, 16); - if (minDynamicAlignment == naturalStackAlignment) + if (minDynamicAlignment == naturalStackAlignment) { minDynamicAlignment <<= 1; + } _naturalStackAlignment = uint8_t(naturalStackAlignment); _minDynamicAlignment = uint8_t(minDynamicAlignment); @@ -137,8 +145,9 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::init(const FuncDetail& func) noexcept { // ==================== ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { - if (!Environment::isValidArch(arch())) + if (!Environment::isValidArch(arch())) { return DebugUtils::errored(kErrorInvalidArch); + } const ArchTraits& archTraits = ArchTraits::byArch(arch()); @@ -148,9 +157,7 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { // The final stack alignment must be updated accordingly to call and local stack alignments. uint32_t stackAlignment = _finalStackAlignment; - ASMJIT_ASSERT(stackAlignment == Support::max(_naturalStackAlignment, - _callStackAlignment, - _localStackAlignment)); + ASMJIT_ASSERT(stackAlignment == Support::max(_naturalStackAlignment, _callStackAlignment, _localStackAlignment)); bool hasFP = hasPreservedFP(); bool hasDA = hasDynamicAlignment(); @@ -165,32 +172,37 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { // Currently required by ARM, if this works differently across architectures we would have to generalize most // likely in CallConv. - if (kLr != BaseReg::kIdBad) + if (kLr != Reg::kIdBad) { _dirtyRegs[RegGroup::kGp] |= Support::bitMask(kLr); + } } // These two are identical if the function doesn't align its stack dynamically. uint32_t saRegId = _saRegId; - if (saRegId == BaseReg::kIdBad) + if (saRegId == Reg::kIdBad) { saRegId = kSp; + } // Fix stack arguments base-register from SP to FP in case it was not picked before and the function performs // dynamic stack alignment. - if (hasDA && saRegId == kSp) + if (hasDA && saRegId == kSp) { saRegId = kFp; + } // Mark as dirty any register but SP if used as SA pointer. - if (saRegId != kSp) + if (saRegId != kSp) { _dirtyRegs[RegGroup::kGp] |= Support::bitMask(saRegId); + } _spRegId = uint8_t(kSp); _saRegId = uint8_t(saRegId); // Setup stack size used to save preserved registers. uint32_t saveRestoreSizes[2] {}; - for (RegGroup group : RegGroupVirtValues{}) + for (RegGroup group : RegGroupVirtValues{}) { saveRestoreSizes[size_t(!archTraits.hasInstPushPop(group))] += Support::alignUp(Support::popcnt(savedRegs(group)) * saveRestoreRegSize(group), saveRestoreAlignment(group)); + } _pushPopSaveSize = uint16_t(saveRestoreSizes[0]); _extraRegSaveSize = uint16_t(saveRestoreSizes[1]); @@ -235,22 +247,25 @@ ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { // (basically the native register/pointer size). We don't adjust it now as `v` now contains the exact size // that the function requires to adjust (call frame + stack frame, vec stack size). The stack (if we consider // this size) is misaligned now, as it's always aligned before the function call - when `call()` is executed - // it pushes the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes (depending on the + // it pushes the current EIP|RIP onto the stack, and unaligns it by 12 or 8 bytes (depending on the // architecture). So count number of bytes needed to align it up to the function's CallFrame (the beginning). - if (v || hasFuncCalls() || !returnAddressSize) + if (v || hasFuncCalls() || !returnAddressSize) { v += Support::alignUpDiff(v + pushPopSaveSize() + returnAddressSize, stackAlignment); + } _pushPopSaveOffset = v; // Store 'pushPopSaveOffset' <- Function's push/pop save/restore starts here. _stackAdjustment = v; // Store 'stackAdjustment' <- SA used by 'add SP, SA' and 'sub SP, SA'. v += _pushPopSaveSize; // Count 'pushPopSaveSize' <- Function's push/pop save/restore ends here. _finalStackSize = v; // Store 'finalStackSize' <- Final stack used by the function. - if (!archTraits.hasLinkReg()) + if (!archTraits.hasLinkReg()) { v += registerSize; // Count 'ReturnAddress' <- As CALL pushes onto stack. + } // If the function performs dynamic stack alignment then the stack-adjustment must be aligned. - if (hasDA) + if (hasDA) { _stackAdjustment = Support::alignUp(_stackAdjustment, stackAlignment); + } // Calculate where the function arguments start relative to SP. _saOffsetFromSP = hasDA ? FuncFrame::kTagInvalidOffset : v; @@ -269,8 +284,9 @@ ASMJIT_FAVOR_SIZE Error FuncArgsAssignment::updateFuncFrame(FuncFrame& frame) co Arch arch = frame.arch(); const FuncDetail* func = funcDetail(); - if (!func) + if (!func) { return DebugUtils::errored(kErrorInvalidState); + } RAConstraints constraints; ASMJIT_PROPAGATE(constraints.init(arch)); @@ -283,4 +299,18 @@ ASMJIT_FAVOR_SIZE Error FuncArgsAssignment::updateFuncFrame(FuncFrame& frame) co return kErrorOk; } +// Func API - Tests +// ================ + +#if defined(ASMJIT_TEST) +UNIT(func_signature) { + FuncSignature signature; + signature.setRetT(); + signature.addArgT(); + signature.addArg(TypeId::kInt32); + + EXPECT_EQ(signature, FuncSignature::build()); +} +#endif + ASMJIT_END_NAMESPACE diff --git a/pe-packer/asmjit/core/func.h b/pe-packer/asmjit/core/func.h index 2dbedc1..da7844c 100644 --- a/pe-packer/asmjit/core/func.h +++ b/pe-packer/asmjit/core/func.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FUNC_H_INCLUDED @@ -28,9 +28,6 @@ ASMJIT_BEGIN_NAMESPACE //! - Target specific - calling conventions that are used by a particular architecture and ABI. For example //! Windows 64-bit calling convention and AMD64 SystemV calling convention. enum class CallConvId : uint8_t { - //! None or invalid (can't be used). - kNone = 0, - // Universal Calling Conventions // ----------------------------- @@ -38,48 +35,38 @@ enum class CallConvId : uint8_t { //! //! This is a universal calling convention, which is used to initialize specific calling conventions based on //! architecture, platform, and its ABI. - kCDecl = 1, + kCDecl = 0, //! `__stdcall` on targets that support this calling convention (X86). //! //! \note This calling convention is only supported on 32-bit X86. If used on environment that doesn't support //! this calling convention it will be replaced by \ref CallConvId::kCDecl. - kStdCall = 2, + kStdCall = 1, //! `__fastcall` on targets that support this calling convention (X86). //! //! \note This calling convention is only supported on 32-bit X86. If used on environment that doesn't support //! this calling convention it will be replaced by \ref CallConvId::kCDecl. - kFastCall = 3, + kFastCall = 2, - //! `__vectorcall` on targets that support this calling convention (X86/X64). + //! `__vectorcall` on targets that support this calling convention (X86|X86_64). //! //! \note This calling convention is only supported on 32-bit and 64-bit X86 architecture on Windows platform. //! If used on environment that doesn't support this calling it will be replaced by \ref CallConvId::kCDecl. - kVectorCall = 4, + kVectorCall = 3, //! `__thiscall` on targets that support this calling convention (X86). //! //! \note This calling convention is only supported on 32-bit X86 Windows platform. If used on environment that //! doesn't support this calling convention it will be replaced by \ref CallConvId::kCDecl. - kThisCall = 5, + kThisCall = 4, //! `__attribute__((regparm(1)))` convention (GCC and Clang). - kRegParm1 = 6, + kRegParm1 = 5, //! `__attribute__((regparm(2)))` convention (GCC and Clang). - kRegParm2 = 7, + kRegParm2 = 6, //! `__attribute__((regparm(3)))` convention (GCC and Clang). - kRegParm3 = 8, - - //! Soft-float calling convention (ARM). - //! - //! Floating point arguments are passed via general purpose registers. - kSoftFloat = 9, - - //! Hard-float calling convention (ARM). - //! - //! Floating point arguments are passed via SIMD registers. - kHardFloat = 10, + kRegParm3 = 7, //! AsmJit specific calling convention designed for calling functions inside a multimedia code that don't use many //! registers internally, but are long enough to be called and not inlined. These functions are usually used to @@ -91,28 +78,23 @@ enum class CallConvId : uint8_t { // ABI-Specific Calling Conventions // -------------------------------- + //! Soft-float calling convention (AArch32). + //! + //! Floating point arguments are passed via general purpose registers. + kSoftFloat = 30, + + //! Hard-float calling convention (AArch32). + //! + //! Floating point arguments are passed via SIMD registers. + kHardFloat = 31, + //! X64 System-V calling convention. kX64SystemV = 32, //! X64 Windows calling convention. kX64Windows = 33, //! Maximum value of `CallConvId`. - kMaxValue = kX64Windows, - - // Host Calling Conventions - // ------------------------ - - //! Host calling convention detected at compile-time. - kHost = -#if defined(_DOXYGEN) - DETECTED_AT_COMPILE_TIME -#elif ASMJIT_ARCH_ARM == 32 && defined(__SOFTFP__) - kSoftFloat -#elif ASMJIT_ARCH_ARM == 32 && !defined(__SOFTFP__) - kHardFloat -#else - kCDecl -#endif + kMaxValue = kX64Windows }; //! Strategy used by calling conventions to assign registers to function arguments. @@ -128,6 +110,8 @@ enum class CallConvStrategy : uint8_t { kX64Windows = 1, //! Windows 64-bit __vectorcall register assignment strategy. kX64VectorCall = 2, + //! Apple's AArch64 calling convention (differs compared to AArch64 calling convention used by Linux). + kAArch64Apple = 3, //! Maximum value of `CallConvStrategy`. kMaxValue = kX64VectorCall @@ -168,7 +152,7 @@ struct CallConv { //! \note This is not really AsmJit's limitation, it's just the number that makes sense considering all common //! calling conventions. Usually even conventions that use registers to pass function arguments are limited to 8 //! and less arguments passed via registers per group. - static constexpr uint32_t kMaxRegArgsPerGroup = 16; + static inline constexpr uint32_t kMaxRegArgsPerGroup = 16; //! \} @@ -189,6 +173,11 @@ struct CallConv { //! Natural stack alignment as defined by OS/ABI. uint8_t _naturalStackAlignment; + //! \cond INTERNAL + //! Reserved for future use. + uint8_t _reserved[2]; + //! \endcond + //! Calling convention flags. CallConvFlags _flags; @@ -229,7 +218,7 @@ struct CallConv { //! as it prevents from using an uninitialized data (CallConv doesn't have a constructor that would initialize it, //! it's just a struct). ASMJIT_INLINE_NODEBUG void reset() noexcept { - memset(this, 0, sizeof(*this)); + *this = CallConv{}; memset(_passedOrder.data(), 0xFF, sizeof(_passedOrder)); } @@ -239,46 +228,66 @@ struct CallConv { //! \{ //! Returns the target architecture of this calling convention. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return _arch; } + //! Sets the target architecture of this calling convention. ASMJIT_INLINE_NODEBUG void setArch(Arch arch) noexcept { _arch = arch; } //! Returns the calling convention id. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CallConvId id() const noexcept { return _id; } + //! Sets the calling convention id. ASMJIT_INLINE_NODEBUG void setId(CallConvId ccId) noexcept { _id = ccId; } //! Returns the strategy used to assign registers to arguments. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CallConvStrategy strategy() const noexcept { return _strategy; } + //! Sets the strategy used to assign registers to arguments. ASMJIT_INLINE_NODEBUG void setStrategy(CallConvStrategy ccStrategy) noexcept { _strategy = ccStrategy; } //! Tests whether the calling convention has the given `flag` set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(CallConvFlags flag) const noexcept { return Support::test(_flags, flag); } + //! Returns the calling convention flags, see `Flags`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CallConvFlags flags() const noexcept { return _flags; } + //! Adds the calling convention flags, see `Flags`. ASMJIT_INLINE_NODEBUG void setFlags(CallConvFlags flag) noexcept { _flags = flag; }; + //! Adds the calling convention flags, see `Flags`. ASMJIT_INLINE_NODEBUG void addFlags(CallConvFlags flags) noexcept { _flags |= flags; }; //! Tests whether this calling convention specifies 'RedZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRedZone() const noexcept { return _redZoneSize != 0; } + //! Tests whether this calling convention specifies 'SpillZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasSpillZone() const noexcept { return _spillZoneSize != 0; } //! Returns size of 'RedZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t redZoneSize() const noexcept { return _redZoneSize; } - //! Returns size of 'SpillZone'. - ASMJIT_INLINE_NODEBUG uint32_t spillZoneSize() const noexcept { return _spillZoneSize; } //! Sets size of 'RedZone'. ASMJIT_INLINE_NODEBUG void setRedZoneSize(uint32_t size) noexcept { _redZoneSize = uint8_t(size); } + + //! Returns size of 'SpillZone'. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t spillZoneSize() const noexcept { return _spillZoneSize; } + //! Sets size of 'SpillZone'. ASMJIT_INLINE_NODEBUG void setSpillZoneSize(uint32_t size) noexcept { _spillZoneSize = uint8_t(size); } //! Returns a natural stack alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t naturalStackAlignment() const noexcept { return _naturalStackAlignment; } + //! Sets a natural stack alignment. //! //! This function can be used to override the default stack alignment in case that you know that it's alignment is @@ -286,22 +295,28 @@ struct CallConv { ASMJIT_INLINE_NODEBUG void setNaturalStackAlignment(uint32_t value) noexcept { _naturalStackAlignment = uint8_t(value); } //! Returns the size of a register (or its part) to be saved and restored of the given `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saveRestoreRegSize(RegGroup group) const noexcept { return _saveRestoreRegSize[group]; } + //! Sets the size of a vector register (or its part) to be saved and restored. ASMJIT_INLINE_NODEBUG void setSaveRestoreRegSize(RegGroup group, uint32_t size) noexcept { _saveRestoreRegSize[group] = uint8_t(size); } //! Returns the alignment of a save-restore area of the given `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saveRestoreAlignment(RegGroup group) const noexcept { return _saveRestoreAlignment[group]; } + //! Sets the alignment of a save-restore area of the given `group`. ASMJIT_INLINE_NODEBUG void setSaveRestoreAlignment(RegGroup group, uint32_t alignment) noexcept { _saveRestoreAlignment[group] = uint8_t(alignment); } //! Returns the order of passed registers of the given `group`. + [[nodiscard]] inline const uint8_t* passedOrder(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _passedOrder[size_t(group)].id; } //! Returns the mask of passed registers of the given `group`. + [[nodiscard]] inline RegMask passedRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _passedRegs[size_t(group)]; @@ -346,6 +361,7 @@ struct CallConv { } //! Returns preserved register mask of the given `group`. + [[nodiscard]] inline RegMask preservedRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _preservedRegs[group]; @@ -362,14 +378,21 @@ struct CallConv { //! Function signature. //! -//! Contains information about function return type, count of arguments and their TypeIds. Function signature is -//! a low level structure which doesn't contain platform specific or calling convention specific information. +//! Contains information about a function return type, count of arguments, and their TypeIds. Function signature +//! is a low level structure which doesn't contain platform specific or calling convention specific information. +//! It's typically used to describe function arguments in a C-API like form, which is then used to calculate a +//! \ref FuncDetail instance, which then maps function signature into a platform and calling convention specific +//! format. +//! +//! Function signature can be built either dynamically by using \ref addArg() and \ref addArgT() functionality, +//! or dynamically by using a template-based \ref FuncSignature::build() function, which maps template types +//! into a function signature. struct FuncSignature { //! \name Constants //! \{ //! Doesn't have variable number of arguments (`...`). - static constexpr uint8_t kNoVarArgs = 0xFFu; + static inline constexpr uint8_t kNoVarArgs = 0xFFu; //! \} @@ -377,121 +400,180 @@ struct FuncSignature { //! \{ //! Calling convention id. - CallConvId _ccId; + CallConvId _ccId = CallConvId::kCDecl; //! Count of arguments. - uint8_t _argCount; + uint8_t _argCount = 0; //! Index of a first VA or `kNoVarArgs`. - uint8_t _vaIndex; + uint8_t _vaIndex = kNoVarArgs; //! Return value TypeId. - TypeId _ret; - //! Function arguments TypeIds. - const TypeId* _args; + TypeId _ret = TypeId::kVoid; + //! Reserved for future use. + uint8_t _reserved[4] {}; + //! Function argument TypeIds. + TypeId _args[Globals::kMaxFuncArgs] {}; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Default constructed function signature, initialized to \ref CallConvId::kCDecl, having no return value and no arguments. + ASMJIT_INLINE_CONSTEXPR FuncSignature() = default; + + //! Copy constructor, which is initialized to the same function signature as `other`. + ASMJIT_INLINE_CONSTEXPR FuncSignature(const FuncSignature& other) = default; + + //! Initializes the function signature with calling convention id `ccId` and variable argument's index `vaIndex`. + ASMJIT_INLINE_CONSTEXPR FuncSignature(CallConvId ccId, uint32_t vaIndex = kNoVarArgs) noexcept + : _ccId(ccId), + _vaIndex(uint8_t(vaIndex)) {} + + //! Initializes the function signature with calling convention id `ccId`, `vaIndex`, return value, and function arguments. + template + ASMJIT_INLINE_CONSTEXPR FuncSignature(CallConvId ccId, uint32_t vaIndex, TypeId ret, Args&&...args) noexcept + : _ccId(ccId), + _argCount(uint8_t(sizeof...(args))), + _vaIndex(uint8_t(vaIndex)), + _ret(ret), + _args{std::forward(args)...} {} + + //! Builds a function signature based on `RetValueAndArgs`. The first template argument is a function return type, + //! and function arguments follow. + //! + //! \note This function returns a new function signature, which can be passed to functions where it's required. It's + //! a convenience function that allows to build function signature statically based on types known at compile time, + //! which is common in JIT code generation. + template + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR FuncSignature build(CallConvId ccId = CallConvId::kCDecl, uint32_t vaIndex = kNoVarArgs) noexcept { + return FuncSignature(ccId, vaIndex, (TypeId(TypeUtils::TypeIdOfT::kTypeId))... ); + } + + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Copy assignment - function signature can be copied by value. + ASMJIT_INLINE FuncSignature& operator=(const FuncSignature& other) noexcept = default; + + //! Compares this function signature with `other` for equality.. + [[nodiscard]] + ASMJIT_INLINE bool operator==(const FuncSignature& other) const noexcept { return equals(other); } + + //! Compares this function signature with `other` for inequality.. + [[nodiscard]] + ASMJIT_INLINE bool operator!=(const FuncSignature& other) const noexcept { return !equals(other); } //! \} //! \name Initialization & Reset //! \{ - //! Initializes the function signature. - inline void init(CallConvId ccId, uint32_t vaIndex, TypeId ret, const TypeId* args, uint32_t argCount) noexcept { - ASMJIT_ASSERT(argCount <= 0xFF); - - _ccId = ccId; - _argCount = uint8_t(argCount); - _vaIndex = uint8_t(vaIndex); - _ret = ret; - _args = args; - } - + //! Resets this function signature to a default constructed state. ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = FuncSignature{}; } //! \} + //! \name Equality & Comparison + //! \{ + + //! Compares this function signature with `other` for equality.. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool equals(const FuncSignature& other) const noexcept { + return _ccId == other._ccId && + _argCount == other._argCount && + _vaIndex == other._vaIndex && + _ret == other._ret && + memcmp(_args, other._args, sizeof(_args)) == 0; + } + + //! \} + //! \name Accessors //! \{ //! Returns the calling convention. - ASMJIT_INLINE_NODEBUG CallConvId callConvId() const noexcept { return _ccId; } - //! Sets the calling convention to `ccId`; - ASMJIT_INLINE_NODEBUG void setCallConvId(CallConvId ccId) noexcept { _ccId = ccId; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR CallConvId callConvId() const noexcept { return _ccId; } - //! Tests whether the function has variable number of arguments (...). - ASMJIT_INLINE_NODEBUG bool hasVarArgs() const noexcept { return _vaIndex != kNoVarArgs; } - //! Returns the variable arguments (...) index, `kNoVarArgs` if none. - ASMJIT_INLINE_NODEBUG uint32_t vaIndex() const noexcept { return _vaIndex; } - //! Sets the variable arguments (...) index to `index`. - ASMJIT_INLINE_NODEBUG void setVaIndex(uint32_t index) noexcept { _vaIndex = uint8_t(index); } - //! Resets the variable arguments index (making it a non-va function). - ASMJIT_INLINE_NODEBUG void resetVaIndex() noexcept { _vaIndex = kNoVarArgs; } + //! Sets the calling convention to `ccId`; + ASMJIT_INLINE_CONSTEXPR void setCallConvId(CallConvId ccId) noexcept { _ccId = ccId; } + + //! Tests whether the function signature has a return value. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasRet() const noexcept { return _ret != TypeId::kVoid; } + + //! Returns the type of the return value. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR TypeId ret() const noexcept { return _ret; } + + //! Sets the return type to `retType`. + ASMJIT_INLINE_CONSTEXPR void setRet(TypeId retType) noexcept { _ret = retType; } + + //! Sets the return type based on `T`. + template + ASMJIT_INLINE_CONSTEXPR void setRetT() noexcept { setRet(TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + + + //! Returns the array of function arguments' types. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR const TypeId* args() const noexcept { return _args; } //! Returns the number of function arguments. - ASMJIT_INLINE_NODEBUG uint32_t argCount() const noexcept { return _argCount; } - - ASMJIT_INLINE_NODEBUG bool hasRet() const noexcept { return _ret != TypeId::kVoid; } - //! Returns the return value type. - ASMJIT_INLINE_NODEBUG TypeId ret() const noexcept { return _ret; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t argCount() const noexcept { return _argCount; } //! Returns the type of the argument at index `i`. + [[nodiscard]] inline TypeId arg(uint32_t i) const noexcept { ASMJIT_ASSERT(i < _argCount); return _args[i]; } - //! Returns the array of function arguments' types. - ASMJIT_INLINE_NODEBUG const TypeId* args() const noexcept { return _args; } - - //! \} -}; - -template -class FuncSignatureT : public FuncSignature { -public: - ASMJIT_INLINE_NODEBUG FuncSignatureT(CallConvId ccId = CallConvId::kHost, uint32_t vaIndex = kNoVarArgs) noexcept { - static constexpr TypeId ret_args[] = { (TypeId(TypeUtils::TypeIdOfT::kTypeId))... }; - init(ccId, vaIndex, ret_args[0], ret_args + 1, uint32_t(ASMJIT_ARRAY_SIZE(ret_args) - 1)); - } -}; - -//! Function signature builder. -class FuncSignatureBuilder : public FuncSignature { -public: - TypeId _builderArgList[Globals::kMaxFuncArgs]; - - //! \name Initialization & Reset - //! \{ - - ASMJIT_INLINE_NODEBUG FuncSignatureBuilder(CallConvId ccId = CallConvId::kHost, uint32_t vaIndex = kNoVarArgs) noexcept { - init(ccId, vaIndex, TypeId::kVoid, _builderArgList, 0); - } - - //! \} - - //! \name Accessors - //! \{ - - //! Sets the return type to `retType`. - ASMJIT_INLINE_NODEBUG void setRet(TypeId retType) noexcept { _ret = retType; } - //! Sets the return type based on `T`. - template - ASMJIT_INLINE_NODEBUG void setRetT() noexcept { setRet(TypeId(TypeUtils::TypeIdOfT::kTypeId)); } //! Sets the argument at index `index` to `argType`. inline void setArg(uint32_t index, TypeId argType) noexcept { ASMJIT_ASSERT(index < _argCount); - _builderArgList[index] = argType; + _args[index] = argType; } + //! Sets the argument at index `i` to the type based on `T`. template inline void setArgT(uint32_t index) noexcept { setArg(index, TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + //! Tests whether an argument can be added to the signature, use before calling \ref addArg() and \ref addArgT(). + //! + //! \note If you know that you are not adding more arguments than \ref Globals::kMaxFuncArgs then it's not necessary + //! to use this function. However, if you are adding arguments based on user input, for example, then either check + //! the number of arguments before using function signature or use \ref canAddArg() before actually adding them to + //! the function signature. + [[nodiscard]] + inline bool canAddArg() const noexcept { return _argCount < Globals::kMaxFuncArgs; } + //! Appends an argument of `type` to the function prototype. inline void addArg(TypeId type) noexcept { ASMJIT_ASSERT(_argCount < Globals::kMaxFuncArgs); - _builderArgList[_argCount++] = type; + _args[_argCount++] = type; } + //! Appends an argument of type based on `T` to the function prototype. template inline void addArgT() noexcept { addArg(TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + //! Tests whether the function has variable number of arguments (...). + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasVarArgs() const noexcept { return _vaIndex != kNoVarArgs; } + + //! Returns the variable arguments (...) index, `kNoVarArgs` if none. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t vaIndex() const noexcept { return _vaIndex; } + + //! Sets the variable arguments (...) index to `index`. + ASMJIT_INLINE_NODEBUG void setVaIndex(uint32_t index) noexcept { _vaIndex = uint8_t(index); } + + //! Resets the variable arguments index (making it a non-va function). + ASMJIT_INLINE_NODEBUG void resetVaIndex() noexcept { _vaIndex = kNoVarArgs; } + //! \} }; @@ -536,15 +618,17 @@ struct FuncValue { //! //! \{ - //! Initializes the `typeId` of this `FuncValue`. + //! Initializes this `FuncValue` only to the `typeId` provided - the rest of the values will be cleared. ASMJIT_INLINE_NODEBUG void initTypeId(TypeId typeId) noexcept { _data = uint32_t(typeId) << kTypeIdShift; } + //! Initializes this `FuncValue` to a register of `regType`, `regId`, and assigns its `typeId` and `flags`. ASMJIT_INLINE_NODEBUG void initReg(RegType regType, uint32_t regId, TypeId typeId, uint32_t flags = 0) noexcept { _data = (uint32_t(regType) << kRegTypeShift) | (regId << kRegIdShift) | (uint32_t(typeId) << kTypeIdShift) | kFlagIsReg | flags; } + //! Initializes this `FuncValue` to a stack at the given `offset` and assigns its `typeId`. ASMJIT_INLINE_NODEBUG void initStack(int32_t offset, TypeId typeId) noexcept { _data = (uint32_t(offset) << kStackOffsetShift) | (uint32_t(typeId) << kTypeIdShift) | kFlagIsStack; } @@ -561,11 +645,13 @@ struct FuncValue { //! //! \{ + //! Assigns a register of `regType` and `regId`. inline void assignRegData(RegType regType, uint32_t regId) noexcept { ASMJIT_ASSERT((_data & (kRegTypeMask | kRegIdMask)) == 0); _data |= (uint32_t(regType) << kRegTypeShift) | (regId << kRegIdShift) | kFlagIsReg; } + //! Assigns a stack location at `offset`. inline void assignStackOffset(int32_t offset) noexcept { ASMJIT_ASSERT((_data & kStackOffsetMask) == 0); _data |= (uint32_t(offset) << kStackOffsetShift) | kFlagIsStack; @@ -579,48 +665,73 @@ struct FuncValue { //! Returns true if the value is initialized (explicit bool cast). ASMJIT_INLINE_NODEBUG explicit operator bool() const noexcept { return _data != 0; } + //! \cond INTERNAL ASMJIT_INLINE_NODEBUG void _replaceValue(uint32_t mask, uint32_t value) noexcept { _data = (_data & ~mask) | value; } + //! \endcond //! Tests whether the `FuncValue` has a flag `flag` set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(uint32_t flag) const noexcept { return Support::test(_data, flag); } + //! Adds `flags` to `FuncValue`. ASMJIT_INLINE_NODEBUG void addFlags(uint32_t flags) noexcept { _data |= flags; } + //! Clears `flags` of `FuncValue`. ASMJIT_INLINE_NODEBUG void clearFlags(uint32_t flags) noexcept { _data &= ~flags; } //! Tests whether the value is initialized (i.e. contains a valid data). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInitialized() const noexcept { return _data != 0; } + //! Tests whether the argument is passed by register. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isReg() const noexcept { return hasFlag(kFlagIsReg); } + //! Tests whether the argument is passed by stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isStack() const noexcept { return hasFlag(kFlagIsStack); } + //! Tests whether the argument is passed by register. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAssigned() const noexcept { return hasFlag(kFlagIsReg | kFlagIsStack); } + //! Tests whether the argument is passed through a pointer (used by WIN64 to pass XMM|YMM|ZMM). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isIndirect() const noexcept { return hasFlag(kFlagIsIndirect); } //! Tests whether the argument was already processed (used internally). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isDone() const noexcept { return hasFlag(kFlagIsDone); } //! Returns a register type of the register used to pass function argument or return value. + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegType regType() const noexcept { return RegType((_data & kRegTypeMask) >> kRegTypeShift); } + //! Sets a register type of the register used to pass function argument or return value. ASMJIT_INLINE_NODEBUG void setRegType(RegType regType) noexcept { _replaceValue(kRegTypeMask, uint32_t(regType) << kRegTypeShift); } //! Returns a physical id of the register used to pass function argument or return value. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t regId() const noexcept { return (_data & kRegIdMask) >> kRegIdShift; } + //! Sets a physical id of the register used to pass function argument or return value. ASMJIT_INLINE_NODEBUG void setRegId(uint32_t regId) noexcept { _replaceValue(kRegIdMask, regId << kRegIdShift); } //! Returns a stack offset of this argument. + [[nodiscard]] ASMJIT_INLINE_NODEBUG int32_t stackOffset() const noexcept { return int32_t(_data & kStackOffsetMask) >> kStackOffsetShift; } + //! Sets a stack offset of this argument. ASMJIT_INLINE_NODEBUG void setStackOffset(int32_t offset) noexcept { _replaceValue(kStackOffsetMask, uint32_t(offset) << kStackOffsetShift); } //! Tests whether the argument or return value has associated `TypeId`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasTypeId() const noexcept { return Support::test(_data, kTypeIdMask); } + //! Returns a TypeId of this argument or return value. + [[nodiscard]] ASMJIT_INLINE_NODEBUG TypeId typeId() const noexcept { return TypeId((_data & kTypeIdMask) >> kTypeIdShift); } + //! Sets a TypeId of this argument or return value. ASMJIT_INLINE_NODEBUG void setTypeId(TypeId typeId) noexcept { _replaceValue(kTypeIdMask, uint32_t(typeId) << kTypeIdShift); } @@ -654,6 +765,7 @@ public: //! \{ //! Calculates how many values are in the pack, checking for non-values from the end. + [[nodiscard]] inline uint32_t count() const noexcept { uint32_t n = Globals::kMaxValuePack; while (n && !_values[n - 1]) @@ -661,40 +773,58 @@ public: return n; } + //! Returns values in this value in the pack. + //! + //! \note The returned array has exactly \ref Globals::kMaxValuePack elements. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncValue* values() noexcept { return _values; } + + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncValue* values() const noexcept { return _values; } + //! Resets a value at the given `index` in the pack, which makes it unassigned. inline void resetValue(size_t index) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); _values[index].reset(); } + //! Tests whether the value at the given `index` in the pack is assigned. inline bool hasValue(size_t index) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); return _values[index].isInitialized(); } - inline void assignReg(size_t index, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + //! Assigns a register at the given `index` to `reg` and an optional `typeId`. + inline void assignReg(size_t index, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); ASMJIT_ASSERT(reg.isPhysReg()); - _values[index].initReg(reg.type(), reg.id(), typeId); + _values[index].initReg(reg.regType(), reg.id(), typeId); } + //! Assigns a register at the given `index` to `regType`, `regId`, and an optional `typeId`. inline void assignReg(size_t index, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); _values[index].initReg(regType, regId, typeId); } + //! Assigns a stack location at the given `index` to `offset` and an optional `typeId`. inline void assignStack(size_t index, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(index < Globals::kMaxValuePack); _values[index].initStack(offset, typeId); } + //! Accesses the value in the pack at the given `index`. + //! + //! \note The maximum index value is `Globals::kMaxValuePack - 1`. + [[nodiscard]] inline FuncValue& operator[](size_t index) { ASMJIT_ASSERT(index < Globals::kMaxValuePack); return _values[index]; } + //! \overload + [[nodiscard]] inline const FuncValue& operator[](size_t index) const { ASMJIT_ASSERT(index < Globals::kMaxValuePack); return _values[index]; @@ -717,28 +847,33 @@ enum class FuncAttributes : uint32_t { kHasFuncCalls = 0x00000020u, //! Function has aligned save/restore of vector registers. kAlignedVecSR = 0x00000040u, + //! Function must begin with an instruction that marks a start of a branch or function. + //! + //! * `ENDBR32/ENDBR64` instruction is inserted at the beginning of the function (X86|X86_64). + //! * `BTI` instruction is inserted at the beginning of the function (AArch64) + kIndirectBranchProtection = 0x00000080u, //! FuncFrame is finalized and can be used by prolog/epilog inserter (PEI). kIsFinalized = 0x00000800u, // X86 Specific Attributes // ----------------------- - //! Enables the use of AVX within the function's body, prolog, and epilog (X86). + //! Enables the use of AVX within the function's body, prolog, and epilog (X86|X86_64). //! //! This flag instructs prolog and epilog emitter to use AVX instead of SSE for manipulating XMM registers. kX86_AVXEnabled = 0x00010000u, - //! Enables the use of AVX-512 within the function's body, prolog, and epilog (X86). + //! Enables the use of AVX-512 within the function's body, prolog, and epilog (X86|X86_64). //! //! This flag instructs Compiler register allocator to use additional 16 registers introduced by AVX-512. //! Additionally, if the functions saves full width of ZMM registers (custom calling conventions only) then //! the prolog/epilog inserter would use AVX-512 move instructions to emit the save and restore sequence. kX86_AVX512Enabled = 0x00020000u, - //! This flag instructs the epilog writer to emit EMMS instruction before RET (X86). + //! This flag instructs the epilog writer to emit EMMS instruction before RET (X86|X86_64). kX86_MMXCleanup = 0x00040000u, - //! This flag instructs the epilog writer to emit VZEROUPPER instruction before RET (X86). + //! This flag instructs the epilog writer to emit VZEROUPPER instruction before RET (X86|X86_64). kX86_AVXCleanup = 0x00080000u }; ASMJIT_DEFINE_ENUM_FLAGS(FuncAttributes) @@ -753,10 +888,8 @@ public: //! \name Constants //! \{ - enum : uint8_t { - //! Doesn't have variable number of arguments (`...`). - kNoVarArgs = 0xFFu - }; + //! Function doesn't have a variable number of arguments (`...`). + static inline constexpr uint8_t kNoVarArgs = 0xFFu; //! \} @@ -785,72 +918,105 @@ public: //! \name Construction & Destruction //! \{ + //! Creates a default constructed \ref FuncDetail. ASMJIT_INLINE_NODEBUG FuncDetail() noexcept {} + + //! Copy constructor. + //! + //! Function details are copyable. ASMJIT_INLINE_NODEBUG FuncDetail(const FuncDetail& other) noexcept = default; //! Initializes this `FuncDetail` to the given signature. ASMJIT_API Error init(const FuncSignature& signature, const Environment& environment) noexcept; - ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = FuncDetail{}; } //! \} //! \name Overloaded Operators //! \{ + //! Assignment operator, copies `other` to this \ref FuncDetail. ASMJIT_INLINE_NODEBUG FuncDetail& operator=(const FuncDetail& other) noexcept = default; //! \} + //! \name Reset + //! \{ + + //! Resets the function detail to its default constructed state. + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = FuncDetail{}; } + + //! \} + //! \name Accessors //! \{ //! Returns the function's calling convention, see `CallConv`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const CallConv& callConv() const noexcept { return _callConv; } //! Returns the associated calling convention flags, see `CallConv::Flags`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CallConvFlags flags() const noexcept { return _callConv.flags(); } + //! Checks whether a CallConv `flag` is set, see `CallConv::Flags`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(CallConvFlags ccFlag) const noexcept { return _callConv.hasFlag(ccFlag); } //! Tests whether the function has a return value. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRet() const noexcept { return bool(_rets[0]); } + //! Returns the number of function arguments. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t argCount() const noexcept { return _argCount; } //! Returns function return values. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncValuePack& retPack() noexcept { return _rets; } + //! Returns function return values. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncValuePack& retPack() const noexcept { return _rets; } //! Returns a function return value associated with the given `valueIndex`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncValue& ret(size_t valueIndex = 0) noexcept { return _rets[valueIndex]; } + //! Returns a function return value associated with the given `valueIndex` (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncValue& ret(size_t valueIndex = 0) const noexcept { return _rets[valueIndex]; } //! Returns function argument packs array. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncValuePack* argPacks() noexcept { return _args; } + //! Returns function argument packs array (const). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncValuePack* argPacks() const noexcept { return _args; } //! Returns function argument pack at the given `argIndex`. + [[nodiscard]] inline FuncValuePack& argPack(size_t argIndex) noexcept { ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); return _args[argIndex]; } //! Returns function argument pack at the given `argIndex` (const). + [[nodiscard]] inline const FuncValuePack& argPack(size_t argIndex) const noexcept { ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); return _args[argIndex]; } //! Returns an argument at `valueIndex` from the argument pack at the given `argIndex`. + [[nodiscard]] inline FuncValue& arg(size_t argIndex, size_t valueIndex = 0) noexcept { ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); return _args[argIndex][valueIndex]; } //! Returns an argument at `valueIndex` from the argument pack at the given `argIndex` (const). + [[nodiscard]] inline const FuncValue& arg(size_t argIndex, size_t valueIndex = 0) const noexcept { ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); return _args[argIndex][valueIndex]; @@ -865,28 +1031,43 @@ public: } //! Tests whether the function has variable arguments. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasVarArgs() const noexcept { return _vaIndex != kNoVarArgs; } + //! Returns an index of a first variable argument. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t vaIndex() const noexcept { return _vaIndex; } //! Tests whether the function passes one or more argument by stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasStackArgs() const noexcept { return _argStackSize != 0; } + //! Returns stack size needed for function arguments passed on the stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t argStackSize() const noexcept { return _argStackSize; } //! Returns red zone size. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t redZoneSize() const noexcept { return _callConv.redZoneSize(); } + //! Returns spill zone size. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t spillZoneSize() const noexcept { return _callConv.spillZoneSize(); } + //! Returns natural stack alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t naturalStackAlignment() const noexcept { return _callConv.naturalStackAlignment(); } //! Returns a mask of all passed registers of the given register `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegMask passedRegs(RegGroup group) const noexcept { return _callConv.passedRegs(group); } + //! Returns a mask of all preserved registers of the given register `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegMask preservedRegs(RegGroup group) const noexcept { return _callConv.preservedRegs(group); } //! Returns a mask of all used registers of the given register `group`. + [[nodiscard]] inline RegMask usedRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _usedRegs[size_t(group)]; @@ -946,10 +1127,15 @@ public: //! \name Constants //! \{ - enum : uint32_t { - //! Tag used to inform that some offset is invalid. - kTagInvalidOffset = 0xFFFFFFFFu - }; + //! Tag used to inform that some offset is invalid. + static inline constexpr uint32_t kTagInvalidOffset = 0xFFFFFFFFu; + + //! \} + + //! \name Types + //! \{ + + using RegMasks = Support::Array; //! \} @@ -957,87 +1143,100 @@ public: //! \{ //! Function attributes. - FuncAttributes _attributes; + FuncAttributes _attributes {}; //! Target architecture. - Arch _arch; + Arch _arch {}; //! SP register ID (to access call stack and local stack). - uint8_t _spRegId; + uint8_t _spRegId = uint8_t(Reg::kIdBad); //! SA register ID (to access stack arguments). - uint8_t _saRegId; + uint8_t _saRegId = uint8_t(Reg::kIdBad); //! Red zone size (copied from CallConv). - uint8_t _redZoneSize; + uint8_t _redZoneSize = 0; //! Spill zone size (copied from CallConv). - uint8_t _spillZoneSize; + uint8_t _spillZoneSize = 0; //! Natural stack alignment (copied from CallConv). - uint8_t _naturalStackAlignment; + uint8_t _naturalStackAlignment = 0; //! Minimum stack alignment to turn on dynamic alignment. - uint8_t _minDynamicAlignment; + uint8_t _minDynamicAlignment = 0; //! Call stack alignment. - uint8_t _callStackAlignment; + uint8_t _callStackAlignment = 0; //! Local stack alignment. - uint8_t _localStackAlignment; + uint8_t _localStackAlignment = 0; //! Final stack alignment. - uint8_t _finalStackAlignment; + uint8_t _finalStackAlignment = 0; //! Adjustment of the stack before returning (X86-STDCALL). - uint16_t _calleeStackCleanup; + uint16_t _calleeStackCleanup = 0; //! Call stack size. - uint32_t _callStackSize; + uint32_t _callStackSize = 0; //! Local stack size. - uint32_t _localStackSize; + uint32_t _localStackSize = 0; //! Final stack size (sum of call stack and local stack). - uint32_t _finalStackSize; + uint32_t _finalStackSize = 0; //! Local stack offset (non-zero only if call stack is used). - uint32_t _localStackOffset; + uint32_t _localStackOffset = 0; //! Offset relative to SP that contains previous SP (before alignment). - uint32_t _daOffset; + uint32_t _daOffset = 0; //! Offset of the first stack argument relative to SP. - uint32_t _saOffsetFromSP; + uint32_t _saOffsetFromSP = 0; //! Offset of the first stack argument relative to SA (_saRegId or FP). - uint32_t _saOffsetFromSA; + uint32_t _saOffsetFromSA = 0; //! Local stack adjustment in prolog/epilog. - uint32_t _stackAdjustment; + uint32_t _stackAdjustment = 0; //! Registers that are dirty. - Support::Array _dirtyRegs; + RegMasks _dirtyRegs {}; //! Registers that must be preserved (copied from CallConv). - Support::Array _preservedRegs; + RegMasks _preservedRegs {}; + //! Registers that are unavailable. + RegMasks _unavailableRegs {}; //! Size to save/restore per register group. - Support::Array _saveRestoreRegSize; + Support::Array _saveRestoreRegSize {}; //! Alignment of save/restore area per register group. - Support::Array _saveRestoreAlignment; + Support::Array _saveRestoreAlignment {}; //! Stack size required to save registers with push/pop. - uint16_t _pushPopSaveSize; + uint16_t _pushPopSaveSize = 0; //! Stack size required to save extra registers that cannot use push/pop. - uint16_t _extraRegSaveSize; + uint16_t _extraRegSaveSize = 0; //! Offset where registers saved/restored via push/pop are stored - uint32_t _pushPopSaveOffset; + uint32_t _pushPopSaveOffset = 0; //! Offset where extra registers that cannot use push/pop are stored. - uint32_t _extraRegSaveOffset; + uint32_t _extraRegSaveOffset = 0; //! \} //! \name Construction & Destruction //! \{ - ASMJIT_INLINE_NODEBUG FuncFrame() noexcept { reset(); } + //! Creates a default constructed function frame, which has initialized all members to their default values. + ASMJIT_INLINE_NODEBUG FuncFrame() noexcept = default; + //! Creates a copy of `other` function frame. ASMJIT_INLINE_NODEBUG FuncFrame(const FuncFrame& other) noexcept = default; - ASMJIT_API Error init(const FuncDetail& func) noexcept; + //! \} - ASMJIT_INLINE_NODEBUG void reset() noexcept { - memset(this, 0, sizeof(FuncFrame)); - _spRegId = BaseReg::kIdBad; - _saRegId = BaseReg::kIdBad; - _daOffset = kTagInvalidOffset; - } + //! \name Initialization & Reset + //! \{ + + //! Initializes the function frame based on `func` detail. + ASMJIT_API Error init(const FuncDetail& func) noexcept; + //! Resets the function frame into its default constructed state. + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = FuncFrame{}; } + + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Copy assignment - function frame is copy assignable. + ASMJIT_INLINE_NODEBUG FuncFrame& operator=(const FuncFrame& other) noexcept = default; //! \} @@ -1045,83 +1244,133 @@ public: //! \{ //! Returns the target architecture of the function frame. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return _arch; } //! Returns function frame attributes, see `Attributes`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FuncAttributes attributes() const noexcept { return _attributes; } + //! Checks whether the FuncFame contains an attribute `attr`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAttribute(FuncAttributes attr) const noexcept { return Support::test(_attributes, attr); } + //! Adds attributes `attrs` to the FuncFrame. ASMJIT_INLINE_NODEBUG void addAttributes(FuncAttributes attrs) noexcept { _attributes |= attrs; } + //! Clears attributes `attrs` from the FrameFrame. ASMJIT_INLINE_NODEBUG void clearAttributes(FuncAttributes attrs) noexcept { _attributes &= ~attrs; } //! Tests whether the function has variable number of arguments. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasVarArgs() const noexcept { return hasAttribute(FuncAttributes::kHasVarArgs); } + //! Sets the variable arguments flag. ASMJIT_INLINE_NODEBUG void setVarArgs() noexcept { addAttributes(FuncAttributes::kHasVarArgs); } + //! Resets variable arguments flag. ASMJIT_INLINE_NODEBUG void resetVarArgs() noexcept { clearAttributes(FuncAttributes::kHasVarArgs); } //! Tests whether the function preserves frame pointer (EBP|ESP on X86). + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasPreservedFP() const noexcept { return hasAttribute(FuncAttributes::kHasPreservedFP); } + //! Enables preserved frame pointer. ASMJIT_INLINE_NODEBUG void setPreservedFP() noexcept { addAttributes(FuncAttributes::kHasPreservedFP); } + //! Disables preserved frame pointer. ASMJIT_INLINE_NODEBUG void resetPreservedFP() noexcept { clearAttributes(FuncAttributes::kHasPreservedFP); } //! Tests whether the function calls other functions. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFuncCalls() const noexcept { return hasAttribute(FuncAttributes::kHasFuncCalls); } - //! Sets `kFlagHasCalls` to true. + + //! Sets `FuncAttributes::kHasFuncCalls` to true. ASMJIT_INLINE_NODEBUG void setFuncCalls() noexcept { addAttributes(FuncAttributes::kHasFuncCalls); } - //! Sets `kFlagHasCalls` to false. + + //! Sets `FuncAttributes::kHasFuncCalls` to false. ASMJIT_INLINE_NODEBUG void resetFuncCalls() noexcept { clearAttributes(FuncAttributes::kHasFuncCalls); } + //! Tests whether the function uses indirect branch protection, see \ref FuncAttributes::kIndirectBranchProtection. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasIndirectBranchProtection() const noexcept { return hasAttribute(FuncAttributes::kIndirectBranchProtection); } + + //! Enabled indirect branch protection (sets `FuncAttributes::kIndirectBranchProtection` attribute to true). + ASMJIT_INLINE_NODEBUG void setIndirectBranchProtection() noexcept { addAttributes(FuncAttributes::kIndirectBranchProtection); } + + //! Disables indirect branch protection (sets `FuncAttributes::kIndirectBranchProtection` attribute to false). + ASMJIT_INLINE_NODEBUG void resetIndirectBranchProtection() noexcept { clearAttributes(FuncAttributes::kIndirectBranchProtection); } + //! Tests whether the function has AVX enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAvxEnabled() const noexcept { return hasAttribute(FuncAttributes::kX86_AVXEnabled); } + //! Enables AVX use. ASMJIT_INLINE_NODEBUG void setAvxEnabled() noexcept { addAttributes(FuncAttributes::kX86_AVXEnabled); } + //! Disables AVX use. ASMJIT_INLINE_NODEBUG void resetAvxEnabled() noexcept { clearAttributes(FuncAttributes::kX86_AVXEnabled); } //! Tests whether the function has AVX-512 enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isAvx512Enabled() const noexcept { return hasAttribute(FuncAttributes::kX86_AVX512Enabled); } + //! Enables AVX-512 use. ASMJIT_INLINE_NODEBUG void setAvx512Enabled() noexcept { addAttributes(FuncAttributes::kX86_AVX512Enabled); } + //! Disables AVX-512 use. ASMJIT_INLINE_NODEBUG void resetAvx512Enabled() noexcept { clearAttributes(FuncAttributes::kX86_AVX512Enabled); } //! Tests whether the function has MMX cleanup - 'emms' instruction in epilog. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasMmxCleanup() const noexcept { return hasAttribute(FuncAttributes::kX86_MMXCleanup); } + //! Enables MMX cleanup. ASMJIT_INLINE_NODEBUG void setMmxCleanup() noexcept { addAttributes(FuncAttributes::kX86_MMXCleanup); } + //! Disables MMX cleanup. ASMJIT_INLINE_NODEBUG void resetMmxCleanup() noexcept { clearAttributes(FuncAttributes::kX86_MMXCleanup); } //! Tests whether the function has AVX cleanup - 'vzeroupper' instruction in epilog. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAvxCleanup() const noexcept { return hasAttribute(FuncAttributes::kX86_AVXCleanup); } + //! Enables AVX cleanup. ASMJIT_INLINE_NODEBUG void setAvxCleanup() noexcept { addAttributes(FuncAttributes::kX86_AVXCleanup); } + //! Disables AVX cleanup. ASMJIT_INLINE_NODEBUG void resetAvxCleanup() noexcept { clearAttributes(FuncAttributes::kX86_AVXCleanup); } //! Tests whether the function uses call stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasCallStack() const noexcept { return _callStackSize != 0; } + //! Tests whether the function uses local stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasLocalStack() const noexcept { return _localStackSize != 0; } + //! Tests whether vector registers can be saved and restored by using aligned reads and writes. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasAlignedVecSR() const noexcept { return hasAttribute(FuncAttributes::kAlignedVecSR); } + //! Tests whether the function has to align stack dynamically. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasDynamicAlignment() const noexcept { return _finalStackAlignment >= _minDynamicAlignment; } //! Tests whether the calling convention specifies 'RedZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasRedZone() const noexcept { return _redZoneSize != 0; } - //! Tests whether the calling convention specifies 'SpillZone'. - ASMJIT_INLINE_NODEBUG bool hasSpillZone() const noexcept { return _spillZoneSize != 0; } //! Returns the size of 'RedZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t redZoneSize() const noexcept { return _redZoneSize; } + + //! Tests whether the calling convention specifies 'SpillZone'. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasSpillZone() const noexcept { return _spillZoneSize != 0; } + //! Returns the size of 'SpillZone'. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t spillZoneSize() const noexcept { return _spillZoneSize; } //! Resets the size of red zone, which would disable it entirely. @@ -1133,20 +1382,31 @@ public: ASMJIT_INLINE_NODEBUG void resetRedZone() noexcept { _redZoneSize = 0; } //! Returns natural stack alignment (guaranteed stack alignment upon entry). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t naturalStackAlignment() const noexcept { return _naturalStackAlignment; } + //! Returns natural stack alignment (guaranteed stack alignment upon entry). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t minDynamicAlignment() const noexcept { return _minDynamicAlignment; } //! Tests whether the callee must adjust SP before returning (X86-STDCALL only) + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasCalleeStackCleanup() const noexcept { return _calleeStackCleanup != 0; } + //! Returns home many bytes of the stack the callee must adjust before returning (X86-STDCALL only) + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t calleeStackCleanup() const noexcept { return _calleeStackCleanup; } //! Returns call stack alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t callStackAlignment() const noexcept { return _callStackAlignment; } + //! Returns local stack alignment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t localStackAlignment() const noexcept { return _localStackAlignment; } + //! Returns final stack alignment (the maximum value of call, local, and natural stack alignments). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t finalStackAlignment() const noexcept { return _finalStackAlignment; } //! Sets call stack alignment. @@ -1182,42 +1442,57 @@ public: } //! Returns call stack size. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t callStackSize() const noexcept { return _callStackSize; } + //! Returns local stack size. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t localStackSize() const noexcept { return _localStackSize; } //! Sets call stack size. ASMJIT_INLINE_NODEBUG void setCallStackSize(uint32_t size) noexcept { _callStackSize = size; } + //! Sets local stack size. ASMJIT_INLINE_NODEBUG void setLocalStackSize(uint32_t size) noexcept { _localStackSize = size; } //! Combines call stack size with `size`, updating it to the greater value. ASMJIT_INLINE_NODEBUG void updateCallStackSize(uint32_t size) noexcept { _callStackSize = Support::max(_callStackSize, size); } + //! Combines local stack size with `size`, updating it to the greater value. ASMJIT_INLINE_NODEBUG void updateLocalStackSize(uint32_t size) noexcept { _localStackSize = Support::max(_localStackSize, size); } //! Returns final stack size (only valid after the FuncFrame is finalized). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t finalStackSize() const noexcept { return _finalStackSize; } //! Returns an offset to access the local stack (non-zero only if call stack is used). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t localStackOffset() const noexcept { return _localStackOffset; } //! Tests whether the function prolog/epilog requires a memory slot for storing unaligned SP. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasDAOffset() const noexcept { return _daOffset != kTagInvalidOffset; } + //! Returns a memory offset used to store DA (dynamic alignment) slot (relative to SP). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t daOffset() const noexcept { return _daOffset; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saOffset(uint32_t regId) const noexcept { return regId == _spRegId ? saOffsetFromSP() : saOffsetFromSA(); } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saOffsetFromSP() const noexcept { return _saOffsetFromSP; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saOffsetFromSA() const noexcept { return _saOffsetFromSA; } //! Returns mask of registers of the given register `group` that are modified by the function. The engine would //! then calculate which registers must be saved & restored by the function by using the data provided by the //! calling convention. + [[nodiscard]] inline RegMask dirtyRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _dirtyRegs[group]; @@ -1239,23 +1514,28 @@ public: } //! \overload - inline void addDirtyRegs(const BaseReg& reg) noexcept { + inline void addDirtyRegs(const Reg& reg) noexcept { ASMJIT_ASSERT(reg.id() < Globals::kMaxPhysRegs); - addDirtyRegs(reg.group(), Support::bitMask(reg.id())); + addDirtyRegs(reg.regGroup(), Support::bitMask(reg.id())); } //! \overload template - inline void addDirtyRegs(const BaseReg& reg, Args&&... args) noexcept { + inline void addDirtyRegs(const Reg& reg, Args&&... args) noexcept { addDirtyRegs(reg); addDirtyRegs(std::forward(args)...); } + //! A helper function to set all registers from all register groups dirty. + //! + //! \note This should not be used in general as it's the most pessimistic case. However, it can be used for testing + //! or in cases in which all registers are considered clobbered. ASMJIT_INLINE_NODEBUG void setAllDirty() noexcept { for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_dirtyRegs); i++) _dirtyRegs[i] = 0xFFFFFFFFu; } + //! A helper function to set all registers from the given register `group` dirty. inline void setAllDirty(RegGroup group) noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); _dirtyRegs[group] = 0xFFFFFFFFu; @@ -1264,54 +1544,134 @@ public: //! Returns a calculated mask of registers of the given `group` that will be saved and restored in the function's //! prolog and epilog, respectively. The register mask is calculated from both `dirtyRegs` (provided by user) and //! `preservedMask` (provided by the calling convention). + [[nodiscard]] inline RegMask savedRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _dirtyRegs[group] & _preservedRegs[group]; } + //! Returns all dirty registers as a Support::Array<> type. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const RegMasks& dirtyRegs() const noexcept { return _dirtyRegs; } + + //! Returns all preserved registers as a Support::Array<> type. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const RegMasks& preservedRegs() const noexcept { return _preservedRegs; } + //! Returns the mask of preserved registers of the given register `group`. //! //! Preserved registers are those that must survive the function call unmodified. The function can only modify //! preserved registers it they are saved and restored in function's prolog and epilog, respectively. + [[nodiscard]] inline RegMask preservedRegs(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _preservedRegs[group]; } + //! Sets which registers (as a mask) are unavailable for allocation. + //! + //! \note This completely overwrites the current unavailable mask. + inline void setUnavailableRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _unavailableRegs[group] = regs; + } + + //! Adds registers (as a mask) to the unavailable set. + inline void addUnavailableRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _unavailableRegs[group] |= regs; + } + + //! Adds a single register to the unavailable set. + inline void addUnavailableRegs(const Reg& reg) noexcept { + ASMJIT_ASSERT(reg.id() < Globals::kMaxPhysRegs); + addUnavailableRegs(reg.regGroup(), Support::bitMask(reg.id())); + } + + //! Adds multiple registers to the unavailable set. + template + inline void addUnavailableRegs(const Reg& reg, Args&&... args) noexcept { + addUnavailableRegs(reg); + addUnavailableRegs(std::forward(args)...); + } + + //! Clears all unavailable registers across all register groups (i.e., makes them all available again). + ASMJIT_INLINE_NODEBUG void clearUnavailableRegs() noexcept { + for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_unavailableRegs); i++) + _unavailableRegs[i] = 0; + } + + //! Clears all unavailable registers in a specific register group. + inline void clearUnavailableRegs(RegGroup group) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _unavailableRegs[group] = 0; + } + + //! Returns the set of unavailable registers for the given group. + [[nodiscard]] + inline RegMask unavailableRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _unavailableRegs[group]; + } + + //! Returns all unavailable registers as a Support::Array<>. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG const RegMasks& unavailableRegs() const noexcept { + return _unavailableRegs; + } + + //! Returns the size of a save-restore are for the required register `group`. + [[nodiscard]] inline uint32_t saveRestoreRegSize(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _saveRestoreRegSize[group]; } + //! Returns the alignment that must be guaranteed to save/restore the required register `group`. + [[nodiscard]] inline uint32_t saveRestoreAlignment(RegGroup group) const noexcept { ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); return _saveRestoreAlignment[group]; } - ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != Reg::kIdBad; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t saRegId() const noexcept { return _saRegId; } + ASMJIT_INLINE_NODEBUG void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } - ASMJIT_INLINE_NODEBUG void resetSARegId() { setSARegId(BaseReg::kIdBad); } + + ASMJIT_INLINE_NODEBUG void resetSARegId() { setSARegId(Reg::kIdBad); } //! Returns stack size required to save/restore registers via push/pop. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t pushPopSaveSize() const noexcept { return _pushPopSaveSize; } + //! Returns an offset to the stack where registers are saved via push/pop. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t pushPopSaveOffset() const noexcept { return _pushPopSaveOffset; } //! Returns stack size required to save/restore extra registers that don't use push/pop/ //! //! \note On X86 this covers all registers except GP registers, on other architectures it can be always //! zero (for example AArch64 saves all registers via push/pop like instructions, so this would be zero). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t extraRegSaveSize() const noexcept { return _extraRegSaveSize; } + //! Returns an offset to the stack where extra registers are saved. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t extraRegSaveOffset() const noexcept { return _extraRegSaveOffset; } //! Tests whether the functions contains stack adjustment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasStackAdjustment() const noexcept { return _stackAdjustment != 0; } + //! Returns function's stack adjustment used in function's prolog and epilog. //! //! If the returned value is zero it means that the stack is not adjusted. This can mean both that the stack //! is not used and/or the stack is only adjusted by instructions that pust/pop registers into/from stack. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t stackAdjustment() const noexcept { return _stackAdjustment; } //! \} @@ -1332,86 +1692,121 @@ public: //! \{ //! Function detail. - const FuncDetail* _funcDetail; + const FuncDetail* _funcDetail {}; //! Register that can be used to access arguments passed by stack. - uint8_t _saRegId; + uint8_t _saRegId = uint8_t(Reg::kIdBad); //! Reserved for future use. - uint8_t _reserved[3]; + uint8_t _reserved[3] {}; //! Mapping of each function argument. - FuncValuePack _argPacks[Globals::kMaxFuncArgs]; + FuncValuePack _argPacks[Globals::kMaxFuncArgs] {}; //! \} //! \name Construction & Destruction //! \{ + //! Creates either a default initialized `FuncArgsAssignment` or to assignment that links to `fd`, if non-null. ASMJIT_INLINE_NODEBUG explicit FuncArgsAssignment(const FuncDetail* fd = nullptr) noexcept { reset(fd); } - ASMJIT_INLINE_NODEBUG FuncArgsAssignment(const FuncArgsAssignment& other) noexcept { - memcpy(this, &other, sizeof(*this)); - } + //! Copy constructor. + ASMJIT_INLINE_NODEBUG FuncArgsAssignment(const FuncArgsAssignment& other) noexcept = default; + //! Resets this `FuncArgsAssignment` to either default constructed state or to assignment that links to `fd`, + //! if non-null. inline void reset(const FuncDetail* fd = nullptr) noexcept { _funcDetail = fd; - _saRegId = uint8_t(BaseReg::kIdBad); + _saRegId = uint8_t(Reg::kIdBad); memset(_reserved, 0, sizeof(_reserved)); memset(_argPacks, 0, sizeof(_argPacks)); } //! \} + //! \name Overloaded Operators + //! \{ + + //! Copy assignment. + ASMJIT_INLINE_NODEBUG FuncArgsAssignment& operator=(const FuncArgsAssignment& other) noexcept = default; + + //! \} + //! \name Accessors //! \{ + //! Returns the associated \ref FuncDetail of this `FuncArgsAssignment`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FuncDetail* funcDetail() const noexcept { return _funcDetail; } + + //! Associates \ref FuncDetails with this `FuncArgsAssignment`. ASMJIT_INLINE_NODEBUG void setFuncDetail(const FuncDetail* fd) noexcept { _funcDetail = fd; } - ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } - ASMJIT_INLINE_NODEBUG uint32_t saRegId() const noexcept { return _saRegId; } - ASMJIT_INLINE_NODEBUG void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } - ASMJIT_INLINE_NODEBUG void resetSARegId() { _saRegId = uint8_t(BaseReg::kIdBad); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasSARegId() const noexcept { return _saRegId != Reg::kIdBad; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t saRegId() const noexcept { return _saRegId; } + + ASMJIT_INLINE_NODEBUG void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } + + ASMJIT_INLINE_NODEBUG void resetSARegId() { _saRegId = uint8_t(Reg::kIdBad); } + + //! Returns assigned argument at `argIndex` and `valueIndex`. + //! + //! \note `argIndex` refers to he function argument and `valueIndex` refers to a value pack (in case multiple + //! values are passed as a single argument). + [[nodiscard]] inline FuncValue& arg(size_t argIndex, size_t valueIndex) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); return _argPacks[argIndex][valueIndex]; } + + //! \overload + [[nodiscard]] inline const FuncValue& arg(size_t argIndex, size_t valueIndex) const noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); return _argPacks[argIndex][valueIndex]; } + //! Tests whether argument at `argIndex` and `valueIndex` has been assigned. + [[nodiscard]] inline bool isAssigned(size_t argIndex, size_t valueIndex) const noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); return _argPacks[argIndex][valueIndex].isAssigned(); } - inline void assignReg(size_t argIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + //! Assigns register at `argIndex` and value index of 0 to `reg` and an optional `typeId`. + inline void assignReg(size_t argIndex, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); ASMJIT_ASSERT(reg.isPhysReg()); - _argPacks[argIndex][0].initReg(reg.type(), reg.id(), typeId); + _argPacks[argIndex][0].initReg(reg.regType(), reg.id(), typeId); } + //! Assigns register at `argIndex` and value index of 0 to `regType`, `regId`, and an optional `typeId`. inline void assignReg(size_t argIndex, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); _argPacks[argIndex][0].initReg(regType, regId, typeId); } + //! Assigns stack at `argIndex` and value index of 0 to `offset` and an optional `typeId`. inline void assignStack(size_t argIndex, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); _argPacks[argIndex][0].initStack(offset, typeId); } - inline void assignRegInPack(size_t argIndex, size_t valueIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + //! Assigns register at `argIndex` and `valueIndex` to `reg` and an optional `typeId`. + inline void assignRegInPack(size_t argIndex, size_t valueIndex, const Reg& reg, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); ASMJIT_ASSERT(reg.isPhysReg()); - _argPacks[argIndex][valueIndex].initReg(reg.type(), reg.id(), typeId); + _argPacks[argIndex][valueIndex].initReg(reg.regType(), reg.id(), typeId); } + //! Assigns register at `argIndex` and `valueIndex` to `regType`, `regId`, and an optional `typeId`. inline void assignRegInPack(size_t argIndex, size_t valueIndex, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); _argPacks[argIndex][valueIndex].initReg(regType, regId, typeId); } + //! Assigns stack at `argIndex` and `valueIndex` to `offset` and an optional `typeId`. inline void assignStackInPack(size_t argIndex, size_t valueIndex, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); _argPacks[argIndex][valueIndex].initStack(offset, typeId); @@ -1420,16 +1815,19 @@ public: // NOTE: All `assignAll()` methods are shortcuts to assign all arguments at once, however, since registers are // passed all at once these initializers don't provide any way to pass TypeId and/or to keep any argument between // the arguments passed unassigned. - inline void _assignAllInternal(size_t argIndex, const BaseReg& reg) noexcept { + inline void _assignAllInternal(size_t argIndex, const Reg& reg) noexcept { assignReg(argIndex, reg); } template - inline void _assignAllInternal(size_t argIndex, const BaseReg& reg, Args&&... args) noexcept { + inline void _assignAllInternal(size_t argIndex, const Reg& reg, Args&&... args) noexcept { assignReg(argIndex, reg); _assignAllInternal(argIndex + 1, std::forward(args)...); } + //! Assigns all argument at once. + //! + //! \note This function can be only used if the arguments don't contain value packs (multiple values per argument). template inline void assignAll(Args&&... args) noexcept { _assignAllInternal(0, std::forward(args)...); @@ -1442,8 +1840,8 @@ public: //! Update `FuncFrame` based on function's arguments assignment. //! - //! \note You MUST call this in order to use `BaseEmitter::emitArgsAssignment()`, otherwise the FuncFrame would - //! not contain the information necessary to assign all arguments into the registers and/or stack specified. + //! \note This function must be called in order to use `BaseEmitter::emitArgsAssignment()`, otherwise the \ref FuncFrame + //! would not contain the information necessary to assign all arguments into the registers and/or stack specified. ASMJIT_API Error updateFuncFrame(FuncFrame& frame) const noexcept; //! \} @@ -1454,4 +1852,3 @@ public: ASMJIT_END_NAMESPACE #endif // ASMJIT_CORE_FUNC_H_INCLUDED - diff --git a/pe-packer/asmjit/core/funcargscontext.cpp b/pe-packer/asmjit/core/funcargscontext.cpp index 1db50a7..f01e685 100644 --- a/pe-packer/asmjit/core/funcargscontext.cpp +++ b/pe-packer/asmjit/core/funcargscontext.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -13,8 +13,9 @@ ASMJIT_BEGIN_NAMESPACE //! \{ FuncArgsContext::FuncArgsContext() noexcept { - for (RegGroup group : RegGroupVirtValues{}) + for (RegGroup group : RegGroupVirtValues{}) { _workData[size_t(group)].reset(); + } } ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args, const RAConstraints* constraints) noexcept { @@ -26,23 +27,29 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co _arch = arch; // Initialize `_archRegs`. - for (RegGroup group : RegGroupVirtValues{}) + for (RegGroup group : RegGroupVirtValues{}) { _workData[group]._archRegs = _constraints->availableRegs(group); + } - if (frame.hasPreservedFP()) + if (frame.hasPreservedFP()) { _workData[size_t(RegGroup::kGp)]._archRegs &= ~Support::bitMask(archTraits().fpRegId()); + } + + uint32_t reassignmentFlagMask = 0; // Extract information from all function arguments/assignments and build Var[] array. uint32_t varId = 0; for (uint32_t argIndex = 0; argIndex < Globals::kMaxFuncArgs; argIndex++) { for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) { const FuncValue& dst_ = args.arg(argIndex, valueIndex); - if (!dst_.isAssigned()) + if (!dst_.isAssigned()) { continue; + } const FuncValue& src_ = func.arg(argIndex, valueIndex); - if (ASMJIT_UNLIKELY(!src_.isAssigned())) + if (ASMJIT_UNLIKELY(!src_.isAssigned())) { return DebugUtils::errored(kErrorInvalidState); + } Var& var = _vars[varId]; var.init(src_, dst_); @@ -51,67 +58,95 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co FuncValue& dst = var.out; RegGroup dstGroup = RegGroup::kMaxValue; - uint32_t dstId = BaseReg::kIdBad; + uint32_t dstId = Reg::kIdBad; WorkData* dstWd = nullptr; // Not supported. - if (src.isIndirect()) + if (src.isIndirect()) { return DebugUtils::errored(kErrorInvalidAssignment); + } if (dst.isReg()) { RegType dstType = dst.regType(); - if (ASMJIT_UNLIKELY(!archTraits().hasRegType(dstType))) + if (ASMJIT_UNLIKELY(!archTraits().hasRegType(dstType))) { return DebugUtils::errored(kErrorInvalidRegType); + } // Copy TypeId from source if the destination doesn't have it. The RA used by BaseCompiler would never - // leave TypeId undefined, but users of FuncAPI can just assign phys regs without specifying the type. - if (!dst.hasTypeId()) - dst.setTypeId(archTraits().regTypeToTypeId(dst.regType())); + // leave TypeId undefined, but users of FuncAPI can just assign phys regs without specifying their types. + if (!dst.hasTypeId()) { + dst.setTypeId(RegUtils::typeIdOf(dst.regType())); + } - dstGroup = archTraits().regTypeToGroup(dstType); - if (ASMJIT_UNLIKELY(dstGroup > RegGroup::kMaxVirt)) + dstGroup = RegUtils::groupOf(dstType); + if (ASMJIT_UNLIKELY(dstGroup > RegGroup::kMaxVirt)) { return DebugUtils::errored(kErrorInvalidRegGroup); + } dstWd = &_workData[dstGroup]; dstId = dst.regId(); - if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId))) - return DebugUtils::errored(kErrorInvalidPhysId); - if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId))) + if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId))) { + return DebugUtils::errored(kErrorInvalidPhysId); + } + + if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId))) { return DebugUtils::errored(kErrorOverlappedRegs); + } dstWd->_dstRegs |= Support::bitMask(dstId); dstWd->_dstShuf |= Support::bitMask(dstId); dstWd->_usedRegs |= Support::bitMask(dstId); } else { - if (!dst.hasTypeId()) + if (!dst.hasTypeId()) { dst.setTypeId(src.typeId()); + } OperandSignature signature = getSuitableRegForMemToMemMove(arch, dst.typeId(), src.typeId()); - if (ASMJIT_UNLIKELY(!signature.isValid())) + if (ASMJIT_UNLIKELY(!signature.isValid())) { return DebugUtils::errored(kErrorInvalidState); + } _stackDstMask = uint8_t(_stackDstMask | Support::bitMask(signature.regGroup())); } if (src.isReg()) { uint32_t srcId = src.regId(); - RegGroup srcGroup = archTraits().regTypeToGroup(src.regType()); + RegGroup srcGroup = RegUtils::groupOf(src.regType()); if (dstGroup == srcGroup) { ASMJIT_ASSERT(dstWd != nullptr); dstWd->assign(varId, srcId); - // The best case, register is allocated where it is expected to be. - if (dstId == srcId) - var.markDone(); + reassignmentFlagMask |= uint32_t(dstId != srcId) << uint32_t(dstGroup); + + if (dstId == srcId) { + // The best case, register is allocated where it is expected to be. However, we should + // not mark this as done if both registers are GP and sign or zero extension is required. + if (dstGroup != RegGroup::kGp) { + var.markDone(); + } + else { + TypeId dt = dst.typeId(); + TypeId st = src.typeId(); + + uint32_t dstSize = TypeUtils::sizeOf(dt); + uint32_t srcSize = TypeUtils::sizeOf(st); + + if (dt == TypeId::kVoid || st == TypeId::kVoid || dstSize <= srcSize) { + var.markDone(); + } + } + } } else { - if (ASMJIT_UNLIKELY(srcGroup > RegGroup::kMaxVirt)) + if (ASMJIT_UNLIKELY(srcGroup > RegGroup::kMaxVirt)) { return DebugUtils::errored(kErrorInvalidState); + } WorkData& srcData = _workData[size_t(srcGroup)]; srcData.assign(varId, srcId); + reassignmentFlagMask |= 1u << uint32_t(dstGroup); } } else { @@ -128,6 +163,7 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co for (RegGroup group : RegGroupVirtValues{}) { _workData[group]._workRegs = (_workData[group].archRegs() & (frame.dirtyRegs(group) | ~frame.preservedRegs(group))) | _workData[group].dstRegs() | _workData[group].assignedRegs(); + _workData[group]._needsScratch = (reassignmentFlagMask >> uint32_t(group)) & 1u; } // Create a variable that represents `SARegId` if necessary. @@ -137,16 +173,18 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co uint32_t saCurRegId = frame.saRegId(); uint32_t saOutRegId = args.saRegId(); - if (saCurRegId != BaseReg::kIdBad) { + if (saCurRegId != Reg::kIdBad) { // Check if the provided `SARegId` doesn't collide with input registers. - if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId))) + if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId))) { return DebugUtils::errored(kErrorOverlappedRegs); + } } - if (saOutRegId != BaseReg::kIdBad) { + if (saOutRegId != Reg::kIdBad) { // Check if the provided `SARegId` doesn't collide with argument assignments. - if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId))) + if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId))) { return DebugUtils::errored(kErrorOverlappedRegs); + } saRegRequired = true; } @@ -160,17 +198,19 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co Var& var = _vars[varId]; var.reset(); - if (saCurRegId == BaseReg::kIdBad) { - if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { + if (saCurRegId == Reg::kIdBad) { + if (saOutRegId != Reg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { saCurRegId = saOutRegId; } else { RegMask availableRegs = gpRegs.availableRegs(); - if (!availableRegs) + if (!availableRegs) { availableRegs = gpRegs.archRegs() & ~gpRegs.workRegs(); + } - if (ASMJIT_UNLIKELY(!availableRegs)) + if (ASMJIT_UNLIKELY(!availableRegs)) { return DebugUtils::errored(kErrorNoMorePhysRegs); + } saCurRegId = Support::ctz(availableRegs); } @@ -180,7 +220,7 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co gpRegs.assign(varId, saCurRegId); gpRegs._workRegs |= Support::bitMask(saCurRegId); - if (saOutRegId != BaseReg::kIdBad) { + if (saOutRegId != Reg::kIdBad) { var.out.initReg(ptrRegType, saOutRegId, ptrTypeId); gpRegs._dstRegs |= Support::bitMask(saOutRegId); gpRegs._workRegs |= Support::bitMask(saOutRegId); @@ -201,14 +241,15 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, co uint32_t srcId = var.cur.regId(); uint32_t dstId = var.out.regId(); - RegGroup group = archTraits().regTypeToGroup(var.cur.regType()); - if (group != archTraits().regTypeToGroup(var.out.regType())) + RegGroup group = RegUtils::groupOf(var.cur.regType()); + if (group != RegUtils::groupOf(var.out.regType())) { continue; + } WorkData& wd = _workData[group]; if (wd.isAssigned(dstId)) { Var& other = _vars[wd._physToVarId[dstId]]; - if (archTraits().regTypeToGroup(other.out.regType()) == group && other.out.regId() == srcId) { + if (RegUtils::groupOf(other.out.regType()) == group && other.out.regId() == srcId) { wd._numSwaps++; _regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group)); } @@ -247,28 +288,32 @@ ASMJIT_FAVOR_SIZE Error FuncArgsContext::markScratchRegs(FuncFrame& frame) noexc for (RegGroup group : RegGroupVirtValues{}) { if (Support::bitTest(groupMask, group)) { WorkData& wd = _workData[group]; + if (wd._needsScratch) { + // Initially, pick some clobbered or dirty register. + RegMask workRegs = wd.workRegs(); + RegMask regs = workRegs & ~(wd.usedRegs() | wd._dstShuf); - // Initially, pick some clobbered or dirty register. - RegMask workRegs = wd.workRegs(); - RegMask regs = workRegs & ~(wd.usedRegs() | wd._dstShuf); + // If that didn't work out pick some register which is not in 'used'. + if (!regs) { + regs = workRegs & ~wd.usedRegs(); + } - // If that didn't work out pick some register which is not in 'used'. - if (!regs) - regs = workRegs & ~wd.usedRegs(); + // If that didn't work out pick any other register that is allocable. + // This last resort case will, however, result in marking one more + // register dirty. + if (!regs) { + regs = wd.archRegs() & ~workRegs; + } - // If that didn't work out pick any other register that is allocable. - // This last resort case will, however, result in marking one more - // register dirty. - if (!regs) - regs = wd.archRegs() & ~workRegs; + // If that didn't work out we will have to use XORs instead of MOVs. + if (!regs) { + continue; + } - // If that didn't work out we will have to use XORs instead of MOVs. - if (!regs) - continue; - - RegMask regMask = Support::blsi(regs); - wd._workRegs |= regMask; - frame.addDirtyRegs(group, regMask); + RegMask regMask = Support::blsi(regs); + wd._workRegs |= regMask; + frame.addDirtyRegs(group, regMask); + } } } diff --git a/pe-packer/asmjit/core/funcargscontext_p.h b/pe-packer/asmjit/core/funcargscontext_p.h index a8ad6b4..bcfc784 100644 --- a/pe-packer/asmjit/core/funcargscontext_p.h +++ b/pe-packer/asmjit/core/funcargscontext_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_FUNCARGSCONTEXT_P_H_INCLUDED @@ -22,32 +22,35 @@ ASMJIT_BEGIN_NAMESPACE static inline OperandSignature getSuitableRegForMemToMemMove(Arch arch, TypeId dstTypeId, TypeId srcTypeId) noexcept { const ArchTraits& archTraits = ArchTraits::byArch(arch); + uint32_t signature = 0u; uint32_t dstSize = TypeUtils::sizeOf(dstTypeId); uint32_t srcSize = TypeUtils::sizeOf(srcTypeId); uint32_t maxSize = Support::max(dstSize, srcSize); uint32_t regSize = Environment::registerSizeFromArch(arch); - OperandSignature signature{0}; - if (maxSize <= regSize || (TypeUtils::isInt(dstTypeId) && TypeUtils::isInt(srcTypeId))) - signature = maxSize <= 4 ? archTraits.regTypeToSignature(RegType::kGp32) - : archTraits.regTypeToSignature(RegType::kGp64); - else if (maxSize <= 8 && archTraits.hasRegType(RegType::kVec64)) - signature = archTraits.regTypeToSignature(RegType::kVec64); - else if (maxSize <= 16 && archTraits.hasRegType(RegType::kVec128)) - signature = archTraits.regTypeToSignature(RegType::kVec128); - else if (maxSize <= 32 && archTraits.hasRegType(RegType::kVec256)) - signature = archTraits.regTypeToSignature(RegType::kVec256); - else if (maxSize <= 64 && archTraits.hasRegType(RegType::kVec512)) - signature = archTraits.regTypeToSignature(RegType::kVec512); + if (maxSize <= regSize || (TypeUtils::isInt(dstTypeId) && TypeUtils::isInt(srcTypeId))) { + signature = maxSize <= 4 ? RegTraits::kSignature + : RegTraits::kSignature; + } + else if (maxSize <= 8 && archTraits.hasRegType(RegType::kVec64)) { + signature = RegTraits::kSignature; + } + else if (maxSize <= 16 && archTraits.hasRegType(RegType::kVec128)) { + signature = RegTraits::kSignature; + } + else if (maxSize <= 32 && archTraits.hasRegType(RegType::kVec256)) { + signature = RegTraits::kSignature; + } + else if (maxSize <= 64 && archTraits.hasRegType(RegType::kVec512)) { + signature = RegTraits::kSignature; + } - return signature; + return OperandSignature{signature}; } class FuncArgsContext { public: - enum VarId : uint32_t { - kVarIdNone = 0xFF - }; + static inline constexpr uint32_t kVarIdNone = 0xFF; //! Contains information about a single argument or SA register that may need shuffling. struct Var { @@ -65,8 +68,8 @@ public: out.reset(); } - inline bool isDone() const noexcept { return cur.isDone(); } - inline void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); } + ASMJIT_INLINE_NODEBUG bool isDone() const noexcept { return cur.isDone(); } + ASMJIT_INLINE_NODEBUG void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); } }; struct WorkData { @@ -86,8 +89,10 @@ public: uint8_t _numSwaps; //! Number of stack loads. uint8_t _numStackArgs; + //! Whether this work data would need reassignment. + uint8_t _needsScratch; //! Reserved (only used as padding). - uint8_t _reserved[6]; + uint8_t _reserved[5]; //! Physical ID to variable ID mapping. uint8_t _physToVarId[32]; @@ -100,10 +105,12 @@ public: _dstShuf = 0; _numSwaps = 0; _numStackArgs = 0; + _needsScratch = 0; memset(_reserved, 0, sizeof(_reserved)); memset(_physToVarId, kVarIdNone, 32); } + [[nodiscard]] inline bool isAssigned(uint32_t regId) const noexcept { ASMJIT_ASSERT(regId < 32); return Support::bitTest(_assignedRegs, regId); @@ -147,12 +154,23 @@ public: _assignedRegs ^= Support::bitMask(regId); } - inline RegMask archRegs() const noexcept { return _archRegs; } - inline RegMask workRegs() const noexcept { return _workRegs; } - inline RegMask usedRegs() const noexcept { return _usedRegs; } - inline RegMask assignedRegs() const noexcept { return _assignedRegs; } - inline RegMask dstRegs() const noexcept { return _dstRegs; } - inline RegMask availableRegs() const noexcept { return _workRegs & ~_assignedRegs; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask archRegs() const noexcept { return _archRegs; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask workRegs() const noexcept { return _workRegs; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask usedRegs() const noexcept { return _usedRegs; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask assignedRegs() const noexcept { return _assignedRegs; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask dstRegs() const noexcept { return _dstRegs; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG RegMask availableRegs() const noexcept { return _workRegs & ~_assignedRegs; } }; //! Architecture traits. @@ -176,13 +194,22 @@ public: FuncArgsContext() noexcept; + [[nodiscard]] ASMJIT_INLINE_NODEBUG const ArchTraits& archTraits() const noexcept { return *_archTraits; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG Arch arch() const noexcept { return _arch; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t varCount() const noexcept { return _varCount; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t indexOf(const Var* var) const noexcept { return (size_t)(var - _vars); } + [[nodiscard]] ASMJIT_INLINE_NODEBUG Var& var(size_t varId) noexcept { return _vars[varId]; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const Var& var(size_t varId) const noexcept { return _vars[varId]; } Error initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args, const RAConstraints* constraints) noexcept; diff --git a/pe-packer/asmjit/core/globals.cpp b/pe-packer/asmjit/core/globals.cpp index 1fbb399..e344882 100644 --- a/pe-packer/asmjit/core/globals.cpp +++ b/pe-packer/asmjit/core/globals.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -87,6 +87,7 @@ ASMJIT_FAVOR_SIZE const char* DebugUtils::errorAsString(Error err) noexcept { "ExpressionOverflow\0" "FailedToOpenAnonymousMemory\0" "FailedToOpenFile\0" + "ProtectionFailure\0" "\0"; static const uint16_t sErrorIndex[] = { @@ -94,7 +95,7 @@ ASMJIT_FAVOR_SIZE const char* DebugUtils::errorAsString(Error err) noexcept { 247, 264, 283, 298, 314, 333, 352, 370, 392, 410, 429, 444, 460, 474, 488, 508, 533, 551, 573, 595, 612, 629, 645, 661, 677, 694, 709, 724, 744, 764, 784, 817, 837, 852, 869, 888, 909, 929, 943, 964, 978, 996, 1012, 1028, 1047, - 1073, 1088, 1104, 1119, 1134, 1164, 1188, 1207, 1235, 1252 + 1073, 1088, 1104, 1119, 1134, 1164, 1188, 1207, 1235, 1252, 1270 }; // @EnumStringEnd@ diff --git a/pe-packer/asmjit/core/globals.h b/pe-packer/asmjit/core/globals.h index 1dfc7de..f5e46ef 100644 --- a/pe-packer/asmjit/core/globals.h +++ b/pe-packer/asmjit/core/globals.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_GLOBALS_H_INCLUDED @@ -14,28 +14,31 @@ ASMJIT_BEGIN_NAMESPACE //! \addtogroup asmjit_utilities //! \{ namespace Support { - //! Cast designed to cast between function and void* pointers. - template - static inline Dst ptr_cast_impl(Src p) noexcept { return (Dst)p; } + +//! Cast designed to cast between function and void* pointers. +template +static inline Dst ptr_cast_impl(Src p) noexcept { return (Dst)p; } + +//! Helper to implement placement new/delete without relying on `` header. +struct PlacementNew { void* ptr; }; + } // {Support} #if defined(ASMJIT_NO_STDCXX) namespace Support { - ASMJIT_FORCE_INLINE void* operatorNew(size_t n) noexcept { return malloc(n); } - ASMJIT_FORCE_INLINE void operatorDelete(void* p) noexcept { if (p) free(p); } + ASMJIT_INLINE void* operatorNew(size_t n) noexcept { return malloc(n); } + ASMJIT_INLINE void operatorDelete(void* p) noexcept { if (p) free(p); } } // {Support} -#define ASMJIT_BASE_CLASS(TYPE) \ - ASMJIT_FORCE_INLINE void* operator new(size_t n) noexcept { \ - return Support::operatorNew(n); \ - } \ - \ - ASMJIT_FORCE_INLINE void operator delete(void* p) noexcept { \ - Support::operatorDelete(p); \ - } \ - \ - ASMJIT_FORCE_INLINE void* operator new(size_t, void* p) noexcept { return p; } \ - ASMJIT_FORCE_INLINE void operator delete(void*, void*) noexcept {} +#define ASMJIT_BASE_CLASS(TYPE) \ + ASMJIT_INLINE void* operator new(size_t n) noexcept { return Support::operatorNew(n); } \ + ASMJIT_INLINE void operator delete(void* ptr) noexcept { Support::operatorDelete(ptr); } \ + \ + ASMJIT_INLINE void* operator new(size_t, void* ptr) noexcept { return ptr; } \ + ASMJIT_INLINE void operator delete(void*, void*) noexcept {} \ + \ + ASMJIT_INLINE void* operator new(size_t, Support::PlacementNew ptr) noexcept { return ptr.ptr; } \ + ASMJIT_INLINE void operator delete(void*, Support::PlacementNew) noexcept {} #else #define ASMJIT_BASE_CLASS(TYPE) #endif @@ -66,17 +69,20 @@ enum class ResetPolicy : uint32_t { kHard = 1 }; -//! Contains typedefs, constants, and variables used globally by AsmJit. +//! Contains constants and variables used globally across AsmJit. namespace Globals { //! Host memory allocator overhead. -static constexpr uint32_t kAllocOverhead = uint32_t(sizeof(intptr_t) * 4); +static constexpr uint32_t kAllocOverhead = uint32_t(sizeof(intptr_t) * 4u); //! Host memory allocator alignment. -static constexpr uint32_t kAllocAlignment = 8; +static constexpr uint32_t kAllocAlignment = 8u; //! Aggressive growing strategy threshold. -static constexpr uint32_t kGrowThreshold = 1024 * 1024 * 16; +static constexpr uint32_t kGrowThreshold = 1024u * 1024u * 16u; + +//! Default alignment of allocation requests to use when using Zone. +static constexpr uint32_t kZoneAlignment = 8u; //! Maximum depth of RB-Tree is: //! @@ -93,10 +99,9 @@ static constexpr uint32_t kMaxTreeHeight = (ASMJIT_ARCH_BITS == 32 ? 30 : 61) + static constexpr uint32_t kMaxOpCount = 6; //! Maximum arguments of a function supported by the Compiler / Function API. -static constexpr uint32_t kMaxFuncArgs = 16; +static constexpr uint32_t kMaxFuncArgs = 32; -//! The number of values that can be assigned to a single function argument or -//! return value. +//! The number of values that can be assigned to a single function argument or return value. static constexpr uint32_t kMaxValuePack = 4; //! Maximum number of physical registers AsmJit can use per register group. @@ -129,14 +134,18 @@ static constexpr uint32_t kNumVirtGroups = 4; struct Init_ {}; struct NoInit_ {}; +//! A decorator used to initialize. static const constexpr Init_ Init {}; +//! A decorator used to not initialize. static const constexpr NoInit_ NoInit {}; } // {Globals} +//! Casts a `void*` pointer `func` to a function pointer `Func`. template static ASMJIT_INLINE_NODEBUG Func ptr_as_func(void* func) noexcept { return Support::ptr_cast_impl(func); } +//! Casts a function pointer `func` to a void pointer `void*`. template static ASMJIT_INLINE_NODEBUG void* func_as_ptr(Func func) noexcept { return Support::ptr_cast_impl(func); } @@ -146,7 +155,7 @@ static ASMJIT_INLINE_NODEBUG void* func_as_ptr(Func func) noexcept { return Supp //! \{ //! AsmJit error type (uint32_t). -typedef uint32_t Error; +using Error = uint32_t; //! AsmJit error codes. enum ErrorCode : uint32_t { @@ -206,8 +215,8 @@ enum ErrorCode : uint32_t { kErrorLabelNameTooLong, //! Label must always be local if it's anonymous (without a name). kErrorInvalidLabelName, - //! Parent id passed to \ref CodeHolder::newNamedLabelEntry() was either invalid or parent is not supported - //! by the requested `LabelType`. + //! Parent id passed to \ref CodeHolder::newNamedLabelId() was either invalid or parent is not supported by + //! the requested `LabelType`. kErrorInvalidParentLabel, //! Invalid section. @@ -272,7 +281,7 @@ enum ErrorCode : uint32_t { kErrorInvalidAddress64BitZeroExtension, //! Invalid displacement (not encodable). kErrorInvalidDisplacement, - //! Invalid segment (X86). + //! Invalid segment (X86|X86_64). kErrorInvalidSegment, //! Invalid immediate (out of bounds on X86 and invalid pattern on ARM). @@ -328,6 +337,10 @@ enum ErrorCode : uint32_t { //! \note This is a generic error that is used by internal filesystem API. kErrorFailedToOpenFile, + //! Protection failure can be returned from a virtual memory allocator or when trying to change memory access + //! permissions. + kErrorProtectionFailure, + // @EnumValuesEnd@ //! Count of AsmJit error codes. @@ -347,9 +360,11 @@ static ASMJIT_INLINE_NODEBUG void unused(Args&&...) noexcept {} //! //! Provided for debugging purposes. Putting a breakpoint inside `errored` can help with tracing the origin of any //! error reported / returned by AsmJit. +[[nodiscard]] static constexpr Error errored(Error err) noexcept { return err; } //! Returns a printable version of `asmjit::Error` code. +[[nodiscard]] ASMJIT_API const char* errorAsString(Error err) noexcept; //! Called to output debugging message(s). @@ -365,7 +380,8 @@ ASMJIT_API void debugOutput(const char* str) noexcept; //! (asmjit/core/globals.cpp). A call stack will be available when such assertion failure is triggered. AsmJit //! always returns errors on failures, assertions are a last resort and usually mean unrecoverable state due to out //! of range array access or totally invalid arguments like nullptr where a valid pointer should be provided, etc... -ASMJIT_API void ASMJIT_NORETURN assertionFailed(const char* file, int line, const char* msg) noexcept; +[[noreturn]] +ASMJIT_API void assertionFailed(const char* file, int line, const char* msg) noexcept; } // {DebugUtils} @@ -373,14 +389,32 @@ ASMJIT_API void ASMJIT_NORETURN assertionFailed(const char* file, int line, cons //! //! AsmJit's own assert macro used in AsmJit code-base. #if defined(ASMJIT_BUILD_DEBUG) -#define ASMJIT_ASSERT(...) \ - do { \ - if (ASMJIT_LIKELY(__VA_ARGS__)) \ - break; \ - ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ - } while (0) + #define ASMJIT_ASSERT(...) \ + do { \ + if (ASMJIT_UNLIKELY(!(__VA_ARGS__))) { \ + ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ + } \ + } while (0) #else -#define ASMJIT_ASSERT(...) ((void)0) + #define ASMJIT_ASSERT(...) ((void)0) +#endif + +#define ASMJIT_RUNTIME_ASSERT(...) \ + do { \ + if (ASMJIT_UNLIKELY(!(__VA_ARGS__))) { \ + ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ + } \ + } while (0) + +//! \def ASMJIT_NOT_REACHED() +//! +//! Run-time assertion used in code that should never be reached. +#if defined(ASMJIT_BUILD_DEBUG) + #define ASMJIT_NOT_REACHED() ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, "ASMJIT_NOT_REACHED()") +#elif defined(__GNUC__) + #define ASMJIT_NOT_REACHED() __builtin_unreachable() +#else + #define ASMJIT_NOT_REACHED() ASMJIT_ASSUME(0) #endif //! \def ASMJIT_PROPAGATE(...) @@ -389,13 +423,24 @@ ASMJIT_API void ASMJIT_NORETURN assertionFailed(const char* file, int line, cons //! internally, but kept public for users that want to use the same technique to propagate errors to the caller. #define ASMJIT_PROPAGATE(...) \ do { \ - ::asmjit::Error _err = __VA_ARGS__; \ - if (ASMJIT_UNLIKELY(_err)) \ - return _err; \ + ::asmjit::Error _err_ = __VA_ARGS__; \ + if (ASMJIT_UNLIKELY(_err_)) { \ + return _err_; \ + } \ } while (0) //! \} ASMJIT_END_NAMESPACE +//! Implementation of a placement new so we don't have to depend on ``. +ASMJIT_INLINE_NODEBUG void* operator new(size_t, const asmjit::Support::PlacementNew& p) noexcept { +#if defined(_MSC_VER) && !defined(__clang__) + __assume(p.ptr != nullptr); // Otherwise MSVC would emit a nullptr check. +#endif + return p.ptr; +} + +ASMJIT_INLINE_NODEBUG void operator delete(void*, const asmjit::Support::PlacementNew&) noexcept {} + #endif // ASMJIT_CORE_GLOBALS_H_INCLUDED diff --git a/pe-packer/asmjit/core/inst.cpp b/pe-packer/asmjit/core/inst.cpp index 8f29d8b..7acb4f2 100644 --- a/pe-packer/asmjit/core/inst.cpp +++ b/pe-packer/asmjit/core/inst.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -21,15 +21,17 @@ ASMJIT_BEGIN_NAMESPACE // =========================== #ifndef ASMJIT_NO_TEXT -Error InstAPI::instIdToString(Arch arch, InstId instId, String& output) noexcept { +Error InstAPI::instIdToString(Arch arch, InstId instId, InstStringifyOptions options, String& output) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) - return x86::InstInternal::instIdToString(arch, instId, output); + if (Environment::isFamilyX86(arch)) { + return x86::InstInternal::instIdToString(instId, options, output); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::InstInternal::instIdToString(arch, instId, output); + if (Environment::isFamilyAArch64(arch)) { + return a64::InstInternal::instIdToString(instId, options, output); + } #endif return DebugUtils::errored(kErrorInvalidArch); @@ -37,13 +39,15 @@ Error InstAPI::instIdToString(Arch arch, InstId instId, String& output) noexcept InstId InstAPI::stringToInstId(Arch arch, const char* s, size_t len) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) - return x86::InstInternal::stringToInstId(arch, s, len); + if (Environment::isFamilyX86(arch)) { + return x86::InstInternal::stringToInstId(s, len); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::InstInternal::stringToInstId(arch, s, len); + if (Environment::isFamilyAArch64(arch)) { + return a64::InstInternal::stringToInstId(s, len); + } #endif return 0; @@ -56,13 +60,20 @@ InstId InstAPI::stringToInstId(Arch arch, const char* s, size_t len) noexcept { #ifndef ASMJIT_NO_VALIDATION Error InstAPI::validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) - return x86::InstInternal::validate(arch, inst, operands, opCount, validationFlags); + if (Environment::isFamilyX86(arch)) { + if (arch == Arch::kX86) { + return x86::InstInternal::validateX86(inst, operands, opCount, validationFlags); + } + else { + return x86::InstInternal::validateX64(inst, operands, opCount, validationFlags); + } + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::InstInternal::validate(arch, inst, operands, opCount, validationFlags); + if (Environment::isFamilyAArch64(arch)) { + return a64::InstInternal::validate(inst, operands, opCount, validationFlags); + } #endif return DebugUtils::errored(kErrorInvalidArch); @@ -74,17 +85,20 @@ Error InstAPI::validate(Arch arch, const BaseInst& inst, const Operand_* operand #ifndef ASMJIT_NO_INTROSPECTION Error InstAPI::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept { - if (ASMJIT_UNLIKELY(opCount > Globals::kMaxOpCount)) + if (ASMJIT_UNLIKELY(opCount > Globals::kMaxOpCount)) { return DebugUtils::errored(kErrorInvalidArgument); + } #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::InstInternal::queryRWInfo(arch, inst, operands, opCount, out); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::InstInternal::queryRWInfo(arch, inst, operands, opCount, out); + if (Environment::isFamilyAArch64(arch)) { + return a64::InstInternal::queryRWInfo(inst, operands, opCount, out); + } #endif return DebugUtils::errored(kErrorInvalidArch); @@ -97,13 +111,15 @@ Error InstAPI::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* oper #ifndef ASMJIT_NO_INTROSPECTION Error InstAPI::queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept { #if !defined(ASMJIT_NO_X86) - if (Environment::isFamilyX86(arch)) + if (Environment::isFamilyX86(arch)) { return x86::InstInternal::queryFeatures(arch, inst, operands, opCount, out); + } #endif #if !defined(ASMJIT_NO_AARCH64) - if (Environment::isFamilyAArch64(arch)) - return a64::InstInternal::queryFeatures(arch, inst, operands, opCount, out); + if (Environment::isFamilyAArch64(arch)) { + return a64::InstInternal::queryFeatures(inst, operands, opCount, out); + } #endif return DebugUtils::errored(kErrorInvalidArch); diff --git a/pe-packer/asmjit/core/inst.h b/pe-packer/asmjit/core/inst.h index d0ddd8e..7c11238 100644 --- a/pe-packer/asmjit/core/inst.h +++ b/pe-packer/asmjit/core/inst.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_INST_H_INCLUDED @@ -26,7 +26,7 @@ ASMJIT_BEGIN_NAMESPACE //! //! - \ref x86::Inst (X86 and X86_64) //! - \ref a64::Inst (AArch64) -typedef uint32_t InstId; +using InstId = uint32_t; //! Instruction id parts. //! @@ -201,6 +201,21 @@ enum class InstSameRegHint : uint8_t { kWO = 2 }; +//! Options that can be used when converting instruction IDs to strings. +enum class InstStringifyOptions : uint32_t { + //! No options. + kNone = 0x00000000u, + + //! Stringify a full instruction name with known aliases. + //! + //! This option is designed for architectures where instruction aliases are common, for example X86, and where + //! multiple aliases can be used in assembly code to distinguish between intention - for example instructions + //! such as JZ and JE are the same, but the first is used in a context of equality to zero, and the second is + //! used when two values equal (for example JE next to CMP). + kAliases = 0x00000001u +}; +ASMJIT_DEFINE_ENUM_FLAGS(InstStringifyOptions) + //! Instruction id, options, and extraReg in a single structure. This structure exists mainly to simplify analysis //! and validation API that requires `BaseInst` and `Operand[]` array. class BaseInst { @@ -241,7 +256,7 @@ public: _options(options), _extraReg(extraReg) {} - ASMJIT_INLINE_NODEBUG BaseInst(InstId instId, InstOptions options, const BaseReg& extraReg) noexcept + ASMJIT_INLINE_NODEBUG BaseInst(InstId instId, InstOptions options, const Reg& extraReg) noexcept : _id(instId), _options(options), _extraReg { extraReg.signature(), extraReg.id() } {} @@ -252,16 +267,21 @@ public: //! \{ //! Returns the instruction id with modifiers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstId id() const noexcept { return _id; } + //! Sets the instruction id and modiiers from `id`. ASMJIT_INLINE_NODEBUG void setId(InstId id) noexcept { _id = id; } + //! Resets the instruction id and modifiers to zero, see \ref kIdNone. ASMJIT_INLINE_NODEBUG void resetId() noexcept { _id = 0; } //! Returns a real instruction id that doesn't contain any modifiers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstId realId() const noexcept { return _id & uint32_t(InstIdParts::kRealId); } template + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t getInstIdPart() const noexcept { return (uint32_t(_id) & uint32_t(kPart)) >> Support::ConstCTZ::value; } @@ -276,11 +296,24 @@ public: //! \name Instruction Options //! \{ + //! Returns instruction options associated with this instruction. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstOptions options() const noexcept { return _options; } + + //! Tests whether the given instruction `option` is enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOption(InstOptions option) const noexcept { return Support::test(_options, option); } + + //! Replaces all instruction options by the given `options`. ASMJIT_INLINE_NODEBUG void setOptions(InstOptions options) noexcept { _options = options; } + + //! Adds instruction options provided by `options`. ASMJIT_INLINE_NODEBUG void addOptions(InstOptions options) noexcept { _options |= options; } + + //! Clears instruction options provided by `options`. ASMJIT_INLINE_NODEBUG void clearOptions(InstOptions options) noexcept { _options &= ~options; } + + //! Resets all instruction options to `InstOptions::kNone` (there will be no instruction options active after reset). ASMJIT_INLINE_NODEBUG void resetOptions() noexcept { _options = InstOptions::kNone; } //! \} @@ -288,11 +321,23 @@ public: //! \name Extra Register //! \{ + //! Tests whether the instruction has associated an extra register. + //! + //! \note Extra registers are currently only used on X86 by AVX-512 masking such as `{k}` and `{k}{z}` and by repeated + //! instructions to explicitly assign a virtual register that would be ECX/RCX. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasExtraReg() const noexcept { return _extraReg.isReg(); } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegOnly& extraReg() noexcept { return _extraReg; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const RegOnly& extraReg() const noexcept { return _extraReg; } - ASMJIT_INLINE_NODEBUG void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + + ASMJIT_INLINE_NODEBUG void setExtraReg(const Reg& reg) noexcept { _extraReg.init(reg); } + ASMJIT_INLINE_NODEBUG void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } + ASMJIT_INLINE_NODEBUG void resetExtraReg() noexcept { _extraReg.reset(); } //! \} @@ -300,34 +345,47 @@ public: //! \name ARM Specific //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG arm::CondCode armCondCode() const noexcept { return (arm::CondCode)getInstIdPart(); } + ASMJIT_INLINE_NODEBUG void setArmCondCode(arm::CondCode cc) noexcept { setInstIdPart(uint32_t(cc)); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG a32::DataType armDt() const noexcept { return (a32::DataType)getInstIdPart(); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG a32::DataType armDt2() const noexcept { return (a32::DataType)getInstIdPart(); } + //! \} //! \name Statics //! \{ - static ASMJIT_INLINE_NODEBUG constexpr InstId composeARMInstId(uint32_t id, arm::CondCode cc) noexcept { + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR InstId composeARMInstId(uint32_t id, arm::CondCode cc) noexcept { return id | (uint32_t(cc) << Support::ConstCTZ::value); } - static ASMJIT_INLINE_NODEBUG constexpr InstId composeARMInstId(uint32_t id, arm::DataType dt, arm::CondCode cc = arm::CondCode::kAL) noexcept { + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR InstId composeARMInstId(uint32_t id, a32::DataType dt, arm::CondCode cc = arm::CondCode::kAL) noexcept { return id | (uint32_t(dt) << Support::ConstCTZ::value) | (uint32_t(cc) << Support::ConstCTZ::value); } - static ASMJIT_INLINE_NODEBUG constexpr InstId composeARMInstId(uint32_t id, arm::DataType dt, arm::DataType dt2, arm::CondCode cc = arm::CondCode::kAL) noexcept { + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR InstId composeARMInstId(uint32_t id, a32::DataType dt, a32::DataType dt2, arm::CondCode cc = arm::CondCode::kAL) noexcept { return id | (uint32_t(dt) << Support::ConstCTZ::value) | (uint32_t(dt2) << Support::ConstCTZ::value) | (uint32_t(cc) << Support::ConstCTZ::value); } - static ASMJIT_INLINE_NODEBUG constexpr InstId extractRealId(uint32_t id) noexcept { + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR InstId extractRealId(uint32_t id) noexcept { return id & uint32_t(InstIdParts::kRealId); } - static ASMJIT_INLINE_NODEBUG constexpr arm::CondCode extractARMCondCode(uint32_t id) noexcept { + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR arm::CondCode extractARMCondCode(uint32_t id) noexcept { return (arm::CondCode)((uint32_t(id) & uint32_t(InstIdParts::kARM_Cond)) >> Support::ConstCTZ::value); } @@ -344,52 +402,62 @@ enum class CpuRWFlags : uint32_t { // Common RW Flags (0x000000FF) // ---------------------------- - //! Carry flag. - kCF = 0x00000001u, //! Signed overflow flag. - kOF = 0x00000002u, - //! Sign flag (negative/sign, if set). - kSF = 0x00000004u, + kOF = 0x00000001u, + //! Carry flag. + kCF = 0x00000002u, //! Zero and/or equality flag (1 if zero/equal). - kZF = 0x00000008u, + kZF = 0x00000004u, + //! Sign flag (negative/sign, if set). + kSF = 0x00000008u, - // X86 Specific RW Flags (0xFFFFFF00) + // X86 Specific RW Flags // ---------------------------------- - //! Carry flag (X86, X86_64). + //! Carry flag (X86|X86_64). kX86_CF = kCF, - //! Overflow flag (X86, X86_64). + //! Overflow flag (X86|X86_64). kX86_OF = kOF, - //! Sign flag (X86, X86_64). + //! Sign flag (X86|X86_64). kX86_SF = kSF, - //! Zero flag (X86, X86_64). + //! Zero flag (X86|X86_64). kX86_ZF = kZF, - //! Adjust flag (X86, X86_64). + //! Adjust flag (X86|X86_64). kX86_AF = 0x00000100u, - //! Parity flag (X86, X86_64). + //! Parity flag (X86|X86_64). kX86_PF = 0x00000200u, - //! Direction flag (X86, X86_64). + //! Direction flag (X86|X86_64). kX86_DF = 0x00000400u, - //! Interrupt enable flag (X86, X86_64). + //! Interrupt enable flag (X86|X86_64). kX86_IF = 0x00000800u, - //! Alignment check flag (X86, X86_64). + //! Alignment check flag (X86|X86_64). kX86_AC = 0x00001000u, - //! FPU C0 status flag (X86, X86_64). + //! FPU C0 status flag (X86|X86_64). kX86_C0 = 0x00010000u, - //! FPU C1 status flag (X86, X86_64). + //! FPU C1 status flag (X86|X86_64). kX86_C1 = 0x00020000u, - //! FPU C2 status flag (X86, X86_64). + //! FPU C2 status flag (X86|X86_64). kX86_C2 = 0x00040000u, - //! FPU C3 status flag (X86, X86_64). - kX86_C3 = 0x00080000u + //! FPU C3 status flag (X86|X86_64). + kX86_C3 = 0x00080000u, + + // ARM Specific RW Flags + // ---------------------------------- + + kARM_V = kOF, + kARM_C = kCF, + kARM_Z = kZF, + kARM_N = kSF, + kARM_Q = 0x00000100u, + kARM_GE = 0x00000200u }; ASMJIT_DEFINE_ENUM_FLAGS(CpuRWFlags) //! Operand read/write flags describe how the operand is accessed and some additional features. -enum class OpRWFlags { +enum class OpRWFlags : uint32_t { //! No flags. kNone = 0, @@ -409,9 +477,8 @@ enum class OpRWFlags { //! //! This flag is used by all architectures to describe instructions that use consecutive registers, where only the //! first one is encoded in the instruction, and the others are just a sequence that starts with the first one. On - //! X86/X86_64 architecture this is used by instructions such as V4FMADDPS, V4FMADDSS, V4FNMADDPS, V4FNMADDSS, - //! VP4DPWSSD, VP4DPWSSDS, VP2INTERSECTD, and VP2INTERSECTQ. On ARM/AArch64 this is used by vector load and store - //! instructions that can load or store multiple registers at once. + //! X86|X86_64 architecture this is used by instructions such as VP2INTERSECTD and VP2INTERSECTQ. On ARM/AArch64 + //! this is used by vector load and store instructions that can load or store multiple registers at once. kConsecutive = 0x00000008u, //! The `extendByteMask()` represents a zero extension. @@ -492,14 +559,15 @@ struct OpRWInfo { //! Resets this operand info (resets all members) and set common information //! to the given `opFlags`, `regSize`, and possibly `physId`. - inline void reset(OpRWFlags opFlags, uint32_t regSize, uint32_t physId = BaseReg::kIdBad) noexcept { + inline void reset(OpRWFlags opFlags, uint32_t regSize, uint32_t physId = Reg::kIdBad) noexcept { _opFlags = opFlags; _physId = uint8_t(physId); _rmSize = Support::test(opFlags, OpRWFlags::kRegMem) ? uint8_t(regSize) : uint8_t(0); _consecutiveLeadCount = 0; _resetReserved(); - uint64_t mask = Support::lsbMask(regSize); + uint64_t mask = Support::lsbMask(Support::min(regSize, 64)); + _readByteMask = Support::test(opFlags, OpRWFlags::kRead) ? mask : uint64_t(0); _writeByteMask = Support::test(opFlags, OpRWFlags::kWrite) ? mask : uint64_t(0); _extendByteMask = 0; @@ -515,39 +583,56 @@ struct OpRWInfo { //! \{ //! Returns operand flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG OpRWFlags opFlags() const noexcept { return _opFlags; } + //! Tests whether operand flags contain the given `flag`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOpFlag(OpRWFlags flag) const noexcept { return Support::test(_opFlags, flag); } //! Adds the given `flags` to operand flags. ASMJIT_INLINE_NODEBUG void addOpFlags(OpRWFlags flags) noexcept { _opFlags |= flags; } + //! Removes the given `flags` from operand flags. ASMJIT_INLINE_NODEBUG void clearOpFlags(OpRWFlags flags) noexcept { _opFlags &= ~flags; } //! Tests whether this operand is read from. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isRead() const noexcept { return hasOpFlag(OpRWFlags::kRead); } + //! Tests whether this operand is written to. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isWrite() const noexcept { return hasOpFlag(OpRWFlags::kWrite); } + //! Tests whether this operand is both read and write. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isReadWrite() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kRW; } + //! Tests whether this operand is read only. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isReadOnly() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kRead; } + //! Tests whether this operand is write only. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kWrite; } //! Returns the type of a lead register, which is followed by consecutive registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t consecutiveLeadCount() const noexcept { return _consecutiveLeadCount; } //! Tests whether this operand is Reg/Mem //! //! Reg/Mem operands can use either register or memory. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isRm() const noexcept { return hasOpFlag(OpRWFlags::kRegMem); } //! Tests whether the operand will be zero extended. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isZExt() const noexcept { return hasOpFlag(OpRWFlags::kZExt); } //! Tests whether the operand must have allocated a unique physical id that cannot be shared with other register //! operands. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isUnique() const noexcept { return hasOpFlag(OpRWFlags::kUnique); } //! \} @@ -557,37 +642,63 @@ struct OpRWInfo { //! Tests whether this is a fake memory operand, which is only used, because of encoding. Fake memory operands do //! not access any memory, they are only used to encode registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemFake() const noexcept { return hasOpFlag(OpRWFlags::kMemFake); } //! Tests whether the instruction's memory BASE register is used. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseUsed() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseRW); } + //! Tests whether the instruction reads from its BASE registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseRead() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseRead); } + //! Tests whether the instruction writes to its BASE registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseWrite() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseWrite); } + //! Tests whether the instruction reads and writes from/to its BASE registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseReadWrite() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseRW; } + //! Tests whether the instruction only reads from its BASE registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseReadOnly() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseRead; } + //! Tests whether the instruction only writes to its BASE registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBaseWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseWrite; } //! Tests whether the instruction modifies the BASE register before it uses it to calculate the target address. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBasePreModify() const noexcept { return hasOpFlag(OpRWFlags::kMemBasePreModify); } + //! Tests whether the instruction modifies the BASE register after it uses it to calculate the target address. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemBasePostModify() const noexcept { return hasOpFlag(OpRWFlags::kMemBasePostModify); } //! Tests whether the instruction's memory INDEX register is used. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexUsed() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexRW); } + //! Tests whether the instruction reads the INDEX registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexRead() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexRead); } + //! Tests whether the instruction writes to its INDEX registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexWrite() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexWrite); } + //! Tests whether the instruction reads and writes from/to its INDEX registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexReadWrite() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexRW; } + //! Tests whether the instruction only reads from its INDEX registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexReadOnly() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexRead; } + //! Tests whether the instruction only writes to its INDEX registers. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMemIndexWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexWrite; } //! \} @@ -597,10 +708,14 @@ struct OpRWInfo { //! Returns a physical id of the register that is fixed for this operand. //! - //! Returns \ref BaseReg::kIdBad if any register can be used. + //! Returns \ref Reg::kIdBad if any register can be used. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t physId() const noexcept { return _physId; } + //! Tests whether \ref physId() would return a valid physical register id. - ASMJIT_INLINE_NODEBUG bool hasPhysId() const noexcept { return _physId != BaseReg::kIdBad; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasPhysId() const noexcept { return _physId != Reg::kIdBad; } + //! Sets physical register id, which would be fixed for this operand. ASMJIT_INLINE_NODEBUG void setPhysId(uint32_t physId) noexcept { _physId = uint8_t(physId); } @@ -610,7 +725,9 @@ struct OpRWInfo { //! \{ //! Returns Reg/Mem size of the operand. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t rmSize() const noexcept { return _rmSize; } + //! Sets Reg/Mem size of the operand. ASMJIT_INLINE_NODEBUG void setRmSize(uint32_t rmSize) noexcept { _rmSize = uint8_t(rmSize); } @@ -620,16 +737,23 @@ struct OpRWInfo { //! \{ //! Returns read mask. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint64_t readByteMask() const noexcept { return _readByteMask; } - //! Returns write mask. - ASMJIT_INLINE_NODEBUG uint64_t writeByteMask() const noexcept { return _writeByteMask; } - //! Returns extend mask. - ASMJIT_INLINE_NODEBUG uint64_t extendByteMask() const noexcept { return _extendByteMask; } //! Sets read mask. ASMJIT_INLINE_NODEBUG void setReadByteMask(uint64_t mask) noexcept { _readByteMask = mask; } + + //! Returns write mask. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint64_t writeByteMask() const noexcept { return _writeByteMask; } + //! Sets write mask. ASMJIT_INLINE_NODEBUG void setWriteByteMask(uint64_t mask) noexcept { _writeByteMask = mask; } + + //! Returns extend mask. + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint64_t extendByteMask() const noexcept { return _extendByteMask; } + //! Sets extend mask. ASMJIT_INLINE_NODEBUG void setExtendByteMask(uint64_t mask) noexcept { _extendByteMask = mask; } @@ -684,12 +808,15 @@ struct InstRWInfo { //! \{ //! Returns flags associated with the instruction, see \ref InstRWFlags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG InstRWFlags instFlags() const noexcept { return _instFlags; } //! Tests whether the instruction flags contain `flag`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasInstFlag(InstRWFlags flag) const noexcept { return Support::test(_instFlags, flag); } //! Tests whether the instruction flags contain \ref InstRWFlags::kMovOp. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isMovOp() const noexcept { return hasInstFlag(InstRWFlags::kMovOp); } //! \} @@ -698,8 +825,11 @@ struct InstRWInfo { //! \{ //! Returns a mask of CPU flags read. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CpuRWFlags readFlags() const noexcept { return _readFlags; } + //! Returns a mask of CPU flags written. + [[nodiscard]] ASMJIT_INLINE_NODEBUG CpuRWFlags writeFlags() const noexcept { return _writeFlags; } //! \} @@ -717,6 +847,7 @@ struct InstRWInfo { //! Some AVX+ instructions may require extra features for replacing registers with memory operands, for example //! VPSLLDQ instruction only supports `vpslldq reg, reg, imm` combination on AVX/AVX2 capable CPUs and requires //! AVX-512 for `vpslldq reg, mem, imm` combination. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t rmFeature() const noexcept { return _rmFeature; } //! \} @@ -725,25 +856,29 @@ struct InstRWInfo { //! \{ //! Returns RW information of extra register operand (extraReg). + [[nodiscard]] ASMJIT_INLINE_NODEBUG const OpRWInfo& extraReg() const noexcept { return _extraReg; } //! Returns RW information of all instruction's operands. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const OpRWInfo* operands() const noexcept { return _operands; } //! Returns RW information of the operand at the given `index`. + [[nodiscard]] inline const OpRWInfo& operand(size_t index) const noexcept { ASMJIT_ASSERT(index < Globals::kMaxOpCount); return _operands[index]; } //! Returns the number of operands this instruction has. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t opCount() const noexcept { return _opCount; } //! \} }; //! Validation flags that can be used with \ref InstAPI::validate(). -enum class ValidationFlags : uint32_t { +enum class ValidationFlags : uint8_t { //! No flags. kNone = 0, //! Allow virtual registers in the instruction. @@ -755,21 +890,28 @@ ASMJIT_DEFINE_ENUM_FLAGS(ValidationFlags) namespace InstAPI { #ifndef ASMJIT_NO_TEXT -//! Appends the name of the instruction specified by `instId` and `instOptions` into the `output` string. +//! Appends the name of the instruction specified by `instId` and `options` into the `output` string. //! //! \note Instruction options would only affect instruction prefix & suffix, other options would be ignored. //! If `instOptions` is zero then only raw instruction name (without any additional text) will be appended. -ASMJIT_API Error instIdToString(Arch arch, InstId instId, String& output) noexcept; +ASMJIT_API Error instIdToString(Arch arch, InstId instId, InstStringifyOptions options, String& output) noexcept; + +[[deprecated("Use `instIdToString()` with `InstStringifyOptions` parameter")]] +static inline Error instIdToString(Arch arch, InstId instId, String& output) noexcept { + return instIdToString(arch, instId, InstStringifyOptions::kNone, output); +} //! Parses an instruction name in the given string `s`. Length is specified by `len` argument, which can be //! `SIZE_MAX` if `s` is known to be null terminated. //! //! Returns the parsed instruction id or \ref BaseInst::kIdNone if no such instruction exists. +[[nodiscard]] ASMJIT_API InstId stringToInstId(Arch arch, const char* s, size_t len) noexcept; #endif // !ASMJIT_NO_TEXT #ifndef ASMJIT_NO_VALIDATION //! Validates the given instruction considering the given `validationFlags`. +[[nodiscard]] ASMJIT_API Error validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags = ValidationFlags::kNone) noexcept; #endif // !ASMJIT_NO_VALIDATION diff --git a/pe-packer/asmjit/core/instdb.cpp b/pe-packer/asmjit/core/instdb.cpp new file mode 100644 index 0000000..deacb22 --- /dev/null +++ b/pe-packer/asmjit/core/instdb.cpp @@ -0,0 +1,142 @@ +// This file is part of AsmJit project +// +// See or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/instdb_p.h" + +ASMJIT_BEGIN_NAMESPACE + +namespace InstNameUtils { + +static constexpr uint32_t kBufferSize = 32; + +static ASMJIT_INLINE_CONSTEXPR char decode5BitChar(uint32_t c) noexcept { + uint32_t base = c <= 26 ? uint32_t('a') - 1u : uint32_t('0') - 27u; + return char(base + c); +} + +static ASMJIT_INLINE size_t decodeToBuffer(char nameOut[kBufferSize], uint32_t nameValue, InstStringifyOptions options, const char* stringTable) noexcept { + size_t i; + + if (nameValue & 0x80000000u) { + // Small string of 5-bit characters. + // + // NOTE: Small string optimization never provides additional + // aliases formatting, so we don't have to consider `options`. + for (i = 0; i < 6; i++, nameValue >>= 5) { + uint32_t c = nameValue & 0x1F; + if (c == 0) + break; + nameOut[i] = decode5BitChar(c); + } + return i; + } + else { + size_t prefixBase = nameValue & 0xFFFu; + size_t prefixSize = (nameValue >> 12) & 0xFu; + + size_t suffixBase = (nameValue >> 16) & 0xFFFu; + size_t suffixSize = (nameValue >> 28) & 0x7u; + + if (Support::test(options, InstStringifyOptions::kAliases) && suffixBase == 0xFFFu) { + // Alias formatting immediately follows the instruction name in string table. + // The first character specifies the length and then string data follows. + prefixBase += prefixSize; + prefixSize = uint8_t(stringTable[prefixBase]); + ASMJIT_ASSERT(prefixSize <= kBufferSize); + + prefixBase += 1; // Skip the byte that specifies the length of a formatted alias. + } + + for (i = 0; i < prefixSize; i++) { + nameOut[i] = stringTable[prefixBase + i]; + } + + char* suffixOut = nameOut + prefixSize; + for (i = 0; i < suffixSize; i++) { + suffixOut[i] = stringTable[suffixBase + i]; + } + + return prefixSize + suffixSize; + } +} + +Error decode(uint32_t nameValue, InstStringifyOptions options, const char* stringTable, String& output) noexcept { + char nameData[kBufferSize]; + size_t nameSize = decodeToBuffer(nameData, nameValue, options, stringTable); + + return output.append(nameData, nameSize); +} + +InstId findInstruction(const char* s, size_t len, const uint32_t* nameTable, const char* stringTable, const InstNameIndex& nameIndex) noexcept { + ASMJIT_ASSERT(s != nullptr); + ASMJIT_ASSERT(len > 0u); + + uint32_t prefix = uint32_t(s[0]) - uint32_t('a'); + if (ASMJIT_UNLIKELY(prefix > uint32_t('z') - uint32_t('a'))) { + return BaseInst::kIdNone; + } + + size_t base = nameIndex.data[prefix].start; + size_t end = nameIndex.data[prefix].end; + + if (ASMJIT_UNLIKELY(!base)) { + return BaseInst::kIdNone; + } + + char nameData[kBufferSize]; + for (size_t lim = end - base; lim != 0; lim >>= 1) { + size_t instId = base + (lim >> 1); + size_t nameSize = decodeToBuffer(nameData, nameTable[instId], InstStringifyOptions::kNone, stringTable); + + int result = Support::compareStringViews(s, len, nameData, nameSize); + if (result < 0) { + continue; + } + + if (result > 0) { + base = instId + 1; + lim--; + continue; + } + + return InstId(instId); + } + + return BaseInst::kIdNone; +} + + +uint32_t findAlias(const char* s, size_t len, const uint32_t* nameTable, const char* stringTable, uint32_t aliasNameCount) noexcept { + ASMJIT_ASSERT(s != nullptr); + ASMJIT_ASSERT(len > 0u); + + size_t base = 0; + char nameData[kBufferSize]; + + for (size_t lim = size_t(aliasNameCount) - base; lim != 0; lim >>= 1) { + size_t index = base + (lim >> 1); + size_t nameSize = decodeToBuffer(nameData, nameTable[index], InstStringifyOptions::kNone, stringTable); + + int result = Support::compareStringViews(s, len, nameData, nameSize); + if (result < 0) { + continue; + } + + if (result > 0) { + base = index + 1; + lim--; + continue; + } + + return uint32_t(index); + } + + return Globals::kInvalidId; +} + +} // {InstNameUtils} + +ASMJIT_END_NAMESPACE diff --git a/pe-packer/asmjit/core/instdb_p.h b/pe-packer/asmjit/core/instdb_p.h new file mode 100644 index 0000000..24f0d92 --- /dev/null +++ b/pe-packer/asmjit/core/instdb_p.h @@ -0,0 +1,41 @@ +// This file is part of AsmJit project +// +// See or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_INSTDB_P_H_INCLUDED +#define ASMJIT_CORE_INSTDB_P_H_INCLUDED + +#include "../core/inst.h" +#include "../core/string.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_instruction_db +//! \{ + +struct InstNameIndex { + struct Span { + uint16_t start; + uint16_t end; + }; + + Span data[26]; + uint16_t maxNameLength; +}; + +namespace InstNameUtils { + +Error decode(uint32_t nameValue, InstStringifyOptions options, const char* stringTable, String& output) noexcept; +InstId findInstruction(const char* s, size_t len, const uint32_t* nameTable, const char* stringTable, const InstNameIndex& nameIndex) noexcept; +uint32_t findAlias(const char* s, size_t len, const uint32_t* nameTable, const char* stringTable, uint32_t aliasNameCount) noexcept; + +} // {InstNameUtils} + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_INSTDB_P_H_INCLUDED diff --git a/pe-packer/asmjit/core/jitallocator.cpp b/pe-packer/asmjit/core/jitallocator.cpp index 0ce1fa3..2ed14ca 100644 --- a/pe-packer/asmjit/core/jitallocator.cpp +++ b/pe-packer/asmjit/core/jitallocator.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -15,6 +15,10 @@ #include "../core/zonelist.h" #include "../core/zonetree.h" +#if defined(ASMJIT_TEST) +#include "../../../test/asmjit_test_random.h" +#endif // ASMJIT_TEST + ASMJIT_BEGIN_NAMESPACE // JitAllocator - Constants @@ -37,12 +41,13 @@ static constexpr uint32_t kJitAllocatorMaxBlockSize = 1024 * 1024 * 64; // =========================== static inline uint32_t JitAllocator_defaultFillPattern() noexcept { +#if ASMJIT_ARCH_X86 // X86 and X86_64 - 4x 'int3' instruction. - if (ASMJIT_ARCH_X86) - return 0xCCCCCCCCu; - + return 0xCCCCCCCCu; +#else // Unknown... return 0u; +#endif } // JitAllocator - BitVectorRangeIterator @@ -56,22 +61,22 @@ public: size_t _end; T _bitWord; - enum : uint32_t { kBitWordSize = Support::bitSizeOf() }; - enum : T { kXorMask = B == 0 ? Support::allOnes() : T(0) }; + static inline constexpr uint32_t kBitWordSize = Support::bitSizeOf(); + static inline constexpr T kXorMask = B == 0 ? Support::allOnes() : T(0); - ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords) noexcept { + ASMJIT_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords) noexcept { init(data, numBitWords); } - ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { + ASMJIT_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { init(data, numBitWords, start, end); } - ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords) noexcept { + ASMJIT_INLINE void init(const T* data, size_t numBitWords) noexcept { init(data, numBitWords, 0, numBitWords * kBitWordSize); } - ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { + ASMJIT_INLINE void init(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { ASMJIT_ASSERT(numBitWords >= (end + kBitWordSize - 1) / kBitWordSize); DebugUtils::unused(numBitWords); @@ -79,8 +84,9 @@ public: const T* ptr = data + (idx / kBitWordSize); T bitWord = 0; - if (idx < end) + if (idx < end) { bitWord = (*ptr ^ kXorMask) & (Support::allOnes() << (start % kBitWordSize)); + } _ptr = ptr; _idx = idx; @@ -88,12 +94,13 @@ public: _bitWord = bitWord; } - ASMJIT_FORCE_INLINE bool nextRange(size_t* rangeStart, size_t* rangeEnd, size_t rangeHint = std::numeric_limits::max()) noexcept { + ASMJIT_INLINE bool nextRange(size_t* rangeStart, size_t* rangeEnd, size_t rangeHint = std::numeric_limits::max()) noexcept { // Skip all empty BitWords. while (_bitWord == 0) { _idx += kBitWordSize; - if (_idx >= _end) + if (_idx >= _end) { return false; + } _bitWord = (*++_ptr) ^ kXorMask; } @@ -106,8 +113,9 @@ public: *rangeEnd = Support::min(_idx + kBitWordSize, _end); while (*rangeEnd - *rangeStart < rangeHint) { _idx += kBitWordSize; - if (_idx >= _end) + if (_idx >= _end) { break; + } _bitWord = (*++_ptr) ^ kXorMask; if (_bitWord != Support::allOnes()) { @@ -143,6 +151,9 @@ class JitAllocatorPool { public: ASMJIT_NONCOPYABLE(JitAllocatorPool) + //! \name Members + //! \{ + //! Double linked list of blocks. ZoneList blocks; //! Where to start looking first. @@ -164,12 +175,14 @@ public: //! Overhead of all blocks (in bytes). size_t totalOverheadBytes = 0; - inline JitAllocatorPool(uint32_t granularity) noexcept + //! \} + + ASMJIT_INLINE JitAllocatorPool(uint32_t granularity) noexcept : blocks(), granularity(uint16_t(granularity)), granularityLog2(uint8_t(Support::ctz(granularity))) {} - inline void reset() noexcept { + ASMJIT_INLINE void reset() noexcept { blocks.reset(); cursor = nullptr; blockCount = 0u; @@ -180,12 +193,11 @@ public: totalOverheadBytes = 0u; } - inline size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; } - inline uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); } + ASMJIT_INLINE_NODEBUG size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; } + ASMJIT_INLINE_NODEBUG uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); } - inline size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept { - using namespace Support; - return alignUp(areaSize, kBitWordSizeInBits) / kBitWordSizeInBits; + ASMJIT_INLINE_NODEBUG size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept { + return Support::alignUp(areaSize, Support::kBitWordSizeInBits) / Support::kBitWordSizeInBits; } }; @@ -202,12 +214,14 @@ public: kFlagInitialPadding = 0x00000001u, //! Block is empty. kFlagEmpty = 0x00000002u, - //! Block is dirty (largestUnusedArea, searchStart, searchEnd). + //! Block is dirty (dirty largestUnusedArea, searchStart, searchEnd, ...). kFlagDirty = 0x00000004u, + //! Block is in incremental mode, which means that there is no memory used after searchStart. + kFlagIncremental = 0x00000008u, //! Block represents memory that is using large pages. - kFlagLargePages = 0x00000008u, + kFlagLargePages = 0x00000010u, //! Block represents memory that is dual-mapped. - kFlagDualMapped = 0x00000010u + kFlagDualMapped = 0x00000020u }; static_assert(kFlagInitialPadding == 1, "JitAllocatorBlock::kFlagInitialPadding must be equal to 1"); @@ -240,14 +254,15 @@ public: //! Stop bit-vector (0 = don't care, 1 = stop). Support::BitWord* _stopBitVector {}; - inline JitAllocatorBlock( + ASMJIT_INLINE JitAllocatorBlock( JitAllocatorPool* pool, VirtMem::DualMapping mapping, size_t blockSize, uint32_t blockFlags, Support::BitWord* usedBitVector, Support::BitWord* stopBitVector, - uint32_t areaSize) noexcept + uint32_t areaSize + ) noexcept : ZoneTreeNodeT(), _pool(pool), _mapping(mapping), @@ -264,37 +279,62 @@ public: clearBlock(); } - inline JitAllocatorPool* pool() const noexcept { return _pool; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG JitAllocatorPool* pool() const noexcept { return _pool; } - inline uint8_t* rxPtr() const noexcept { return static_cast(_mapping.rx); } - inline uint8_t* rwPtr() const noexcept { return static_cast(_mapping.rw); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint8_t* rxPtr() const noexcept { return static_cast(_mapping.rx); } - inline bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; } - inline void addFlags(uint32_t f) noexcept { _flags |= f; } - inline void clearFlags(uint32_t f) noexcept { _flags &= ~f; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint8_t* rwPtr() const noexcept { return static_cast(_mapping.rw); } - inline bool empty() const noexcept { return hasFlag(kFlagEmpty); } - inline bool isDirty() const noexcept { return hasFlag(kFlagDirty); } - inline void makeDirty() noexcept { addFlags(kFlagDirty); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; } - inline bool hasLargePages() const noexcept { return hasFlag(kFlagLargePages); } - inline bool hasInitialPadding() const noexcept { return hasFlag(kFlagInitialPadding); } + ASMJIT_INLINE_NODEBUG void addFlags(uint32_t f) noexcept { _flags |= f; } + ASMJIT_INLINE_NODEBUG void clearFlags(uint32_t f) noexcept { _flags &= ~f; } - inline uint32_t initialAreaStart() const noexcept { return initialAreaStartByFlags(_flags); } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isEmpty() const noexcept { return hasFlag(kFlagEmpty); } - inline size_t blockSize() const noexcept { return _blockSize; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isDirty() const noexcept { return hasFlag(kFlagDirty); } - inline uint32_t areaSize() const noexcept { return _areaSize; } - inline uint32_t areaUsed() const noexcept { return _areaUsed; } - inline uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; } - inline uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; } + ASMJIT_INLINE_NODEBUG void makeDirty() noexcept { addFlags(kFlagDirty); } - inline void decreaseUsedArea(uint32_t value) noexcept { + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool isIncremental() const noexcept { return hasFlag(kFlagIncremental); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasLargePages() const noexcept { return hasFlag(kFlagLargePages); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG bool hasInitialPadding() const noexcept { return hasFlag(kFlagInitialPadding); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t initialAreaStart() const noexcept { return initialAreaStartByFlags(_flags); } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG size_t blockSize() const noexcept { return _blockSize; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t areaSize() const noexcept { return _areaSize; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t areaUsed() const noexcept { return _areaUsed; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; } + + [[nodiscard]] + ASMJIT_INLINE_NODEBUG uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; } + + ASMJIT_INLINE void decreaseUsedArea(uint32_t value) noexcept { _areaUsed -= value; _pool->totalAreaUsed[size_t(hasLargePages())] -= value; } - inline void clearBlock() noexcept { + ASMJIT_INLINE void clearBlock() noexcept { bool bit = hasInitialPadding(); size_t numBitWords = _pool->bitWordCountFromAreaSize(_areaSize); @@ -310,11 +350,11 @@ public: _searchStart = start; _searchEnd = _areaSize; - addFlags(JitAllocatorBlock::kFlagEmpty); + addFlags(JitAllocatorBlock::kFlagEmpty | JitAllocatorBlock::kFlagIncremental); clearFlags(JitAllocatorBlock::kFlagDirty); } - inline void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept { + ASMJIT_INLINE void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept { uint32_t allocatedAreaSize = allocatedAreaEnd - allocatedAreaStart; // Mark the newly allocated space as occupied and also the sentinel. @@ -333,42 +373,57 @@ public: clearFlags(kFlagDirty | kFlagEmpty); } else { - if (_searchStart == allocatedAreaStart) + if (_searchStart == allocatedAreaStart) { _searchStart = allocatedAreaEnd; - if (_searchEnd == allocatedAreaEnd) + } + + if (_searchEnd == allocatedAreaEnd) { _searchEnd = allocatedAreaStart; + } addFlags(kFlagDirty); clearFlags(kFlagEmpty); } } - inline void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept { + ASMJIT_INLINE void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept { uint32_t releasedAreaSize = releasedAreaEnd - releasedAreaStart; // Update the search region and statistics. _pool->totalAreaUsed[size_t(hasLargePages())] -= releasedAreaSize; _areaUsed -= releasedAreaSize; - _searchStart = Support::min(_searchStart, releasedAreaStart); - _searchEnd = Support::max(_searchEnd, releasedAreaEnd); // Unmark occupied bits and also the sentinel. Support::bitVectorClear(_usedBitVector, releasedAreaStart, releasedAreaSize); Support::bitVectorSetBit(_stopBitVector, releasedAreaEnd - 1, false); - if (areaUsed() == initialAreaStart()) { - _searchStart = initialAreaStart(); - _searchEnd = _areaSize; - _largestUnusedArea = _areaSize - initialAreaStart(); - addFlags(kFlagEmpty); - clearFlags(kFlagDirty); + if (Support::bool_and(isIncremental(), _searchStart == releasedAreaEnd)) { + // Incremental mode: If the area released is at the end of the fully used area, we would like to + // keep the incremental mode of the block. In order to do that, we are going to use a different + // approach - decrement `_searchStart` and increment `_largestUnusedArea`, which are essential + // for incremental mode blocks. + ASMJIT_ASSERT(_searchStart >= releasedAreaSize); + _searchStart -= releasedAreaSize; + _largestUnusedArea += releasedAreaSize; } else { - addFlags(kFlagDirty); + _searchStart = Support::min(_searchStart, releasedAreaStart); + _searchEnd = Support::max(_searchEnd, releasedAreaEnd); + clearFlags(kFlagDirty | kFlagIncremental); + + if (areaUsed() == initialAreaStart()) { + _searchStart = initialAreaStart(); + _searchEnd = _areaSize; + _largestUnusedArea = _areaSize - initialAreaStart(); + addFlags(kFlagEmpty); + } + else { + addFlags(kFlagDirty); + } } } - inline void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept { + ASMJIT_INLINE void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept { uint32_t shrunkAreaSize = shrunkAreaEnd - shrunkAreaStart; // Shrunk area cannot start at zero as it would mean that we have shrunk the first @@ -379,24 +434,32 @@ public: // Update the search region and statistics. _pool->totalAreaUsed[size_t(hasLargePages())] -= shrunkAreaSize; _areaUsed -= shrunkAreaSize; - _searchStart = Support::min(_searchStart, shrunkAreaStart); - _searchEnd = Support::max(_searchEnd, shrunkAreaEnd); + + if (Support::bool_and(isIncremental(), _searchStart == shrunkAreaEnd)) { + _searchStart -= shrunkAreaSize; + _largestUnusedArea += shrunkAreaSize; + } + else { + _searchStart = Support::min(_searchStart, shrunkAreaStart); + _searchEnd = Support::max(_searchEnd, shrunkAreaEnd); + + clearFlags(kFlagIncremental); + addFlags(kFlagDirty); + } // Unmark the released space and move the sentinel. Support::bitVectorClear(_usedBitVector, shrunkAreaStart, shrunkAreaSize); Support::bitVectorSetBit(_stopBitVector, shrunkAreaEnd - 1, false); Support::bitVectorSetBit(_stopBitVector, shrunkAreaStart - 1, true); - - addFlags(kFlagDirty); } // RBTree default CMP uses '<' and '>' operators. - inline bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); } - inline bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); } + ASMJIT_INLINE_NODEBUG bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); } + ASMJIT_INLINE_NODEBUG bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); } // Special implementation for querying blocks by `key`, which must be in `[BlockPtr, BlockPtr + BlockSize)` range. - inline bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; } - inline bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; } + ASMJIT_INLINE_NODEBUG bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; } + ASMJIT_INLINE_NODEBUG bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; } }; // JitAllocator - PrivateImpl @@ -404,6 +467,9 @@ public: class JitAllocatorPrivateImpl : public JitAllocator::Impl { public: + //! \name Members + //! \{ + //! Lock for thread safety. mutable Lock lock; //! System page size (also a minimum block size). @@ -418,13 +484,15 @@ public: //! Number of allocator pools. size_t poolCount; - inline JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept + //! \} + + ASMJIT_INLINE JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept : JitAllocator::Impl {}, pageSize(0), allocationCount(0), pools(pools), poolCount(poolCount) {} - inline ~JitAllocatorPrivateImpl() noexcept {} + ASMJIT_INLINE ~JitAllocatorPrivateImpl() noexcept {} }; static const JitAllocator::Impl JitAllocatorImpl_none {}; @@ -436,8 +504,9 @@ static const JitAllocator::CreateParams JitAllocatorParams_none {}; static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator::CreateParams* params) noexcept { VirtMem::Info vmInfo = VirtMem::info(); - if (!params) + if (!params) { params = &JitAllocatorParams_none; + } JitAllocatorOptions options = params->options; uint32_t blockSize = params->blockSize; @@ -446,37 +515,44 @@ static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator:: // Setup pool count to [1..3]. size_t poolCount = 1; - if (Support::test(options, JitAllocatorOptions::kUseMultiplePools)) - poolCount = kJitAllocatorMultiPoolCount;; + if (Support::test(options, JitAllocatorOptions::kUseMultiplePools)) { + poolCount = kJitAllocatorMultiPoolCount; + } // Setup block size [64kB..256MB]. - if (blockSize < 64 * 1024 || blockSize > 256 * 1024 * 1024 || !Support::isPowerOf2(blockSize)) + if (blockSize < 64 * 1024 || blockSize > 256 * 1024 * 1024 || !Support::isPowerOf2(blockSize)) { blockSize = vmInfo.pageGranularity; + } // Setup granularity [64..256]. - if (granularity < 64 || granularity > 256 || !Support::isPowerOf2(granularity)) + if (granularity < 64 || granularity > 256 || !Support::isPowerOf2(granularity)) { granularity = kJitAllocatorBaseGranularity; + } // Setup fill-pattern. - if (uint32_t(options & JitAllocatorOptions::kCustomFillPattern) == 0) + if (uint32_t(options & JitAllocatorOptions::kCustomFillPattern) == 0) { fillPattern = JitAllocator_defaultFillPattern(); + } size_t size = sizeof(JitAllocatorPrivateImpl) + sizeof(JitAllocatorPool) * poolCount; void* p = ::malloc(size); - if (ASMJIT_UNLIKELY(!p)) + + if (ASMJIT_UNLIKELY(!p)) { return nullptr; + } VirtMem::HardenedRuntimeInfo hardenedRtInfo = VirtMem::hardenedRuntimeInfo(); if (Support::test(hardenedRtInfo.flags, VirtMem::HardenedRuntimeFlags::kEnabled)) { // If we are running within a hardened environment (mapping RWX is not allowed) then we have to use dual mapping // or other runtime capabilities like Apple specific MAP_JIT. There is no point in not enabling these as otherwise // the allocation would fail and JitAllocator would not be able to allocate memory. - if (!Support::test(hardenedRtInfo.flags, VirtMem::HardenedRuntimeFlags::kMapJit)) + if (!Support::test(hardenedRtInfo.flags, VirtMem::HardenedRuntimeFlags::kMapJit)) { options |= JitAllocatorOptions::kUseDualMapping; + } } JitAllocatorPool* pools = reinterpret_cast((uint8_t*)p + sizeof(JitAllocatorPrivateImpl)); - JitAllocatorPrivateImpl* impl = new(p) JitAllocatorPrivateImpl(pools, poolCount); + JitAllocatorPrivateImpl* impl = new(Support::PlacementNew{p}) JitAllocatorPrivateImpl(pools, poolCount); impl->options = options; impl->blockSize = blockSize; @@ -484,24 +560,26 @@ static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator:: impl->fillPattern = fillPattern; impl->pageSize = vmInfo.pageSize; - for (size_t poolId = 0; poolId < poolCount; poolId++) - new(&pools[poolId]) JitAllocatorPool(granularity << poolId); + for (size_t poolId = 0; poolId < poolCount; poolId++) { + new(Support::PlacementNew{&pools[poolId]}) JitAllocatorPool(granularity << poolId); + } return impl; } -static inline void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept { +static ASMJIT_INLINE void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept { impl->~JitAllocatorPrivateImpl(); ::free(impl); } -static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept { size_t poolId = impl->poolCount - 1; size_t granularity = size_t(impl->granularity) << poolId; while (poolId) { - if (Support::alignUp(size, granularity) == size) + if (Support::alignUp(size, granularity) == size) { break; + } poolId--; granularity >>= 1; } @@ -509,30 +587,33 @@ static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl return poolId; } -static inline size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept { using Support::kBitWordSizeInBits; return ((areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits) * sizeof(Support::BitWord); } -static inline size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept { +static ASMJIT_INLINE size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept { JitAllocatorBlock* last = pool->blocks.last(); size_t blockSize = last ? last->blockSize() : size_t(impl->blockSize); // We have to increase the allocationSize if we know that the block must provide padding. if (!Support::test(impl->options, JitAllocatorOptions::kDisableInitialPadding)) { size_t granularity = pool->granularity; - if (SIZE_MAX - allocationSize < granularity) + if (SIZE_MAX - allocationSize < granularity) { return 0; // Overflown + } allocationSize += granularity; } - if (blockSize < kJitAllocatorMaxBlockSize) + if (blockSize < kJitAllocatorMaxBlockSize) { blockSize *= 2u; + } if (allocationSize > blockSize) { blockSize = Support::alignUp(allocationSize, impl->blockSize); - if (ASMJIT_UNLIKELY(blockSize < allocationSize)) + if (ASMJIT_UNLIKELY(blockSize < allocationSize)) { return 0; // Overflown. + } } return blockSize; @@ -561,8 +642,9 @@ ASMJIT_FAVOR_SPEED static void JitAllocatorImpl_fillPattern(void* mem, uint32_t uint32_t* mem32 = static_cast(mem); size_t n = byteSize / 4u; - for (size_t i = 0; i < n; i++) + for (size_t i = 0; i < n; i++) { mem32[i] = pattern; + } } // Allocate a new `JitAllocatorBlock` for the given `blockSize`. @@ -574,8 +656,9 @@ static Error JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocat using Support::kBitWordSizeInBits; uint32_t blockFlags = 0; - if (!Support::test(impl->options, JitAllocatorOptions::kDisableInitialPadding)) + if (!Support::test(impl->options, JitAllocatorOptions::kDisableInitialPadding)) { blockFlags |= JitAllocatorBlock::kFlagInitialPadding; + } VirtMem::DualMapping virtMem {}; VirtMem::MemoryFlags memFlags = VirtMem::MemoryFlags::kAccessRWX; @@ -618,10 +701,12 @@ static Error JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocat // Out of memory... if (ASMJIT_UNLIKELY(blockPtr == nullptr)) { - if (Support::test(impl->options, JitAllocatorOptions::kUseDualMapping)) - VirtMem::releaseDualMapping(&virtMem, blockSize); - else - VirtMem::release(virtMem.rx, blockSize); + if (Support::test(impl->options, JitAllocatorOptions::kUseDualMapping)) { + (void)VirtMem::releaseDualMapping(&virtMem, blockSize); + } + else { + (void)VirtMem::release(virtMem.rx, blockSize); + } return DebugUtils::errored(kErrorOutOfMemory); } @@ -632,17 +717,19 @@ static Error JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocat } BitWord* bitWords = reinterpret_cast(blockPtr + sizeof(JitAllocatorBlock)); - *dst = new(blockPtr) JitAllocatorBlock(pool, virtMem, blockSize, blockFlags, bitWords, bitWords + numBitWords, areaSize); + *dst = new(Support::PlacementNew{blockPtr}) JitAllocatorBlock(pool, virtMem, blockSize, blockFlags, bitWords, bitWords + numBitWords, areaSize); return kErrorOk; } static void JitAllocatorImpl_deleteBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { DebugUtils::unused(impl); - if (block->hasFlag(JitAllocatorBlock::kFlagDualMapped)) - VirtMem::releaseDualMapping(&block->_mapping, block->blockSize()); - else - VirtMem::release(block->rxPtr(), block->blockSize()); + if (block->hasFlag(JitAllocatorBlock::kFlagDualMapped)) { + (void)VirtMem::releaseDualMapping(&block->_mapping, block->blockSize()); + } + else { + (void)VirtMem::release(block->rxPtr(), block->blockSize()); + } ::free(block); } @@ -650,8 +737,9 @@ static void JitAllocatorImpl_deleteBlock(JitAllocatorPrivateImpl* impl, JitAlloc static void JitAllocatorImpl_insertBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { JitAllocatorPool* pool = block->pool(); - if (!pool->cursor) + if (!pool->cursor) { pool->cursor = block; + } // Add to RBTree and List. impl->tree.insert(block); @@ -669,8 +757,9 @@ static void JitAllocatorImpl_removeBlock(JitAllocatorPrivateImpl* impl, JitAlloc JitAllocatorPool* pool = block->pool(); // Remove from RBTree and List. - if (pool->cursor == block) + if (pool->cursor == block) { pool->cursor = block->hasPrev() ? block->prev() : block->next(); + } impl->tree.remove(block); pool->blocks.unlink(block); @@ -684,8 +773,9 @@ static void JitAllocatorImpl_removeBlock(JitAllocatorPrivateImpl* impl, JitAlloc } static void JitAllocatorImpl_wipeOutBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { - if (block->hasFlag(JitAllocatorBlock::kFlagEmpty)) + if (block->hasFlag(JitAllocatorBlock::kFlagEmpty)) { return; + } JitAllocatorPool* pool = block->pool(); if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) { @@ -716,13 +806,15 @@ static void JitAllocatorImpl_wipeOutBlock(JitAllocatorPrivateImpl* impl, JitAllo JitAllocator::JitAllocator(const CreateParams* params) noexcept { _impl = JitAllocatorImpl_new(params); - if (ASMJIT_UNLIKELY(!_impl)) + if (ASMJIT_UNLIKELY(!_impl)) { _impl = const_cast(&JitAllocatorImpl_none); + } } JitAllocator::~JitAllocator() noexcept { - if (_impl == &JitAllocatorImpl_none) + if (_impl == &JitAllocatorImpl_none) { return; + } reset(ResetPolicy::kHard); JitAllocatorImpl_destroy(static_cast(_impl)); @@ -732,8 +824,9 @@ JitAllocator::~JitAllocator() noexcept { // ==================== void JitAllocator::reset(ResetPolicy resetPolicy) noexcept { - if (_impl == &JitAllocatorImpl_none) + if (_impl == &JitAllocatorImpl_none) { return; + } JitAllocatorPrivateImpl* impl = static_cast(_impl); impl->tree.reset(); @@ -743,26 +836,28 @@ void JitAllocator::reset(ResetPolicy resetPolicy) noexcept { JitAllocatorPool& pool = impl->pools[poolId]; JitAllocatorBlock* block = pool.blocks.first(); - JitAllocatorBlock* blockToKeep = nullptr; - if (resetPolicy != ResetPolicy::kHard && uint32_t(impl->options & JitAllocatorOptions::kImmediateRelease) == 0) { - blockToKeep = block; - block = block->next(); - } - - while (block) { - JitAllocatorBlock* next = block->next(); - JitAllocatorImpl_deleteBlock(impl, block); - block = next; - } - pool.reset(); - if (blockToKeep) { - blockToKeep->_listNodes[0] = nullptr; - blockToKeep->_listNodes[1] = nullptr; - JitAllocatorImpl_wipeOutBlock(impl, blockToKeep); - JitAllocatorImpl_insertBlock(impl, blockToKeep); - pool.emptyBlockCount = 1; + if (block) { + JitAllocatorBlock* blockToKeep = nullptr; + if (resetPolicy != ResetPolicy::kHard && uint32_t(impl->options & JitAllocatorOptions::kImmediateRelease) == 0) { + blockToKeep = block; + block = block->next(); + } + + while (block) { + JitAllocatorBlock* next = block->next(); + JitAllocatorImpl_deleteBlock(impl, block); + block = next; + } + + if (blockToKeep) { + blockToKeep->_listNodes[0] = nullptr; + blockToKeep->_listNodes[1] = nullptr; + JitAllocatorImpl_wipeOutBlock(impl, blockToKeep); + JitAllocatorImpl_insertBlock(impl, blockToKeep); + pool.emptyBlockCount = 1; + } } } } @@ -797,21 +892,20 @@ JitAllocator::Statistics JitAllocator::statistics() const noexcept { // ============================== Error JitAllocator::alloc(Span& out, size_t size) noexcept { - out = Span{}; - - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) - return DebugUtils::errored(kErrorNotInitialized); + constexpr uint32_t kNoIndex = std::numeric_limits::max(); + constexpr size_t kMaxRequestSize = std::numeric_limits::max() / 2u; JitAllocatorPrivateImpl* impl = static_cast(_impl); - constexpr uint32_t kNoIndex = std::numeric_limits::max(); + bool notInitialized = _impl == &JitAllocatorImpl_none; // Align to the minimum granularity by default. size = Support::alignUp(size, impl->granularity); - if (ASMJIT_UNLIKELY(size == 0)) - return DebugUtils::errored(kErrorInvalidArgument); + out = Span{}; - if (ASMJIT_UNLIKELY(size > std::numeric_limits::max() / 2)) - return DebugUtils::errored(kErrorTooLarge); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, size - 1u >= kMaxRequestSize))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : + size == 0u ? kErrorInvalidArgument : kErrorTooLarge); + } LockGuard guard(impl->lock); JitAllocatorPool* pool = &impl->pools[JitAllocatorImpl_sizeToPoolId(impl, size)]; @@ -820,13 +914,27 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { uint32_t areaSize = uint32_t(pool->areaSizeFromByteSize(size)); // Try to find the requested memory area in existing blocks. - JitAllocatorBlock* block = pool->blocks.first(); + JitAllocatorBlock* block = pool->cursor; + if (block) { JitAllocatorBlock* initial = block; + do { - JitAllocatorBlock* next = block->hasNext() ? block->next() : pool->blocks.first(); - if (block->areaAvailable() >= areaSize) { - if (block->isDirty() || block->largestUnusedArea() >= areaSize) { + uint32_t largestUnusedArea = block->largestUnusedArea(); + + if (Support::bool_and(block->isIncremental(), largestUnusedArea >= areaSize)) { + // Fast path: If the block is in incremental mode, which means that it's guaranteed it's full before + // `searchStart` and completely empty after it, we can just quickly increment `searchStart` and be + // done with the allocation. This is a little bit faster than constructing a BitVectorRangeIterator + // and searching for zero bit clusters. When a block is in incremental mode its `largestUnusedArea` + // is basically the free space after `searchStart`, so that's the only thing to check. + areaIndex = block->_searchStart; + block->_largestUnusedArea -= areaSize; + break; + } + else if (block->areaAvailable() >= areaSize) { + // Regular path: Search for a cluster of bits that would mark an empty area we want to allocate. + if (Support::bool_or(block->isDirty(), largestUnusedArea >= areaSize)) { BitVectorRangeIterator it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize()), block->_searchStart, block->_searchEnd); size_t rangeStart = 0; @@ -846,8 +954,9 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { largestArea = Support::max(largestArea, rangeSize); } - if (areaIndex != kNoIndex) + if (areaIndex != kNoIndex) { break; + } if (searchStart != SIZE_MAX) { // Because we have iterated over the entire block, we can now mark the @@ -862,15 +971,18 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { } } - block = next; + // The block cursor doesn't have to start with the first block and we want to + // iterate all before concluding that there is no free space in any block. + block = block->hasNext() ? block->next() : pool->blocks.first(); } while (block != initial); } // Allocate a new block if there is no region of a required size. if (areaIndex == kNoIndex) { size_t blockSize = JitAllocatorImpl_calculateIdealBlockSize(impl, pool, size); - if (ASMJIT_UNLIKELY(!blockSize)) + if (ASMJIT_UNLIKELY(!blockSize)) { return DebugUtils::errored(kErrorOutOfMemory); + } ASMJIT_PROPAGATE(JitAllocatorImpl_newBlock(impl, &block, pool, blockSize)); areaIndex = block->initialAreaStart(); @@ -901,18 +1013,19 @@ Error JitAllocator::alloc(Span& out, size_t size) noexcept { } Error JitAllocator::release(void* rx) noexcept { - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) - return DebugUtils::errored(kErrorNotInitialized); + bool notInitialized = _impl == &JitAllocatorImpl_none; - if (ASMJIT_UNLIKELY(!rx)) - return DebugUtils::errored(kErrorInvalidArgument); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, !rx))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : kErrorInvalidArgument); + } JitAllocatorPrivateImpl* impl = static_cast(_impl); LockGuard guard(impl->lock); JitAllocatorBlock* block = impl->tree.get(static_cast(rx)); - if (ASMJIT_UNLIKELY(!block)) + if (ASMJIT_UNLIKELY(!block)) { return DebugUtils::errored(kErrorInvalidState); + } // Offset relative to the start of the block. JitAllocatorPool* pool = block->pool(); @@ -936,7 +1049,7 @@ Error JitAllocator::release(void* rx) noexcept { } // Release the whole block if it became empty. - if (block->empty()) { + if (block->isEmpty()) { if (pool->emptyBlockCount || Support::test(impl->options, JitAllocatorOptions::kImmediateRelease)) { JitAllocatorImpl_removeBlock(impl, block); JitAllocatorImpl_deleteBlock(impl, block); @@ -951,8 +1064,9 @@ Error JitAllocator::release(void* rx) noexcept { static Error JitAllocatorImpl_shrink(JitAllocatorPrivateImpl* impl, JitAllocator::Span& span, size_t newSize, bool alreadyUnderWriteScope) noexcept { JitAllocatorBlock* block = static_cast(span._block); - if (ASMJIT_UNLIKELY(!block)) + if (ASMJIT_UNLIKELY(!block)) { return DebugUtils::errored(kErrorInvalidArgument); + } LockGuard guard(impl->lock); @@ -965,16 +1079,18 @@ static Error JitAllocatorImpl_shrink(JitAllocatorPrivateImpl* impl, JitAllocator // Don't trust `span.size()` - if it has been already truncated we would be off... bool isUsed = Support::bitVectorGetBit(block->_usedBitVector, areaStart); - if (ASMJIT_UNLIKELY(!isUsed)) + if (ASMJIT_UNLIKELY(!isUsed)) { return DebugUtils::errored(kErrorInvalidArgument); + } uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaStart, true)) + 1; uint32_t areaPrevSize = areaEnd - areaStart; uint32_t spanPrevSize = areaPrevSize * pool->granularity; uint32_t areaShrunkSize = pool->areaSizeFromByteSize(newSize); - if (ASMJIT_UNLIKELY(areaShrunkSize > areaPrevSize)) + if (ASMJIT_UNLIKELY(areaShrunkSize > areaPrevSize)) { return DebugUtils::errored(kErrorInvalidArgument); + } uint32_t areaDiff = areaPrevSize - areaShrunkSize; if (areaDiff) { @@ -1000,11 +1116,11 @@ static Error JitAllocatorImpl_shrink(JitAllocatorPrivateImpl* impl, JitAllocator } Error JitAllocator::shrink(Span& span, size_t newSize) noexcept { - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) - return DebugUtils::errored(kErrorNotInitialized); + bool notInitialized = _impl == &JitAllocatorImpl_none; - if (ASMJIT_UNLIKELY(!span.rx())) - return DebugUtils::errored(kErrorInvalidArgument); + if (ASMJIT_UNLIKELY(Support::bool_or(notInitialized, !span.rx()))) { + return DebugUtils::errored(notInitialized ? kErrorNotInitialized : kErrorInvalidArgument); + } if (ASMJIT_UNLIKELY(newSize == 0)) { Error err = release(span.rx()); @@ -1018,15 +1134,17 @@ Error JitAllocator::shrink(Span& span, size_t newSize) noexcept { Error JitAllocator::query(Span& out, void* rx) const noexcept { out = Span{}; - if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) + if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) { return DebugUtils::errored(kErrorNotInitialized); + } JitAllocatorPrivateImpl* impl = static_cast(_impl); LockGuard guard(impl->lock); JitAllocatorBlock* block = impl->tree.get(static_cast(rx)); - if (ASMJIT_UNLIKELY(!block)) + if (ASMJIT_UNLIKELY(!block)) { return DebugUtils::errored(kErrorInvalidArgument); + } // Offset relative to the start of the block. JitAllocatorPool* pool = block->pool(); @@ -1036,8 +1154,9 @@ Error JitAllocator::query(Span& out, void* rx) const noexcept { uint32_t areaStart = uint32_t(offset >> pool->granularityLog2); bool isUsed = Support::bitVectorGetBit(block->_usedBitVector, areaStart); - if (ASMJIT_UNLIKELY(!isUsed)) + if (ASMJIT_UNLIKELY(!isUsed)) { return DebugUtils::errored(kErrorInvalidArgument); + } uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaStart, true)) + 1; size_t byteOffset = pool->byteSizeFromAreaSize(areaStart); @@ -1054,22 +1173,27 @@ Error JitAllocator::query(Span& out, void* rx) const noexcept { // JitAllocator - Write // ==================== -static ASMJIT_FORCE_INLINE VirtMem::CachePolicy JitAllocator_defaultPolicyForSpan(const JitAllocator::Span& span) noexcept { - if (Support::test(span.flags(), JitAllocator::Span::Flags::kInstructionCacheClean)) +static ASMJIT_INLINE VirtMem::CachePolicy JitAllocator_defaultPolicyForSpan(const JitAllocator::Span& span) noexcept { + if (Support::test(span.flags(), JitAllocator::Span::Flags::kInstructionCacheClean)) { return VirtMem::CachePolicy::kNeverFlush; - else + } + else { return VirtMem::CachePolicy::kFlushAfterWrite; + } } Error JitAllocator::write(Span& span, size_t offset, const void* src, size_t size, VirtMem::CachePolicy policy) noexcept { - if (ASMJIT_UNLIKELY(span._block == nullptr || offset > span.size() || span.size() - offset < size)) + if (ASMJIT_UNLIKELY(span._block == nullptr || offset > span.size() || span.size() - offset < size)) { return DebugUtils::errored(kErrorInvalidArgument); + } - if (ASMJIT_UNLIKELY(size == 0)) + if (ASMJIT_UNLIKELY(size == 0)) { return kErrorOk; + } - if (policy == VirtMem::CachePolicy::kDefault) + if (policy == VirtMem::CachePolicy::kDefault) { policy = JitAllocator_defaultPolicyForSpan(span); + } VirtMem::ProtectJitReadWriteScope writeScope(span.rx(), span.size(), policy); memcpy(static_cast(span.rw()) + offset, src, size); @@ -1077,15 +1201,18 @@ Error JitAllocator::write(Span& span, size_t offset, const void* src, size_t siz } Error JitAllocator::write(Span& span, WriteFunc writeFunc, void* userData, VirtMem::CachePolicy policy) noexcept { - if (ASMJIT_UNLIKELY(span._block == nullptr) || span.size() == 0) + if (ASMJIT_UNLIKELY(span._block == nullptr) || span.size() == 0) { return DebugUtils::errored(kErrorInvalidArgument); + } size_t size = span.size(); - if (ASMJIT_UNLIKELY(size == 0)) + if (ASMJIT_UNLIKELY(size == 0)) { return kErrorOk; + } - if (policy == VirtMem::CachePolicy::kDefault) + if (policy == VirtMem::CachePolicy::kDefault) { policy = JitAllocator_defaultPolicyForSpan(span); + } VirtMem::ProtectJitReadWriteScope writeScope(span.rx(), span.size(), policy); ASMJIT_PROPAGATE(writeFunc(span, userData)); @@ -1110,30 +1237,34 @@ Error JitAllocator::beginWriteScope(WriteScopeData& scope, VirtMem::CachePolicy } Error JitAllocator::endWriteScope(WriteScopeData& scope) noexcept { - if (ASMJIT_UNLIKELY(!scope._allocator)) + if (ASMJIT_UNLIKELY(!scope._allocator)) { return DebugUtils::errored(kErrorInvalidArgument); + } return kErrorOk; } Error JitAllocator::flushWriteScope(WriteScopeData& scope) noexcept { - if (ASMJIT_UNLIKELY(!scope._allocator)) + if (ASMJIT_UNLIKELY(!scope._allocator)) { return DebugUtils::errored(kErrorInvalidArgument); + } return kErrorOk; } Error JitAllocator::scopedWrite(WriteScopeData& scope, Span& span, size_t offset, const void* src, size_t size) noexcept { - if (ASMJIT_UNLIKELY(!scope._allocator)) + if (ASMJIT_UNLIKELY(!scope._allocator)) { return DebugUtils::errored(kErrorInvalidArgument); + } VirtMem::CachePolicy policy = VirtMem::CachePolicy(scope._data[0]); return scope._allocator->write(span, offset, src, size, policy); } Error JitAllocator::scopedWrite(WriteScopeData& scope, Span& span, WriteFunc writeFunc, void* userData) noexcept { - if (ASMJIT_UNLIKELY(!scope._allocator)) + if (ASMJIT_UNLIKELY(!scope._allocator)) { return DebugUtils::errored(kErrorInvalidArgument); + } VirtMem::CachePolicy policy = VirtMem::CachePolicy(scope._data[0]); return scope._allocator->write(span, writeFunc, userData, policy); @@ -1143,56 +1274,6 @@ Error JitAllocator::scopedWrite(WriteScopeData& scope, Span& span, WriteFunc wri // ==================== #if defined(ASMJIT_TEST) -// A pseudo random number generator based on a paper by Sebastiano Vigna: -// http://vigna.di.unimi.it/ftp/papers/xorshiftplus.pdf -class Random { -public: - // Constants suggested as `23/18/5`. - enum Steps : uint32_t { - kStep1_SHL = 23, - kStep2_SHR = 18, - kStep3_SHR = 5 - }; - - inline explicit Random(uint64_t seed = 0) noexcept { reset(seed); } - inline Random(const Random& other) noexcept = default; - - inline void reset(uint64_t seed = 0) noexcept { - // The number is arbitrary, it means nothing. - constexpr uint64_t kZeroSeed = 0x1F0A2BE71D163FA0u; - - // Generate the state data by using splitmix64. - for (uint32_t i = 0; i < 2; i++) { - seed += 0x9E3779B97F4A7C15u; - uint64_t x = seed; - x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9u; - x = (x ^ (x >> 27)) * 0x94D049BB133111EBu; - x = (x ^ (x >> 31)); - _state[i] = x != 0 ? x : kZeroSeed; - } - } - - inline uint32_t nextUInt32() noexcept { - return uint32_t(nextUInt64() >> 32); - } - - inline uint64_t nextUInt64() noexcept { - uint64_t x = _state[0]; - uint64_t y = _state[1]; - - x ^= x << kStep1_SHL; - y ^= y >> kStep3_SHR; - x ^= x >> kStep2_SHR; - x ^= y; - - _state[0] = y; - _state[1] = x; - return x + y; - } - - uint64_t _state[2]; -}; - namespace JitAllocatorUtils { static void fillPattern64(void* p_, uint64_t pattern, size_t sizeInBytes) noexcept { uint64_t* p = static_cast(p_); @@ -1260,20 +1341,19 @@ public: }; Zone _zone; - ZoneAllocator _heap; + ZonePool _recordPool; ZoneTree _records; JitAllocator _allocator; - Random _rng; + TestUtils::Random _rng; explicit JitAllocatorWrapper(const JitAllocator::CreateParams* params) noexcept - : _zone(1024 * 1024), - _heap(&_zone), + : _zone(1024u * 1024u), _allocator(params), _rng(0x123456789u) {} void _insert(void* pRX, void* pRW, size_t size) noexcept { uint8_t* p = static_cast(pRX); - uint8_t* pEnd = p + size - 1; + uint8_t* pEnd = p + size - 1u; Record* record; @@ -1286,7 +1366,7 @@ public: .message("Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size); uint64_t pattern = _rng.nextUInt64(); - record = _heap.newT(pRX, pRW, size, pattern); + record = new(Support::PlacementNew{_recordPool.alloc(_zone)}) Record(pRX, pRW, size, pattern); EXPECT_NOT_NULL(record); { @@ -1308,7 +1388,7 @@ public: EXPECT_TRUE(JitAllocatorUtils::verifyPattern64(record->rw(), record->pattern, record->size)); _records.remove(record); - _heap.release(record, sizeof(Record)); + _recordPool.release(record); } void* alloc(size_t size) noexcept { @@ -1331,8 +1411,9 @@ public: Record* record = _records.get(static_cast(p)); EXPECT_NOT_NULL(record); - if (!newSize) + if (!newSize) { return release(p); + } JitAllocator::Span span; EXPECT_EQ(_allocator.query(span, p), kErrorOk); @@ -1344,7 +1425,7 @@ public: } }; -static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, Random& prng) noexcept { +static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, TestUtils::Random& prng) noexcept { for (size_t i = 0; i < count; ++i) std::swap(ptrArray[i], ptrArray[size_t(prng.nextUInt32() % count)]); } @@ -1358,7 +1439,7 @@ static void JitAllocatorTest_usage(JitAllocator& allocator) noexcept { } template -static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcept { +static void BitVectorRangeIterator_testRandom(TestUtils::Random& rnd, size_t count) noexcept { for (size_t i = 0; i < count; i++) { T in[kPatternSize]; T out[kPatternSize]; @@ -1372,10 +1453,12 @@ static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcep BitVectorRangeIterator it(in, kPatternSize); size_t rangeStart, rangeEnd; while (it.nextRange(&rangeStart, &rangeEnd)) { - if (Bit) + if (Bit) { Support::bitVectorFill(out, rangeStart, rangeEnd - rangeStart); - else + } + else { Support::bitVectorClear(out, rangeStart, rangeEnd - rangeStart); + } } } @@ -1386,6 +1469,11 @@ static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcep } } +static void test_jit_allocator_reset_empty() noexcept { + JitAllocator allocator; + allocator.reset(ResetPolicy::kSoft); +} + static void test_jit_allocator_alloc_release() noexcept { size_t kCount = BrokenAPI::hasArg("--quick") ? 20000 : 100000; @@ -1398,11 +1486,12 @@ static void test_jit_allocator_alloc_release() noexcept { using Opt = JitAllocatorOptions; + VirtMem::HardenedRuntimeInfo hri = VirtMem::hardenedRuntimeInfo(); + TestParams testParams[] = { { "Default" , Opt::kNone, 0, 0 }, { "16MB blocks" , Opt::kNone, 16 * 1024 * 1024, 0 }, { "256B granularity" , Opt::kNone, 0, 256 }, - { "kUseDualMapping" , Opt::kUseDualMapping , 0, 0 }, { "kUseMultiplePools" , Opt::kUseMultiplePools, 0, 0 }, { "kFillUnusedMemory" , Opt::kFillUnusedMemory, 0, 0 }, { "kImmediateRelease" , Opt::kImmediateRelease, 0, 0 }, @@ -1410,22 +1499,29 @@ static void test_jit_allocator_alloc_release() noexcept { { "kUseLargePages" , Opt::kUseLargePages, 0, 0 }, { "kUseLargePages | kFillUnusedMemory" , Opt::kUseLargePages | Opt::kFillUnusedMemory, 0, 0 }, { "kUseLargePages | kAlignBlockSizeToLargePage", Opt::kUseLargePages | Opt::kAlignBlockSizeToLargePage, 0, 0 }, + { "kUseDualMapping" , Opt::kUseDualMapping , 0, 0 }, { "kUseDualMapping | kFillUnusedMemory" , Opt::kUseDualMapping | Opt::kFillUnusedMemory, 0, 0 } }; INFO("BitVectorRangeIterator"); { - Random rnd; + TestUtils::Random rnd; BitVectorRangeIterator_testRandom(rnd, kCount); } INFO("BitVectorRangeIterator"); { - Random rnd; + TestUtils::Random rnd; BitVectorRangeIterator_testRandom(rnd, kCount); } for (uint32_t testId = 0; testId < ASMJIT_ARRAY_SIZE(testParams); testId++) { + // Don't try to allocate dual-mapping if dual mapping is not possible - it would fail the test. + if (Support::test(testParams[testId].options, JitAllocatorOptions::kUseDualMapping) && + !Support::test(hri.flags, VirtMem::HardenedRuntimeFlags::kDualMapping)) { + continue; + } + INFO("JitAllocator(%s)", testParams[testId].name); JitAllocator::CreateParams params {}; @@ -1436,7 +1532,7 @@ static void test_jit_allocator_alloc_release() noexcept { size_t fixedBlockSize = 256; JitAllocatorWrapper wrapper(¶ms); - Random prng(100); + TestUtils::Random prng(100); size_t i; @@ -1447,80 +1543,94 @@ static void test_jit_allocator_alloc_release() noexcept { // Random blocks tests... INFO(" Allocating random blocks..."); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Releasing all allocated blocks from the beginning..."); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { wrapper.release(ptrArray[i]); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Allocating random blocks again...", kCount); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Shuffling allocated blocks..."); JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng); INFO(" Releasing 50%% of allocated blocks..."); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { wrapper.release(ptrArray[i]); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Allocating 50%% more blocks again..."); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Releasing all allocated blocks from the end..."); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { wrapper.release(ptrArray[kCount - i - 1]); + } JitAllocatorTest_usage(wrapper._allocator); // Fixed blocks tests... INFO(" Allocating %zuB blocks...", fixedBlockSize); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { ptrArray[i] = wrapper.alloc(fixedBlockSize); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Shrinking each %zuB block to 1 byte", fixedBlockSize); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { wrapper.shrink(ptrArray[i], 1); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Allocating more 64B blocks...", 64); - for (i = kCount / 2; i < kCount; i++) + for (i = kCount / 2; i < kCount; i++) { ptrArray[i] = wrapper.alloc(64); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Releasing all blocks from the beginning..."); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { wrapper.release(ptrArray[i]); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Allocating %zuB blocks...", fixedBlockSize); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { ptrArray[i] = wrapper.alloc(fixedBlockSize); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Shuffling allocated blocks..."); JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng); INFO(" Releasing 50%% of allocated blocks..."); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { wrapper.release(ptrArray[i]); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Allocating 50%% more %zuB blocks again...", fixedBlockSize); - for (i = 0; i < kCount / 2; i++) + for (i = 0; i < kCount / 2; i++) { ptrArray[i] = wrapper.alloc(fixedBlockSize); + } JitAllocatorTest_usage(wrapper._allocator); INFO(" Releasing all allocated blocks from the end..."); - for (i = 0; i < kCount; i++) + for (i = 0; i < kCount; i++) { wrapper.release(ptrArray[kCount - i - 1]); + } JitAllocatorTest_usage(wrapper._allocator); ::free(ptrArray); @@ -1544,6 +1654,7 @@ static void test_jit_allocator_query() noexcept { } UNIT(jit_allocator) { + test_jit_allocator_reset_empty(); test_jit_allocator_alloc_release(); test_jit_allocator_query(); } diff --git a/pe-packer/asmjit/core/jitallocator.h b/pe-packer/asmjit/core/jitallocator.h index 1a03823..4e4c49b 100644 --- a/pe-packer/asmjit/core/jitallocator.h +++ b/pe-packer/asmjit/core/jitallocator.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_JITALLOCATOR_H_INCLUDED @@ -65,7 +65,7 @@ enum class JitAllocatorOptions : uint32_t { //! since AsmJit can be compiled as a shared library and used by applications instrumented by UBSAN, it's not //! possible to conditionally compile the support only when necessary. //! - //! \important This flag controls a workaround to make it possible to use LLVM UBSAN with AsmJit's \ref JitAllocator. + //! \remarks This flag controls a workaround to make it possible to use LLVM UBSAN with AsmJit's \ref JitAllocator. //! There is no undefined behavior even when `kDisableInitialPadding` is used, however, that doesn't really matter //! as LLVM's UBSAN introduces one, and according to LLVM developers it's a "trade-off". This flag is safe to use //! when the code is not instrumented with LLVM's UBSAN. @@ -73,7 +73,7 @@ enum class JitAllocatorOptions : uint32_t { //! Enables the use of large pages, if they are supported and the process can actually allocate them. //! - //! \important This flag is a hint - if large pages can be allocated, JitAllocator would try to allocate them. + //! \remarks This flag is a hint - if large pages can be allocated, JitAllocator would try to allocate them. //! However, if the allocation fails, it will still try to fallback to use regular pages as \ref JitAllocator //! is designed to minimize allocation failures, so a regular page is better than no page at all. Also, if a //! block \ref JitAllocator wants to allocate is too small to consume a whole large page, regular page(s) will @@ -174,7 +174,7 @@ public: uint32_t fillPattern = 0; // Reset the content of `CreateParams`. - inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = CreateParams{}; } }; //! Creates a `JitAllocator` instance. @@ -182,6 +182,7 @@ public: //! Destroys the `JitAllocator` instance and release all blocks held. ASMJIT_API ~JitAllocator() noexcept; + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isInitialized() const noexcept { return _impl->blockSize == 0; } //! Free all allocated memory - makes all pointers returned by `alloc()` invalid. @@ -196,15 +197,23 @@ public: //! \{ //! Returns allocator options, see `Flags`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG JitAllocatorOptions options() const noexcept { return _impl->options; } + //! Tests whether the allocator has the given `option` set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasOption(JitAllocatorOptions option) const noexcept { return uint32_t(_impl->options & option) != 0; } //! Returns a base block size (a minimum size of block that the allocator would allocate). + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t blockSize() const noexcept { return _impl->blockSize; } + //! Returns granularity of the allocator. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t granularity() const noexcept { return _impl->granularity; } + //! Returns pattern that is used to fill unused memory if `kFlagUseFillPattern` is set. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t fillPattern() const noexcept { return _impl->fillPattern; } //! \} @@ -265,6 +274,7 @@ public: //! Returns a pointer having Read & Execute permissions (references executable memory). //! //! This pointer is never NULL if the allocation succeeded, it points to an executable memory. + [[nodiscard]] ASMJIT_INLINE_NODEBUG void* rx() const noexcept { return _rx; } //! Returns a pointer having Read & Write permissions (references writable memory). @@ -272,23 +282,27 @@ public: //! Depending on the type of the allocation strategy this could either be: //! //! - the same address as returned by `rx()` if the allocator uses RWX mapping (pages have all of Read, Write, - //! and Execute permissions) or MAP_JIT, which requires either \ref ProtectJitReadWriteScope or to call - //! VirtMem::protectJitMemory() manually. + //! and Execute permissions) or MAP_JIT, which requires either \ref VirtMem::ProtectJitReadWriteScope or to + //! call \ref VirtMem::protectJitMemory() manually. //! - a valid pointer, but not the same as `rx` - this would be valid if dual mapping is used. //! - NULL pointer, in case that the allocation strategy doesn't use RWX, MAP_JIT, or dual mapping. In this - //! case only \ref JitAllocator can copy new code into the executable memory referenced by \ref Addr. + //! case only \ref JitAllocator can copy new code into the executable memory referenced by \ref Span. //! //! \note If `rw()` returns a non-null pointer it's important to use either VirtMem::protectJitMemory() or - //! \ref ProtectJitReadWriteScope to guard the write, because in case of `MAP_JIT` it would temporarily switch - //! the permissions of the pointer to RW (that's per thread permissions). if \ref ProtectJitReadWriteScope is - //! not used it's important to clear the instruction cache via \ref VirtMem::flushInstructionCache() after the - //! write is done. + //! \ref VirtMem::ProtectJitReadWriteScope to guard the write, because in case of `MAP_JIT` it would temporarily + //! switch the permissions of the pointer to RW (that's per thread permissions). + //! + //! If \ref VirtMem::ProtectJitReadWriteScope is not used it's important to clear the instruction cache via + //! \ref VirtMem::flushInstructionCache() after the write is done. + [[nodiscard]] ASMJIT_INLINE_NODEBUG void* rw() const noexcept { return _rw; } //! Returns size of this span, aligned to the allocator granularity. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t size() const noexcept { return _size; } //! Returns span flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG Flags flags() const noexcept { return _flags; } //! Shrinks this span to `newSize`. @@ -299,12 +313,14 @@ public: ASMJIT_INLINE_NODEBUG void shrink(size_t newSize) noexcept { _size = Support::min(_size, newSize); } //! Returns whether \ref rw() returns a non-null pointer. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool isDirectlyWritable() const noexcept { return _rw != nullptr; } //! \} }; //! Allocates a new memory span of the requested `size`. + [[nodiscard]] ASMJIT_API Error alloc(Span& out, size_t size) noexcept; //! Releases a memory block returned by `alloc()`. @@ -320,43 +336,15 @@ public: //! Queries information about an allocated memory block that contains the given `rx`, and writes it to `out`. //! //! If the pointer is matched, the function returns `kErrorOk` and fills `out` with the corresponding span. + [[nodiscard]] ASMJIT_API Error query(Span& out, void* rx) const noexcept; -#if !defined(ASMJIT_NO_DEPRECATED) - //! Allocates a new memory block of the requested `size`. - ASMJIT_DEPRECATED("Use alloc(Span& out, size_t size) instead") - ASMJIT_FORCE_INLINE Error alloc(void** rxPtrOut, void** rwPtrOut, size_t size) noexcept { - Span span; - Error err = alloc(span, size); - *rwPtrOut = span.rw(); - *rxPtrOut = span.rx(); - return err; - } - - ASMJIT_DEPRECATED("Use shrink(Span& span, size_t newSize) instead") - ASMJIT_FORCE_INLINE Error shrink(void* rxPtr, size_t newSize) noexcept { - Span span; - ASMJIT_PROPAGATE(query(span, rxPtr)); - return (span.size() > newSize) ? shrink(span, newSize) : Error(kErrorOk); - } - - ASMJIT_DEPRECATED("Use query(Span& out, void* rx) instead") - ASMJIT_FORCE_INLINE Error query(void* rxPtr, void** rxPtrOut, void** rwPtrOut, size_t* sizeOut) const noexcept { - Span span; - Error err = query(span, rxPtr); - *rxPtrOut = span.rx(); - *rwPtrOut = span.rw(); - *sizeOut = span.size(); - return err; - } -#endif - //! \} //! \name Write Operations //! \{ - typedef Error (ASMJIT_CDECL* WriteFunc)(Span& span, void* userData) ASMJIT_NOEXCEPT_TYPE; + using WriteFunc = Error (ASMJIT_CDECL*)(Span& span, void* userData) noexcept; ASMJIT_API Error write( Span& span, @@ -372,7 +360,7 @@ public: VirtMem::CachePolicy policy = VirtMem::CachePolicy::kDefault) noexcept; template - ASMJIT_FORCE_INLINE Error write( + ASMJIT_INLINE Error write( Span& span, Lambda&& lambdaFunc, VirtMem::CachePolicy policy = VirtMem::CachePolicy::kDefault) noexcept { @@ -430,12 +418,12 @@ public: //! This is mostly for internal purposes, please use \ref WriteScope::write() instead. ASMJIT_API Error scopedWrite(WriteScopeData& scope, Span& span, size_t offset, const void* src, size_t size) noexcept; - //! Alternative to `JitAllocator::write(span. writeFunc, userData)`, but under a write `scope`. + //! Alternative to `JitAllocator::write(span, writeFunc, userData)`, but under a write `scope`. //! //! This is mostly for internal purposes, please use \ref WriteScope::write() instead. ASMJIT_API Error scopedWrite(WriteScopeData& scope, Span& span, WriteFunc writeFunc, void* userData) noexcept; - //! Alternative to `JitAllocator::write(span. )`, but under a write `scope`. + //! Alternative to `JitAllocator::write(span, [lambda])`, but under a write `scope`. //! //! This is mostly for internal purposes, please use \ref WriteScope::write() instead. template @@ -473,7 +461,12 @@ public: //! \name Accessors //! \{ + //! Returns \ref JitAllocator associated with this write scope. + [[nodiscard]] ASMJIT_INLINE_NODEBUG JitAllocator* allocator() const noexcept { return _allocator; } + + //! Returns cache policy this write scope is using. + [[nodiscard]] ASMJIT_INLINE_NODEBUG VirtMem::CachePolicy policy() const noexcept { return _policy; } //! \} @@ -527,27 +520,40 @@ public: ASMJIT_INLINE_NODEBUG void reset() noexcept { *this = Statistics{}; } //! Returns count of blocks managed by `JitAllocator` at the moment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t blockCount() const noexcept { return _blockCount; } + //! Returns the number of active allocations. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t allocationCount() const noexcept { return _allocationCount; } //! Returns how many bytes are currently used. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t usedSize() const noexcept { return _usedSize; } + //! Returns the number of bytes unused by the allocator at the moment. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t unusedSize() const noexcept { return _reservedSize - _usedSize; } - //! Returns the total number of bytes bytes reserved by the allocator (sum of sizes of all blocks). + + //! Returns the total number of bytes reserved by the allocator (sum of sizes of all blocks). + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t reservedSize() const noexcept { return _reservedSize; } + //! Returns the number of bytes the allocator needs to manage the allocated memory. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t overheadSize() const noexcept { return _overheadSize; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG double usedSizeAsPercent() const noexcept { return (double(usedSize()) / (double(reservedSize()) + 1e-16)) * 100.0; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG double unusedSizeAsPercent() const noexcept { return (double(unusedSize()) / (double(reservedSize()) + 1e-16)) * 100.0; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG double overheadSizeAsPercent() const noexcept { return (double(overheadSize()) / (double(reservedSize()) + 1e-16)) * 100.0; } @@ -556,6 +562,7 @@ public: //! Returns JIT allocator statistics. //! //! \remarks This function is thread-safe. + [[nodiscard]] ASMJIT_API Statistics statistics() const noexcept; //! \} diff --git a/pe-packer/asmjit/core/jitruntime.cpp b/pe-packer/asmjit/core/jitruntime.cpp index 0cc0269..cfb4c03 100644 --- a/pe-packer/asmjit/core/jitruntime.cpp +++ b/pe-packer/asmjit/core/jitruntime.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -24,17 +24,19 @@ Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { *dst = nullptr; ASMJIT_PROPAGATE(code->flatten()); - ASMJIT_PROPAGATE(code->resolveUnresolvedLinks()); + ASMJIT_PROPAGATE(code->resolveCrossSectionFixups()); size_t estimatedCodeSize = code->codeSize(); - if (ASMJIT_UNLIKELY(estimatedCodeSize == 0)) + if (ASMJIT_UNLIKELY(estimatedCodeSize == 0)) { return DebugUtils::errored(kErrorNoCodeGenerated); + } JitAllocator::Span span; ASMJIT_PROPAGATE(_allocator.alloc(span, estimatedCodeSize)); // Relocate the code. - Error err = code->relocateToBase(uintptr_t(span.rx())); + CodeHolder::RelocationSummary relocationSummary; + Error err = code->relocateToBase(uintptr_t(span.rx()), &relocationSummary); if (ASMJIT_UNLIKELY(err)) { _allocator.release(span.rx()); return err; @@ -42,8 +44,10 @@ Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { // Recalculate the final code size and shrink the memory we allocated for it // in case that some relocations didn't require records in an address table. - size_t codeSize = code->codeSize(); - ASMJIT_ASSERT(codeSize <= estimatedCodeSize); + size_t codeSize = estimatedCodeSize - relocationSummary.codeSizeReduction; + + // If not true it means that `relocateToBase()` filled wrong information in `relocationSummary`. + ASMJIT_ASSERT(codeSize == code->codeSize()); _allocator.write(span, [&](JitAllocator::Span& span) noexcept -> Error { uint8_t* rw = static_cast(span.rw()); diff --git a/pe-packer/asmjit/core/jitruntime.h b/pe-packer/asmjit/core/jitruntime.h index a8d0882..4b726ea 100644 --- a/pe-packer/asmjit/core/jitruntime.h +++ b/pe-packer/asmjit/core/jitruntime.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_JITRUNTIME_H_INCLUDED @@ -20,8 +20,10 @@ class CodeHolder; //! \addtogroup asmjit_virtual_memory //! \{ -//! JIT execution runtime is a special `Target` that is designed to store and -//! execute the generated code. +//! JIT execution runtime is a special `Target` that is designed to store and execute a generated code. +//! +//! JIT runtime is the easiest way of using AsmJit as it abstracts allocation and deallocation of virtual memory +//! where executable code can be placed and from which it can be executed as well. class ASMJIT_VIRTAPI JitRuntime : public Target { public: ASMJIT_NONCOPYABLE(JitRuntime) @@ -34,9 +36,24 @@ public: //! Creates a `JitRuntime` instance. ASMJIT_API explicit JitRuntime(const JitAllocator::CreateParams* params = nullptr) noexcept; - //! Destroys the `JitRuntime` instance. - ASMJIT_API virtual ~JitRuntime() noexcept; + //! Creates a `JitRuntime` instance. + ASMJIT_INLINE explicit JitRuntime(const JitAllocator::CreateParams& params) noexcept + : JitRuntime(¶ms) {} + + //! Destroys the `JitRuntime` instance. + ASMJIT_API ~JitRuntime() noexcept override; + + //! \} + + //! \name Accessors + //! \{ + + //! Resets the \ref JitRuntime, freeing everything that was allocated by it. + //! + //! Depending on `resetPolicy` the currently held memory can be either freed entirely when ResetPolicy::kHard is used, + //! or the allocator can keep some of it for next allocations when ResetPolicy::kSoft is used, which is the default + //! behavior. ASMJIT_INLINE_NODEBUG void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept { _allocator.reset(resetPolicy); } @@ -47,6 +64,7 @@ public: //! \{ //! Returns the associated `JitAllocator`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG JitAllocator* allocator() const noexcept { return const_cast(&_allocator); } //! \} diff --git a/pe-packer/asmjit/core/logger.cpp b/pe-packer/asmjit/core/logger.cpp index 9bc14ba..dfe4489 100644 --- a/pe-packer/asmjit/core/logger.cpp +++ b/pe-packer/asmjit/core/logger.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -52,11 +52,13 @@ FileLogger::FileLogger(FILE* file) noexcept FileLogger::~FileLogger() noexcept {} Error FileLogger::_log(const char* data, size_t size) noexcept { - if (!_file) + if (!_file) { return kErrorOk; + } - if (size == SIZE_MAX) + if (size == SIZE_MAX) { size = strlen(data); + } fwrite(data, 1, size, _file); return kErrorOk; diff --git a/pe-packer/asmjit/core/logger.h b/pe-packer/asmjit/core/logger.h index b599d2c..b91e951 100644 --- a/pe-packer/asmjit/core/logger.h +++ b/pe-packer/asmjit/core/logger.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_LOGGING_H_INCLUDED @@ -47,36 +47,53 @@ public: //! \{ //! Returns \ref FormatOptions of this logger. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FormatOptions& options() noexcept { return _options; } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const FormatOptions& options() const noexcept { return _options; } + //! Sets formatting options of this Logger to `options`. ASMJIT_INLINE_NODEBUG void setOptions(const FormatOptions& options) noexcept { _options = options; } + //! Resets formatting options of this Logger to defaults. ASMJIT_INLINE_NODEBUG void resetOptions() noexcept { _options.reset(); } //! Returns formatting flags. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FormatFlags flags() const noexcept { return _options.flags(); } + //! Tests whether the logger has the given `flag` enabled. + [[nodiscard]] ASMJIT_INLINE_NODEBUG bool hasFlag(FormatFlags flag) const noexcept { return _options.hasFlag(flag); } + //! Sets formatting flags to `flags`. ASMJIT_INLINE_NODEBUG void setFlags(FormatFlags flags) noexcept { _options.setFlags(flags); } + //! Enables the given formatting `flags`. ASMJIT_INLINE_NODEBUG void addFlags(FormatFlags flags) noexcept { _options.addFlags(flags); } + //! Disables the given formatting `flags`. ASMJIT_INLINE_NODEBUG void clearFlags(FormatFlags flags) noexcept { _options.clearFlags(flags); } //! Returns indentation of a given indentation `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t indentation(FormatIndentationGroup type) const noexcept { return _options.indentation(type); } + //! Sets indentation of the given indentation `group` to `n` spaces. ASMJIT_INLINE_NODEBUG void setIndentation(FormatIndentationGroup type, uint32_t n) noexcept { _options.setIndentation(type, n); } + //! Resets indentation of the given indentation `group` to 0 spaces. ASMJIT_INLINE_NODEBUG void resetIndentation(FormatIndentationGroup type) noexcept { _options.resetIndentation(type); } //! Returns padding of a given padding `group`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t padding(FormatPaddingGroup type) const noexcept { return _options.padding(type); } + //! Sets padding of a given padding `group` to `n`. ASMJIT_INLINE_NODEBUG void setPadding(FormatPaddingGroup type, uint32_t n) noexcept { _options.setPadding(type, n); } + //! Resets padding of a given padding `group` to 0, which means that a default will be used. ASMJIT_INLINE_NODEBUG void resetPadding(FormatPaddingGroup type) noexcept { _options.resetPadding(type); } @@ -90,7 +107,7 @@ public: //! The function can accept either a null terminated string if `size` is `SIZE_MAX` or a non-null terminated //! string of the given `size`. The function cannot assume that the data is null terminated and must handle //! non-null terminated inputs. - ASMJIT_API virtual Error _log(const char* data, size_t size) noexcept = 0; + ASMJIT_API virtual Error _log(const char* data, size_t size) noexcept; //! Logs string `str`, which is either null terminated or having size `size`. ASMJIT_INLINE_NODEBUG Error log(const char* data, size_t size = SIZE_MAX) noexcept { return _log(data, size); } @@ -119,7 +136,7 @@ public: //! Creates a new `FileLogger` that logs to `FILE*`. ASMJIT_API FileLogger(FILE* file = nullptr) noexcept; //! Destroys the `FileLogger`. - ASMJIT_API virtual ~FileLogger() noexcept; + ASMJIT_API ~FileLogger() noexcept override; //! \} @@ -127,6 +144,7 @@ public: //! \{ //! Returns the logging output stream or null if the logger has no output stream. + [[nodiscard]] ASMJIT_INLINE_NODEBUG FILE* file() const noexcept { return _file; } //! Sets the logging output stream to `stream` or null. @@ -155,7 +173,7 @@ public: //! Create new `StringLogger`. ASMJIT_API StringLogger() noexcept; //! Destroys the `StringLogger`. - ASMJIT_API virtual ~StringLogger() noexcept; + ASMJIT_API ~StringLogger() noexcept override; //! \} @@ -165,15 +183,21 @@ public: //! Returns the content of the logger as \ref String. //! //! It can be moved, if desired. + [[nodiscard]] ASMJIT_INLINE_NODEBUG String& content() noexcept { return _content; } + //! \overload + [[nodiscard]] ASMJIT_INLINE_NODEBUG const String& content() const noexcept { return _content; } //! Returns aggregated logger data as `char*` pointer. //! //! The pointer is owned by `StringLogger`, it can't be modified or freed. + [[nodiscard]] ASMJIT_INLINE_NODEBUG const char* data() const noexcept { return _content.data(); } + //! Returns size of the data returned by `data()`. + [[nodiscard]] ASMJIT_INLINE_NODEBUG size_t dataSize() const noexcept { return _content.size(); } //! \} diff --git a/pe-packer/asmjit/core/misc_p.h b/pe-packer/asmjit/core/misc_p.h index 5cd934e..284b9eb 100644 --- a/pe-packer/asmjit/core/misc_p.h +++ b/pe-packer/asmjit/core/misc_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_MISC_P_H_INCLUDED diff --git a/pe-packer/asmjit/core/operand.cpp b/pe-packer/asmjit/core/operand.cpp index b78dc54..252d378 100644 --- a/pe-packer/asmjit/core/operand.cpp +++ b/pe-packer/asmjit/core/operand.cpp @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" @@ -20,7 +20,7 @@ enum class StrongEnumForImmTests : uint32_t { UNIT(operand) { INFO("Checking operand sizes"); EXPECT_EQ(sizeof(Operand), 16u); - EXPECT_EQ(sizeof(BaseReg), 16u); + EXPECT_EQ(sizeof(Reg), 16u); EXPECT_EQ(sizeof(BaseMem), 16u); EXPECT_EQ(sizeof(Imm), 16u); EXPECT_EQ(sizeof(Label), 16u); @@ -43,19 +43,19 @@ UNIT(operand) { EXPECT_FALSE(label.isValid()); EXPECT_EQ(label.id(), Globals::kInvalidId); - INFO("Checking basic functionality of BaseReg"); - EXPECT_TRUE(BaseReg().isReg()); - EXPECT_FALSE(BaseReg().isValid()); - EXPECT_EQ(BaseReg()._data[0], 0u); - EXPECT_EQ(BaseReg()._data[1], 0u); - EXPECT_FALSE(dummy.as().isValid()); + INFO("Checking basic functionality of Reg"); + EXPECT_TRUE(Reg().isReg()); + EXPECT_FALSE(Reg().isValid()); + EXPECT_EQ(Reg()._data[0], 0u); + EXPECT_EQ(Reg()._data[1], 0u); + EXPECT_FALSE(dummy.as().isValid()); // Create some register (not specific to any architecture). OperandSignature rSig = OperandSignature::fromOpType(OperandType::kReg) | OperandSignature::fromRegType(RegType::kVec128) | OperandSignature::fromRegGroup(RegGroup::kVec) | OperandSignature::fromSize(8); - BaseReg r1(rSig, 5); + Reg r1(rSig, 5); EXPECT_TRUE(r1.isValid()); EXPECT_TRUE(r1.isReg()); @@ -63,8 +63,8 @@ UNIT(operand) { EXPECT_TRUE(r1.isPhysReg()); EXPECT_FALSE(r1.isVirtReg()); EXPECT_EQ(r1.signature(), rSig); - EXPECT_EQ(r1.type(), RegType::kVec128); - EXPECT_EQ(r1.group(), RegGroup::kVec); + EXPECT_EQ(r1.regType(), RegType::kVec128); + EXPECT_EQ(r1.regGroup(), RegGroup::kVec); EXPECT_EQ(r1.size(), 8u); EXPECT_EQ(r1.id(), 5u); EXPECT_TRUE(r1.isReg(RegType::kVec128, 5)); // RegType and Id. @@ -72,15 +72,15 @@ UNIT(operand) { EXPECT_EQ(r1._data[1], 0u); // The same type of register having different id. - BaseReg r2(r1, 6); + Reg r2(r1, 6); EXPECT_TRUE(r2.isValid()); EXPECT_TRUE(r2.isReg()); EXPECT_TRUE(r2.isReg(RegType::kVec128)); EXPECT_TRUE(r2.isPhysReg()); EXPECT_FALSE(r2.isVirtReg()); EXPECT_EQ(r2.signature(), rSig); - EXPECT_EQ(r2.type(), r1.type()); - EXPECT_EQ(r2.group(), r1.group()); + EXPECT_EQ(r2.regType(), r1.regType()); + EXPECT_EQ(r2.regGroup(), r1.regGroup()); EXPECT_EQ(r2.size(), r1.size()); EXPECT_EQ(r2.id(), 6u); EXPECT_TRUE(r2.isReg(RegType::kVec128, 6)); diff --git a/pe-packer/asmjit/core/operand.h b/pe-packer/asmjit/core/operand.h index 6ffde24..6ba988f 100644 --- a/pe-packer/asmjit/core/operand.h +++ b/pe-packer/asmjit/core/operand.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OPERAND_H_INCLUDED @@ -23,22 +23,24 @@ enum class OperandType : uint32_t { kReg = 1, //! Operand is a memory. kMem = 2, + //! Operand is a register-list. + kRegList = 3, //! Operand is an immediate value. - kImm = 3, + kImm = 4, //! Operand is a label. - kLabel = 4, + kLabel = 5, //! Maximum value of `OperandType`. - kMaxValue = kLabel + kMaxValue = kRegList }; static_assert(uint32_t(OperandType::kMem) == uint32_t(OperandType::kReg) + 1, - "AsmJit requires that `OperandType::kMem` equals to `OperandType::kReg + 1`"); + "AsmJit requires that `OperandType::kMem` equals `OperandType::kReg + 1`"); //! Register mask is a convenience typedef that describes a mask where each bit describes a physical register id //! in the same \ref RegGroup. At the moment 32 bits are enough as AsmJit doesn't support any architecture that //! would provide more than 32 registers for a register group. -typedef uint32_t RegMask; +using RegMask = uint32_t; //! Register type. //! @@ -47,114 +49,72 @@ enum class RegType : uint8_t { //! No register - unused, invalid, multiple meanings. kNone = 0, - //! This is not a register type. This value is reserved for a \ref Label that used in \ref BaseMem as a base. + //! This is not a register type. This value is reserved for a \ref Label that's used in \ref BaseMem as a base. //! //! Label tag is used as a sub-type, forming a unique signature across all operand types as 0x1 is never associated //! with any register type. This means that a memory operand's BASE register can be constructed from virtually any //! operand (register vs. label) by just assigning its type (register type or label-tag) and operand id. kLabelTag = 1, - //! Universal type describing program counter (PC) or instruction pointer (IP) register, if the target architecture - //! actually exposes it as a separate register type, which most modern targets do. - kPC = 2, + //! 8-bit low general purpose register (X86|X86_64). + kGp8Lo = 2, + //! 8-bit high general purpose register (X86|X86_64). + kGp8Hi = 3, + //! 16-bit general purpose register (X86|X86_64). + kGp16 = 4, + //! 32-bit general purpose register (X86|X86_64|AArch32|AArch64). + kGp32 = 5, + //! 64-bit general purpose register (X86_64|AArch64). + kGp64 = 6, - //! 8-bit low general purpose register (X86). - kGp8Lo = 3, - //! 8-bit high general purpose register (X86). - kGp8Hi = 4, - //! 16-bit general purpose register (X86). - kGp16 = 5, - //! 32-bit general purpose register (X86|ARM). - kGp32 = 6, - //! 64-bit general purpose register (X86|ARM). - kGp64 = 7, - //! 8-bit view of a vector register (ARM). - kVec8 = 8, - //! 16-bit view of a vector register (ARM). - kVec16 = 9, - //! 32-bit view of a vector register (ARM). - kVec32 = 10, - //! 64-bit view of a vector register (ARM). + //! 8-bit view of a vector register (AArch64). + kVec8 = 7, + //! 16-bit view of a vector register (AArch64). + kVec16 = 8, + //! 32-bit view of a vector register (AArch32|AArch64). + kVec32 = 9, + //! 64-bit view of a vector register (AArch32|AArch64). //! //! \note This is never used for MMX registers on X86, MMX registers have its own category. - kVec64 = 11, - //! 128-bit view of a vector register (X86|ARM). - kVec128 = 12, - //! 256-bit view of a vector register (X86). - kVec256 = 13, - //! 512-bit view of a vector register (X86). - kVec512 = 14, + kVec64 = 10, + //! 128-bit view of a vector register (X86|X86_64|AArch32|AArch64). + kVec128 = 11, + //! 256-bit view of a vector register (X86|X86_64). + kVec256 = 12, + //! 512-bit view of a vector register (X86|X86_64). + kVec512 = 13, //! 1024-bit view of a vector register (future). - kVec1024 = 15, + kVec1024 = 14, //! View of a vector register, which width is implementation specific (AArch64). - kVecNLen = 16, + kVecNLen = 15, - //! Mask register (X86). - kMask = 17, + //! Mask register (X86|X86_64|AArch64). + kMask = 16, + //! Tile register (X86_64: `TMM`). + kTile = 17, - //! Start of architecture dependent register types. - kExtra = 18, + //! Segment register (X86|X86_64: None, ES, CS, SS, DS, FS, GS). + kSegment = 25, + //! Control register (X86|X86_64: `CR`). + kControl = 26, + //! Debug register (X86|X86_64: `DR`). + kDebug = 27, - // X86 Specific Register Types - // --------------------------- - - // X86 Specific Register Types - // =========================== + //! MMX register (X86|X86_64: `MM`). + kX86_Mm = 28, + //! FPU (x87) register (X86|X86_64: `ST`). + kX86_St = 29, + //! Bound register (X86|X86_64: `BND`). + kX86_Bnd = 30, + //! Universal type describing program counter (PC) or instruction pointer (EIP/RIP) register, if the target + //! architecture actually exposes it as a separate register type, which most modern architectures do. + //! + //! X86 Specific + //! ------------ + //! //! Instruction pointer (RIP), only addressable in \ref x86::Mem in 64-bit targets. - kX86_Rip = kPC, - //! Low GPB register (AL, BL, CL, DL, ...). - kX86_GpbLo = kGp8Lo, - //! High GPB register (AH, BH, CH, DH only). - kX86_GpbHi = kGp8Hi, - //! GPW register. - kX86_Gpw = kGp16, - //! GPD register. - kX86_Gpd = kGp32, - //! GPQ register (64-bit). - kX86_Gpq = kGp64, - //! XMM register (SSE+). - kX86_Xmm = kVec128, - //! YMM register (AVX+). - kX86_Ymm = kVec256, - //! ZMM register (AVX512+). - kX86_Zmm = kVec512, - //! K register (AVX512+). - kX86_KReg = kMask, - //! MMX register. - kX86_Mm = kExtra + 0, - //! Segment register (None, ES, CS, SS, DS, FS, GS). - kX86_SReg = kExtra + 1, - //! Control register (CR). - kX86_CReg = kExtra + 2, - //! Debug register (DR). - kX86_DReg = kExtra + 3, - //! FPU (x87) register. - kX86_St = kExtra + 4, - //! Bound register (BND). - kX86_Bnd = kExtra + 5, - //! TMM register (AMX_TILE) - kX86_Tmm = kExtra + 6, - - // ARM Specific Register Types - // =========================== - - //! Program pointer (PC) register (AArch64). - kARM_PC = kPC, - //! 32-bit general purpose register (R or W). - kARM_GpW = kGp32, - //! 64-bit general purpose register (X). - kARM_GpX = kGp64, - //! 8-bit view of VFP/ASIMD register (B). - kARM_VecB = kVec8, - //! 16-bit view of VFP/ASIMD register (H). - kARM_VecH = kVec16, - //! 32-bit view of VFP/ASIMD register (S). - kARM_VecS = kVec32, - //! 64-bit view of VFP/ASIMD register (D). - kARM_VecD = kVec64, - //! 128-bit view of VFP/ASIMD register (Q|V). - kARM_VecV = kVec128, + kPC = 31, //! Maximum value of `RegType`. kMaxValue = 31 @@ -167,55 +127,51 @@ ASMJIT_DEFINE_ENUM_COMPARE(RegType) enum class RegGroup : uint8_t { //! General purpose register group compatible with all backends. kGp = 0, + //! Vector register group compatible with all backends. //! - //! Describes X86 XMM|YMM|ZMM registers ARM/AArch64 V registers. + //! Describes `XMM|YMM|ZMM` registers on X86|X86_64 targets and `V|Q|D|S|H|B` registers on ARM/AArch64 targets. kVec = 1, - //! Extra virtual group #2 that can be used by Compiler for register allocation. - kExtraVirt2 = 2, + //! Mask register group compatible with all backends that can use masking. + //! + //! Describes `K` registers on X86|X86_64 targets (AVX-512) and `P` registers on AArch64 targets (SVE/SVE2). + kMask = 2, + //! Extra virtual group #3 that can be used by Compiler for register allocation. kExtraVirt3 = 3, - //! Program counter group. - kPC = 4, + //! TMM register group (X86|X86_64). + kTile = 4, - //! Extra non-virtual group that can be used by registers not managed by Compiler. - kExtraNonVirt = 5, + //! Segment register group (X86|X86_64). + kSegment = 10, - // X86 Specific Register Groups - // ---------------------------- + //! Control register group (X86|X86_64). + kControl = 11, - //! K register group (KReg) - maps to \ref RegGroup::kExtraVirt2 (X86, X86_64). - kX86_K = kExtraVirt2, - //! MMX register group (MM) - maps to \ref RegGroup::kExtraVirt3 (X86, X86_64). + //! Debug register group (X86|X86_64). + kDebug = 12, + + //! MMX register group (MM) - maps to \ref RegGroup::kExtraVirt3 (X86|X86_64). kX86_MM = kExtraVirt3, + //! FPU register group (X86|X86_64). + kX86_St = 13, + //! BND register group (X86|X86_64). + kX86_Bnd = 14, - //! Instruction pointer (X86, X86_64). - kX86_Rip = kPC, - //! Segment register group (X86, X86_64). - kX86_SReg = kExtraNonVirt + 0, - //! CR register group (X86, X86_64). - kX86_CReg = kExtraNonVirt + 1, - //! DR register group (X86, X86_64). - kX86_DReg = kExtraNonVirt + 2, - //! FPU register group (X86, X86_64). - kX86_St = kExtraNonVirt + 3, - //! BND register group (X86, X86_64). - kX86_Bnd = kExtraNonVirt + 4, - //! TMM register group (X86, X86_64). - kX86_Tmm = kExtraNonVirt + 5, + //! Program counter group (represents also EIP/RIP on X86|X86_64 targets). + kPC = 15, - //! First group - only used in loops. - k0 = 0, - //! Last value of a virtual register that is managed by \ref BaseCompiler. - kMaxVirt = Globals::kNumVirtGroups - 1, //! Maximum value of `RegGroup`. - kMaxValue = 15 + kMaxValue = 15, + + //! Last value of a virtual register that is managed by \ref BaseCompiler. + kMaxVirt = Globals::kNumVirtGroups - 1 }; ASMJIT_DEFINE_ENUM_COMPARE(RegGroup) -typedef Support::EnumValues RegGroupVirtValues; +using RegGroupVirtValues = Support::EnumValues; //! Operand signature is a 32-bit number describing \ref Operand and some of its payload. //! @@ -227,57 +183,55 @@ struct OperandSignature { //! \name Constants //! \{ - enum : uint32_t { - // Operand type (3 least significant bits). - // |........|........|........|.....XXX| - kOpTypeShift = 0, - kOpTypeMask = 0x07u << kOpTypeShift, + // Operand type (3 least significant bits). + // |........|........|........|.....XXX| + static inline constexpr uint32_t kOpTypeShift = 0; + static inline constexpr uint32_t kOpTypeMask = 0x07u << kOpTypeShift; - // Register type (5 bits). - // |........|........|........|XXXXX...| - kRegTypeShift = 3, - kRegTypeMask = 0x1Fu << kRegTypeShift, + // Register type (5 bits). + // |........|........|........|XXXXX...| + static inline constexpr uint32_t kRegTypeShift = 3; + static inline constexpr uint32_t kRegTypeMask = 0x1Fu << kRegTypeShift; - // Register group (4 bits). - // |........|........|....XXXX|........| - kRegGroupShift = 8, - kRegGroupMask = 0x0Fu << kRegGroupShift, + // Register group (4 bits). + // |........|........|....XXXX|........| + static inline constexpr uint32_t kRegGroupShift = 8; + static inline constexpr uint32_t kRegGroupMask = 0x0Fu << kRegGroupShift; - // Memory base type (5 bits). - // |........|........|........|XXXXX...| - kMemBaseTypeShift = 3, - kMemBaseTypeMask = 0x1Fu << kMemBaseTypeShift, + // Memory base type (5 bits). + // |........|........|........|XXXXX...| + static inline constexpr uint32_t kMemBaseTypeShift = 3; + static inline constexpr uint32_t kMemBaseTypeMask = 0x1Fu << kMemBaseTypeShift; - // Memory index type (5 bits). - // |........|........|...XXXXX|........| - kMemIndexTypeShift = 8, - kMemIndexTypeMask = 0x1Fu << kMemIndexTypeShift, + // Memory index type (5 bits). + // |........|........|...XXXXX|........| + static inline constexpr uint32_t kMemIndexTypeShift = 8; + static inline constexpr uint32_t kMemIndexTypeMask = 0x1Fu << kMemIndexTypeShift; - // Memory base+index combined (10 bits). - // |........|........|...XXXXX|XXXXX...| - kMemBaseIndexShift = 3, - kMemBaseIndexMask = 0x3FFu << kMemBaseIndexShift, + // Memory base+index combined (10 bits). + // |........|........|...XXXXX|XXXXX...| + static inline constexpr uint32_t kMemBaseIndexShift = 3; + static inline constexpr uint32_t kMemBaseIndexMask = 0x3FFu << kMemBaseIndexShift; - // This memory operand represents a home-slot or stack (Compiler) (1 bit). - // |........|........|..X.....|........| - kMemRegHomeShift = 13, - kMemRegHomeFlag = 0x01u << kMemRegHomeShift, + // This memory operand represents a home-slot or stack (Compiler) (1 bit). + // |........|........|..X.....|........| + static inline constexpr uint32_t kMemRegHomeShift = 13; + static inline constexpr uint32_t kMemRegHomeFlag = 0x01u << kMemRegHomeShift; - // Immediate type (1 bit). - // |........|........|........|....X...| - kImmTypeShift = 3, - kImmTypeMask = 0x01u << kImmTypeShift, + // Immediate type (1 bit). + // |........|........|........|....X...| + static inline constexpr uint32_t kImmTypeShift = 3; + static inline constexpr uint32_t kImmTypeMask = 0x01u << kImmTypeShift; - // Predicate used by either registers or immediate values (4 bits). - // |........|XXXX....|........|........| - kPredicateShift = 20, - kPredicateMask = 0x0Fu << kPredicateShift, + // Predicate used by either registers or immediate values (4 bits). + // |........|XXXX....|........|........| + static inline constexpr uint32_t kPredicateShift = 20; + static inline constexpr uint32_t kPredicateMask = 0x0Fu << kPredicateShift; - // Operand size (8 most significant bits). - // |XXXXXXXX|........|........|........| - kSizeShift = 24, - kSizeMask = 0xFFu << kSizeShift - }; + // Operand size (8 most significant bits). + // |XXXXXXXX|........|........|........| + static inline constexpr uint32_t kSizeShift = 24; + static inline constexpr uint32_t kSizeMask = 0xFFu << kSizeShift; //! \} @@ -288,158 +242,234 @@ struct OperandSignature { //! \} + //! \name Static Constructors + //! \{ + + //! Constructs operand signature from the given `bits`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromBits(uint32_t bits) noexcept { + return OperandSignature{bits}; + } + + //! Constructs operand signature from the given `value`, use `kFieldMask` to describe where the value is in the signature. + template + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromValue(const T& value) noexcept { + return OperandSignature{uint32_t(value) << Support::ConstCTZ::value}; + } + + //! Constructs operand signature describing the given operand type `opType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromOpType(OperandType opType) noexcept { + return OperandSignature{uint32_t(opType) << kOpTypeShift}; + } + + //! Constructs operand signature describing the given register type `regType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegType(RegType regType) noexcept { + return OperandSignature{uint32_t(regType) << kRegTypeShift}; + } + + //! Constructs operand signature describing the given register group `regGroup`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegGroup(RegGroup regGroup) noexcept { + return OperandSignature{uint32_t(regGroup) << kRegGroupShift}; + } + + //! Constructs operand signature describing both register type `regType` and register group `regGroup`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromRegTypeAndGroup(RegType regType, RegGroup regGroup) noexcept { + return fromRegType(regType) | fromRegGroup(regGroup); + } + + //! Constructs operand signature describing a memory base type `baseType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemBaseType(RegType baseType) noexcept { + return OperandSignature{uint32_t(baseType) << kMemBaseTypeShift}; + } + + //! Constructs operand signature describing a memory index type `indexType`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromMemIndexType(RegType indexType) noexcept { + return OperandSignature{uint32_t(indexType) << kMemIndexTypeShift}; + } + + //! Constructs operand signature describing a `predicate`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromPredicate(uint32_t predicate) noexcept { + return OperandSignature{predicate << kPredicateShift}; + } + + //! Constructs operand signature describing a `size`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR OperandSignature fromSize(uint32_t size) noexcept { + return OperandSignature{size << kSizeShift}; + } + + //! \} + //! \name Overloaded Operators //! //! Overloaded operators make `OperandSignature` behave like regular integer. //! //! \{ - ASMJIT_INLINE_NODEBUG constexpr bool operator!() const noexcept { return _bits == 0; } - ASMJIT_INLINE_NODEBUG constexpr explicit operator bool() const noexcept { return _bits != 0; } + ASMJIT_INLINE_CONSTEXPR bool operator!() const noexcept { return _bits == 0; } + ASMJIT_INLINE_CONSTEXPR explicit operator bool() const noexcept { return _bits != 0; } - ASMJIT_INLINE_NODEBUG OperandSignature& operator|=(uint32_t x) noexcept { _bits |= x; return *this; } - ASMJIT_INLINE_NODEBUG OperandSignature& operator&=(uint32_t x) noexcept { _bits &= x; return *this; } - ASMJIT_INLINE_NODEBUG OperandSignature& operator^=(uint32_t x) noexcept { _bits ^= x; return *this; } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator|=(uint32_t x) noexcept { _bits |= x; return *this; } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator&=(uint32_t x) noexcept { _bits &= x; return *this; } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator^=(uint32_t x) noexcept { _bits ^= x; return *this; } - ASMJIT_INLINE_NODEBUG OperandSignature& operator|=(const OperandSignature& other) noexcept { return operator|=(other._bits); } - ASMJIT_INLINE_NODEBUG OperandSignature& operator&=(const OperandSignature& other) noexcept { return operator&=(other._bits); } - ASMJIT_INLINE_NODEBUG OperandSignature& operator^=(const OperandSignature& other) noexcept { return operator^=(other._bits); } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator|=(const OperandSignature& other) noexcept { return operator|=(other._bits); } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator&=(const OperandSignature& other) noexcept { return operator&=(other._bits); } + ASMJIT_INLINE_CONSTEXPR OperandSignature& operator^=(const OperandSignature& other) noexcept { return operator^=(other._bits); } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator~() const noexcept { return OperandSignature{~_bits}; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator~() const noexcept { return OperandSignature{~_bits}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator|(uint32_t x) const noexcept { return OperandSignature{_bits | x}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator&(uint32_t x) const noexcept { return OperandSignature{_bits & x}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator^(uint32_t x) const noexcept { return OperandSignature{_bits ^ x}; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator|(uint32_t x) const noexcept { return OperandSignature{_bits | x}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator|(const OperandSignature& other) const noexcept { return OperandSignature{_bits | other._bits}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator&(const OperandSignature& other) const noexcept { return OperandSignature{_bits & other._bits}; } - ASMJIT_INLINE_NODEBUG constexpr OperandSignature operator^(const OperandSignature& other) const noexcept { return OperandSignature{_bits ^ other._bits}; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator&(uint32_t x) const noexcept { return OperandSignature{_bits & x}; } - ASMJIT_INLINE_NODEBUG constexpr bool operator==(uint32_t x) const noexcept { return _bits == x; } - ASMJIT_INLINE_NODEBUG constexpr bool operator!=(uint32_t x) const noexcept { return _bits != x; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator^(uint32_t x) const noexcept { return OperandSignature{_bits ^ x}; } - ASMJIT_INLINE_NODEBUG constexpr bool operator==(const OperandSignature& other) const noexcept { return _bits == other._bits; } - ASMJIT_INLINE_NODEBUG constexpr bool operator!=(const OperandSignature& other) const noexcept { return _bits != other._bits; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator|(const OperandSignature& other) const noexcept { return OperandSignature{_bits | other._bits}; } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator&(const OperandSignature& other) const noexcept { return OperandSignature{_bits & other._bits}; } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature operator^(const OperandSignature& other) const noexcept { return OperandSignature{_bits ^ other._bits}; } + + ASMJIT_INLINE_CONSTEXPR bool operator==(uint32_t x) const noexcept { return _bits == x; } + ASMJIT_INLINE_CONSTEXPR bool operator!=(uint32_t x) const noexcept { return _bits != x; } + + ASMJIT_INLINE_CONSTEXPR bool operator==(const OperandSignature& other) const noexcept { return _bits == other._bits; } + ASMJIT_INLINE_CONSTEXPR bool operator!=(const OperandSignature& other) const noexcept { return _bits != other._bits; } //! \} //! \name Accessors //! \{ - ASMJIT_INLINE_NODEBUG void reset() noexcept { _bits = 0; } + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { _bits = 0; } - ASMJIT_INLINE_NODEBUG constexpr uint32_t bits() const noexcept { return _bits; } - ASMJIT_INLINE_NODEBUG void setBits(uint32_t bits) noexcept { _bits = bits; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t bits() const noexcept { return _bits; } - template::value> - ASMJIT_INLINE_NODEBUG constexpr bool hasField() const noexcept { + ASMJIT_INLINE_CONSTEXPR void setBits(uint32_t bits) noexcept { _bits = bits; } + + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasField() const noexcept { return (_bits & kFieldMask) != 0; } - template::value> - ASMJIT_INLINE_NODEBUG constexpr bool hasField(uint32_t value) const noexcept { - return (_bits & kFieldMask) != value << kFieldShift; + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasField(uint32_t value) const noexcept { + return (_bits & kFieldMask) != value << Support::ConstCTZ::value; } - template::value> - ASMJIT_INLINE_NODEBUG constexpr uint32_t getField() const noexcept { - return (_bits >> kFieldShift) & (kFieldMask >> kFieldShift); - } - - template::value> - ASMJIT_INLINE_NODEBUG void setField(uint32_t value) noexcept { - ASMJIT_ASSERT((value & ~(kFieldMask >> kFieldShift)) == 0); - _bits = (_bits & ~kFieldMask) | (value << kFieldShift); - } - - ASMJIT_INLINE_NODEBUG constexpr OperandSignature subset(uint32_t mask) const noexcept { return OperandSignature{_bits & mask}; } - - template::value> - ASMJIT_INLINE_NODEBUG constexpr OperandSignature replacedValue(uint32_t value) const noexcept { return OperandSignature{(_bits & ~kFieldMask) | (value << kFieldShift)}; } - template - ASMJIT_INLINE_NODEBUG constexpr bool matchesSignature(const OperandSignature& signature) const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t getField() const noexcept { + return (_bits >> Support::ConstCTZ::value) & (kFieldMask >> Support::ConstCTZ::value); + } + + template + ASMJIT_INLINE_CONSTEXPR void setField(uint32_t value) noexcept { + ASMJIT_ASSERT(((value << Support::ConstCTZ::value) & ~kFieldMask) == 0); + _bits = (_bits & ~kFieldMask) | (value << Support::ConstCTZ::value); + } + + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature subset(uint32_t mask) const noexcept { return OperandSignature{_bits & mask}; } + + template::value> + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature replacedValue(uint32_t value) const noexcept { return OperandSignature{(_bits & ~kFieldMask) | (value << kFieldShift)}; } + + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool matchesSignature(const OperandSignature& signature) const noexcept { return (_bits & kFieldMask) == signature._bits; } template - ASMJIT_INLINE_NODEBUG constexpr bool matchesFields(uint32_t bits) const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool matchesFields(uint32_t bits) const noexcept { return (_bits & kFieldMask) == bits; } template - ASMJIT_INLINE_NODEBUG constexpr bool matchesFields(const OperandSignature& fields) const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool matchesFields(const OperandSignature& fields) const noexcept { return (_bits & kFieldMask) == fields._bits; } - ASMJIT_INLINE_NODEBUG constexpr bool isValid() const noexcept { return _bits != 0; } + //! Tests whether the operand signature is valid (describes a valid operand, and not \ref OperandType::kNone. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return _bits != 0; } - ASMJIT_INLINE_NODEBUG constexpr OperandType opType() const noexcept { return (OperandType)getField(); } + //! Returns operand type this operand signature describes. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandType opType() const noexcept { return (OperandType)getField(); } - ASMJIT_INLINE_NODEBUG constexpr RegType regType() const noexcept { return (RegType)getField(); } - ASMJIT_INLINE_NODEBUG constexpr RegGroup regGroup() const noexcept { return (RegGroup)getField(); } + //! Tests whether the operand type matches opType + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isOpType(OperandType opType) const noexcept { return getField() == uint32_t(opType); } - ASMJIT_INLINE_NODEBUG constexpr RegType memBaseType() const noexcept { return (RegType)getField(); } - ASMJIT_INLINE_NODEBUG constexpr RegType memIndexType() const noexcept { return (RegType)getField(); } + //! Tests whether the operand signature represents a register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return isOpType(OperandType::kReg); } - ASMJIT_INLINE_NODEBUG constexpr uint32_t predicate() const noexcept { return getField(); } - ASMJIT_INLINE_NODEBUG constexpr uint32_t size() const noexcept { return getField(); } - - ASMJIT_INLINE_NODEBUG void setOpType(OperandType opType) noexcept { setField(uint32_t(opType)); } - ASMJIT_INLINE_NODEBUG void setRegType(RegType regType) noexcept { setField(uint32_t(regType)); } - ASMJIT_INLINE_NODEBUG void setRegGroup(RegGroup regGroup) noexcept { setField(uint32_t(regGroup)); } - - ASMJIT_INLINE_NODEBUG void setMemBaseType(RegGroup baseType) noexcept { setField(uint32_t(baseType)); } - ASMJIT_INLINE_NODEBUG void setMemIndexType(RegGroup indexType) noexcept { setField(uint32_t(indexType)); } - - ASMJIT_INLINE_NODEBUG void setPredicate(uint32_t predicate) noexcept { setField(predicate); } - ASMJIT_INLINE_NODEBUG void setSize(uint32_t size) noexcept { setField(size); } - - //! \} - - //! \name Static Constructors - //! \{ - - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromBits(uint32_t bits) noexcept { - return OperandSignature{bits}; + //! Tests whether the operand signature represents a register of the given register type `regType`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { + constexpr uint32_t kMask = kOpTypeMask | kRegTypeMask; + return subset(kMask) == (fromOpType(OperandType::kReg) | fromRegType(regType)); } - template - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromValue(const T& value) noexcept { - return OperandSignature{uint32_t(value) << Support::ConstCTZ::value}; + //! Tests whether the operand signature represents a register of the given register type `regType`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { + constexpr uint32_t kMask = kOpTypeMask | kRegGroupMask; + return subset(kMask) == (fromOpType(OperandType::kReg) | fromRegGroup(regGroup)); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromOpType(OperandType opType) noexcept { - return OperandSignature{uint32_t(opType) << kOpTypeShift}; - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return (RegType)getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromRegType(RegType regType) noexcept { - return OperandSignature{uint32_t(regType) << kRegTypeShift}; - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup regGroup() const noexcept { return (RegGroup)getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromRegGroup(RegGroup regGroup) noexcept { - return OperandSignature{uint32_t(regGroup) << kRegGroupShift}; - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType memBaseType() const noexcept { return (RegType)getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromRegTypeAndGroup(RegType regType, RegGroup regGroup) noexcept { - return fromRegType(regType) | fromRegGroup(regGroup); - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType memIndexType() const noexcept { return (RegType)getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromMemBaseType(RegType baseType) noexcept { - return OperandSignature{uint32_t(baseType) << kMemBaseTypeShift}; - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t predicate() const noexcept { return getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromMemIndexType(RegType indexType) noexcept { - return OperandSignature{uint32_t(indexType) << kMemIndexTypeShift}; - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t size() const noexcept { return getField(); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromPredicate(uint32_t predicate) noexcept { - return OperandSignature{predicate << kPredicateShift}; - } + ASMJIT_INLINE_CONSTEXPR void setOpType(OperandType opType) noexcept { setField(uint32_t(opType)); } + ASMJIT_INLINE_CONSTEXPR void setRegType(RegType regType) noexcept { setField(uint32_t(regType)); } + ASMJIT_INLINE_CONSTEXPR void setRegGroup(RegGroup regGroup) noexcept { setField(uint32_t(regGroup)); } - static ASMJIT_INLINE_NODEBUG constexpr OperandSignature fromSize(uint32_t size) noexcept { - return OperandSignature{size << kSizeShift}; - } + ASMJIT_INLINE_CONSTEXPR void setMemBaseType(RegType baseType) noexcept { setField(uint32_t(baseType)); } + ASMJIT_INLINE_CONSTEXPR void setMemIndexType(RegType indexType) noexcept { setField(uint32_t(indexType)); } + + ASMJIT_INLINE_CONSTEXPR void setPredicate(uint32_t predicate) noexcept { setField(predicate); } + ASMJIT_INLINE_CONSTEXPR void setSize(uint32_t size) noexcept { setField(size); } //! \} }; @@ -461,31 +491,29 @@ struct Operand_ { //! \name Types //! \{ - typedef OperandSignature Signature; + using Signature = OperandSignature; //! \} //! \name Constants //! \{ - // Indexes to `_data` array. - enum DataIndex : uint32_t { - kDataMemIndexId = 0, - kDataMemOffsetLo = 1, + //! Memory index offset in a `_data[2]` array. + static inline constexpr uint32_t kDataMemIndexId = 0; + //! Low 32-bit offset value a `_data[2]` array. + static inline constexpr uint32_t kDataMemOffsetLo = 1; - kDataImmValueLo = ASMJIT_ARCH_LE ? 0 : 1, - kDataImmValueHi = ASMJIT_ARCH_LE ? 1 : 0 - }; + //! Low 32-bit immediate value in a `_data[2]` array. + static inline constexpr uint32_t kDataImmValueLo = ASMJIT_ARCH_LE ? 0 : 1; + //! High 32-bit immediate value in a `_data[2]` array. + static inline constexpr uint32_t kDataImmValueHi = ASMJIT_ARCH_LE ? 1 : 0; - //! Constants useful for VirtId <-> Index translation. - enum VirtIdConstants : uint32_t { - //! Minimum valid packed-id. - kVirtIdMin = 256, - //! Maximum valid packed-id, excludes Globals::kInvalidId. - kVirtIdMax = Globals::kInvalidId - 1, - //! Count of valid packed-ids. - kVirtIdCount = uint32_t(kVirtIdMax - kVirtIdMin + 1) - }; + //! Minimum valid packed-id. + static inline constexpr uint32_t kVirtIdMin = 256; + //! Maximum valid packed-id, excludes Globals::kInvalidId. + static inline constexpr uint32_t kVirtIdMax = Globals::kInvalidId - 1; + //! Count of valid packed-ids. + static inline constexpr uint32_t kVirtIdCount = uint32_t(kVirtIdMax - kVirtIdMin + 1); //! \} @@ -509,18 +537,23 @@ struct Operand_ { //! Tests whether the given `id` is a valid virtual register id. Since AsmJit supports both physical and virtual //! registers it must be able to distinguish between these two. The idea is that physical registers are always //! limited in size, so virtual identifiers start from `kVirtIdMin` and end at `kVirtIdMax`. - static ASMJIT_INLINE_NODEBUG bool isVirtId(uint32_t id) noexcept { return id - kVirtIdMin < uint32_t(kVirtIdCount); } + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR bool isVirtId(uint32_t id) noexcept { return id - kVirtIdMin < uint32_t(kVirtIdCount); } + //! Converts a real-id into a packed-id that can be stored in Operand. - static ASMJIT_INLINE_NODEBUG uint32_t indexToVirtId(uint32_t id) noexcept { return id + kVirtIdMin; } + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR uint32_t indexToVirtId(uint32_t id) noexcept { return id + kVirtIdMin; } + //! Converts a packed-id back to real-id. - static ASMJIT_INLINE_NODEBUG uint32_t virtIdToIndex(uint32_t id) noexcept { return id - kVirtIdMin; } + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR uint32_t virtIdToIndex(uint32_t id) noexcept { return id - kVirtIdMin; } //! \name Construction & Destruction //! \{ //! \cond INTERNAL - //! Initializes a `BaseReg` operand from `signature` and register `id`. - ASMJIT_INLINE_NODEBUG void _initReg(const Signature& signature, uint32_t id) noexcept { + //! Initializes a `Reg` operand from `signature` and register `id`. + ASMJIT_INLINE_CONSTEXPR void _initReg(const Signature& signature, uint32_t id) noexcept { _signature = signature; _baseId = id; _data[0] = 0; @@ -529,7 +562,12 @@ struct Operand_ { //! \endcond //! Initializes the operand from `other` operand (used by operator overloads). - ASMJIT_INLINE_NODEBUG void copyFrom(const Operand_& other) noexcept { memcpy(this, &other, sizeof(Operand_)); } + ASMJIT_INLINE_CONSTEXPR void copyFrom(const Operand_& other) noexcept { + _signature._bits = other._signature._bits; + _baseId = other._baseId; + _data[0] = other._data[0]; + _data[1] = other._data[1]; + } //! Resets the `Operand` to none. //! @@ -558,7 +596,7 @@ struct Operand_ { //! memset(&b, 0, sizeof(Operand)); //! assert(a == b); //! ``` - ASMJIT_INLINE_NODEBUG void reset() noexcept { + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { _signature.reset(); _baseId = 0; _data[0] = 0; @@ -571,9 +609,12 @@ struct Operand_ { //! \{ //! Tests whether this operand is the same as `other`. - ASMJIT_INLINE_NODEBUG constexpr bool operator==(const Operand_& other) const noexcept { return equals(other); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool operator==(const Operand_& other) const noexcept { return equals(other); } + //! Tests whether this operand is not the same as `other`. - ASMJIT_INLINE_NODEBUG constexpr bool operator!=(const Operand_& other) const noexcept { return !equals(other); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool operator!=(const Operand_& other) const noexcept { return !equals(other); } //! \} @@ -582,50 +623,114 @@ struct Operand_ { //! Casts this operand to `T` type. template - ASMJIT_INLINE_NODEBUG T& as() noexcept { return static_cast(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR T& as() noexcept { return static_cast(*this); } //! Casts this operand to `T` type (const). template - ASMJIT_INLINE_NODEBUG const T& as() const noexcept { return static_cast(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR const T& as() const noexcept { return static_cast(*this); } //! \} - //! \name Accessors + //! \name Equality //! \{ - //! Tests whether the operand's signature matches the signature of the `other` operand. - ASMJIT_INLINE_NODEBUG constexpr bool hasSignature(const Operand_& other) const noexcept { return _signature == other._signature; } - //! Tests whether the operand's signature matches the given signature `sign`. - ASMJIT_INLINE_NODEBUG constexpr bool hasSignature(const Signature& other) const noexcept { return _signature == other; } + //! Tests whether the operand is 100% equal to `other` operand. + //! + //! \note This basically performs a binary comparison, if aby bit is + //! different the operands are not equal. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool equals(const Operand_& other) const noexcept { + return Support::bool_and( + _signature == other._signature, + _baseId == other._baseId, + _data[0] == other._data[0], + _data[1] == other._data[1] + ); + } + + //! \} + + //! \name Generic Accessors + //! \{ //! Returns operand signature as unsigned 32-bit integer. //! //! Signature is first 4 bytes of the operand data. It's used mostly for operand checking as it's //! much faster to check packed 4 bytes at once than having to check these bytes individually. - ASMJIT_INLINE_NODEBUG constexpr Signature signature() const noexcept { return _signature; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Signature signature() const noexcept { return _signature; } + + //! Tests whether the operand's signature matches the signature of the `other` operand. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSignature(const Operand_& other) const noexcept { return _signature == other._signature; } + + //! Tests whether the operand's signature matches the given signature `sign`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSignature(Signature sign) const noexcept { return _signature == sign; } //! Sets the operand signature, see `signature()`. //! //! \note Improper use of `setSignature()` can lead to hard-to-debug errors. - ASMJIT_INLINE_NODEBUG void setSignature(const Signature& signature) noexcept { _signature = signature; } + ASMJIT_INLINE_CONSTEXPR void setSignature(const Signature& signature) noexcept { _signature = signature; } + + //! \overload + ASMJIT_INLINE_CONSTEXPR void setSignature(uint32_t signature) noexcept { _signature._bits = signature; } //! Returns the type of the operand, see `OpType`. - ASMJIT_INLINE_NODEBUG constexpr OperandType opType() const noexcept { return _signature.opType(); } - //! Tests whether the operand is none (`OperandType::kNone`). - ASMJIT_INLINE_NODEBUG constexpr bool isNone() const noexcept { return _signature == Signature::fromBits(0); } - //! Tests whether the operand is a register (`OperandType::kReg`). - ASMJIT_INLINE_NODEBUG constexpr bool isReg() const noexcept { return opType() == OperandType::kReg; } - //! Tests whether the operand is a memory location (`OperandType::kMem`). - ASMJIT_INLINE_NODEBUG constexpr bool isMem() const noexcept { return opType() == OperandType::kMem; } - //! Tests whether the operand is an immediate (`OperandType::kImm`). - ASMJIT_INLINE_NODEBUG constexpr bool isImm() const noexcept { return opType() == OperandType::kImm; } - //! Tests whether the operand is a label (`OperandType::kLabel`). - ASMJIT_INLINE_NODEBUG constexpr bool isLabel() const noexcept { return opType() == OperandType::kLabel; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandType opType() const noexcept { return _signature.opType(); } - //! Tests whether the operand is a physical register. - ASMJIT_INLINE_NODEBUG constexpr bool isPhysReg() const noexcept { return isReg() && _baseId < 0xFFu; } - //! Tests whether the operand is a virtual register. - ASMJIT_INLINE_NODEBUG constexpr bool isVirtReg() const noexcept { return isReg() && _baseId > 0xFFu; } + //! Tests whether the operand's type matches the given `type`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isOpType(OperandType opType) const noexcept { return _signature.isOpType(opType); } + + //! Tests whether the operand is none (`OperandType::kNone`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isNone() const noexcept { return _signature == Signature::fromBits(0); } + + //! Tests whether the operand is a register (`OperandType::kReg`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return isOpType(OperandType::kReg); } + + //! Tests whether the operand is a register-list. + //! + //! \note Register-list is currently only used by 32-bit ARM architecture. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegList() const noexcept { return isOpType(OperandType::kRegList); } + + //! Tests whether the operand is a memory location (`OperandType::kMem`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMem() const noexcept { return isOpType(OperandType::kMem); } + + //! Tests whether the operand is an immediate (`OperandType::kImm`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isImm() const noexcept { return isOpType(OperandType::kImm); } + + //! Tests whether the operand is a label (`OperandType::kLabel`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isLabel() const noexcept { return isOpType(OperandType::kLabel); } + + //! Tests whether the operand is a register or memory. + //! + //! \note This is useful on X86 and X86_64 architectures as many instructions support Reg/Mem operand combination. + //! So if the user code works with just \ref Operand, it's possible to check whether the operand is either a register + //! or memory location with a single check. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegOrMem() const noexcept { + return Support::isBetween(uint32_t(opType()), uint32_t(OperandType::kReg), uint32_t(OperandType::kMem)); + } + + //! Tests whether the operand is a register, register-list, or memory. + //! + //! \note This is useful on 32-bit ARM architecture to check whether an operand references a register. It can be + //! used in other architectures too, but it would work identically to \ref isRegOrMem() as other architectures + //! don't provide register lists. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegOrRegListOrMem() const noexcept { + return Support::isBetween(uint32_t(opType()), uint32_t(OperandType::kReg), uint32_t(OperandType::kRegList)); + } //! Returns the operand id. //! @@ -636,59 +741,268 @@ struct Operand_ { //! * Imm - Should be `0`. //! * Label - Label id if it was created by using `newLabel()` or `Globals::kInvalidId` if the label is invalid or //! not initialized. - ASMJIT_INLINE_NODEBUG constexpr uint32_t id() const noexcept { return _baseId; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _baseId; } - //! Tests whether the operand is 100% equal to `other` operand. - //! - //! \note This basically performs a binary comparison, if aby bit is - //! different the operands are not equal. - ASMJIT_INLINE_NODEBUG constexpr bool equals(const Operand_& other) const noexcept { - return (_signature == other._signature) & - (_baseId == other._baseId ) & - (_data[0] == other._data[0] ) & - (_data[1] == other._data[1] ) ; - } + //! \} + + //! \name Register Accessors + //! \{ + + //! Tests whether the operand is a physical register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return isReg() && _baseId < 0xFFu; } + + //! Tests whether the operand is a virtual register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return isReg() && _baseId > 0xFFu; } //! Tests whether the operand is a register matching the given register `type`. - ASMJIT_INLINE_NODEBUG constexpr bool isReg(RegType type) const noexcept { - return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegType(type)); - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { return _signature.isReg(regType); } - //! Tests whether the operand is register and of register `type` and `id`. - ASMJIT_INLINE_NODEBUG constexpr bool isReg(RegType type, uint32_t id) const noexcept { - return isReg(type) && this->id() == id; - } + //! Tests whether the operand is register and of register type `regType` and `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType, uint32_t regId) const noexcept { return Support::bool_and(isReg(regType), _baseId == regId); } - //! Tests whether the operand is a register or memory. - ASMJIT_INLINE_NODEBUG constexpr bool isRegOrMem() const noexcept { - return Support::isBetween(uint32_t(opType()), uint32_t(OperandType::kReg), uint32_t(OperandType::kMem)); + //! Tests whether the operand is a register of the provided register group `regGroup`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { return _signature.isReg(regGroup); } + + //! Tests whether the operand is register and of register group `regGroup` and `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup, uint32_t regId) const noexcept { return Support::bool_and(isReg(regGroup), _baseId == regId); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPc() const noexcept { return isReg(RegType::kPC); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isReg(RegGroup::kGp); } + + //! Tests whether the operand is a general purpose register of any type having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp(uint32_t regId) const noexcept { return isReg(RegGroup::kGp, regId); } + + //! Tests whether the register is an 8-bit low or high general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8() const noexcept { return Support::bool_or(isReg(RegType::kGp8Lo), isReg(RegType::kGp8Hi)); } + + //! Tests whether the register is an 8-bit low or high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8(uint32_t regId) const noexcept { return Support::bool_and(isGp8(), id() == regId); } + + //! Tests whether the register is an 8-bit low general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo() const noexcept { return isReg(RegType::kGp8Lo); } + + //! Tests whether the register is an 8-bit low general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo(uint32_t regId) const noexcept { return isReg(RegType::kGp8Lo, regId); } + + //! Tests whether the register is an 8-bit high general purpose register (X86|X86_64 only - AH, BH, CH, DH). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi() const noexcept { return isReg(RegType::kGp8Hi); } + + //! Tests whether the register is an 8-bit high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi(uint32_t regId) const noexcept { return isReg(RegType::kGp8Hi, regId); } + + //! Tests whether the register is a 16-bit general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16() const noexcept { return isReg(RegType::kGp16); } + + //! Tests whether the register is a 16-bit general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16(uint32_t regId) const noexcept { return isReg(RegType::kGp16, regId); } + + //! Tests whether the register is a 32-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32() const noexcept { return isReg(RegType::kGp32); } + + //! Tests whether the register is a 32-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32(uint32_t regId) const noexcept { return isReg(RegType::kGp32, regId); } + + //! Tests whether the register is a 64-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64() const noexcept { return isReg(RegType::kGp64); } + + //! Tests whether the register is a 64-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64(uint32_t regId) const noexcept { return isReg(RegType::kGp64, regId); } + + //! Tests whether the register is a vector register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isReg(RegGroup::kVec); } + + //! Tests whether the register is a vector register of any size having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec(uint32_t regId) const noexcept { return isReg(RegGroup::kVec, regId); } + + //! Tests whether the register is an 8-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return isReg(RegType::kVec8); } + + //! Tests whether the register is an 8-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8(uint32_t regId) const noexcept { return isReg(RegType::kVec8, regId); } + + //! Tests whether the register is a 16-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return isReg(RegType::kVec16); } + + //! Tests whether the register is a 16-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16(uint32_t regId) const noexcept { return isReg(RegType::kVec16, regId); } + + //! Tests whether the register is a 32-bit vector register or view (AArch32, AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return isReg(RegType::kVec32); } + + //! Tests whether the register is a 32-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32(uint32_t regId) const noexcept { return isReg(RegType::kVec32, regId); } + + //! Tests whether the register is a 64-bit vector register or view (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return isReg(RegType::kVec64); } + + //! Tests whether the register is a 64-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64(uint32_t regId) const noexcept { return isReg(RegType::kVec64, regId); } + + //! Tests whether the register is a 128-bit vector register or view (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return isReg(RegType::kVec128); } + + //! Tests whether the register is a 128-bit vector register or view having the given id `regId` (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128(uint32_t regId) const noexcept { return isReg(RegType::kVec128, regId); } + + //! Tests whether the register is a 256-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256() const noexcept { return isReg(RegType::kVec256); } + + //! Tests whether the register is a 256-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256(uint32_t regId) const noexcept { return isReg(RegType::kVec256, regId); } + + //! Tests whether the register is a 512-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512() const noexcept { return isReg(RegType::kVec512); } + + //! Tests whether the register is a 512-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512(uint32_t regId) const noexcept { return isReg(RegType::kVec512, regId); } + + //! Tests whether the register is a mask register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register of any size having the given id `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) - alias of \ref isMask(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) of any size having the given id + //! `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a tile register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register of the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) - alias of \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) of the given id `regId` - alias of + //! \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a segment register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg() const noexcept { return isReg(RegType::kSegment); } + + //! Tests whether the register is a segment register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg(uint32_t regId) const noexcept { return isReg(RegType::kSegment, regId); } + + //! Tests whether the register is a control register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg() const noexcept { return isReg(RegType::kControl); } + + //! Tests whether the register is a control register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg(uint32_t regId) const noexcept { return isReg(RegType::kControl, regId); } + + //! Tests whether the register is a debug register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg() const noexcept { return isReg(RegType::kDebug); } + + //! Tests whether the register is a debug register of the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg(uint32_t regId) const noexcept { return isReg(RegType::kDebug, regId); } + + //! Tests whether the register is an MMX register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg() const noexcept { return isReg(RegType::kX86_Mm); } + + //! Tests whether the register is an MMX register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Mm, regId); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg() const noexcept { return isReg(RegType::kX86_St); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_St, regId); } + + //! Tests whether the register is a BND register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg() const noexcept { return isReg(RegType::kX86_Bnd); } + + //! Tests whether the register is a BND register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Bnd, regId); } + + //! \} + + //! \name Register-List Accessors + //! \{ + + //! Tests whether the operand is a register matching the given register `type`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegList(RegType type) const noexcept { + return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kRegList) | Signature::fromRegType(type)); } //! \} - //! \name Accessors (X86 Specific) + //! \name X86-Specific Accessors //! \{ //! Returns a size of a register or an X86 memory operand. //! - //! At the moment only X86 and X86_64 memory operands have a size - other memory operands can use bits that represent - //! size as an additional payload. This means that memory size is architecture specific and should be accessed via - //! \ref x86::Mem::size(). Sometimes when the user knows that the operand is either a register or memory operand this - //! function can be helpful as it avoids casting. - ASMJIT_INLINE_NODEBUG constexpr uint32_t x86RmSize() const noexcept { - return _signature.size(); - } - -#if !defined(ASMJIT_NO_DEPRECATED) - ASMJIT_DEPRECATED("hasSize() is no longer portable - use x86RmSize() instead if, your target is X86/X86_64") - ASMJIT_INLINE_NODEBUG constexpr bool hasSize() const noexcept { return x86RmSize() != 0u; } - - ASMJIT_DEPRECATED("hasSize() is no longer portable - use x86RmSize() instead if, your target is X86/X86_64") - ASMJIT_INLINE_NODEBUG constexpr bool hasSize(uint32_t s) const noexcept { return x86RmSize() == s; } - - ASMJIT_DEPRECATED("size() is no longer portable - use x86RmSize() instead, if your target is X86/X86_64") - ASMJIT_INLINE_NODEBUG constexpr uint32_t size() const noexcept { return _signature.getField(); } -#endif + //! \remarks At the moment only X86 and X86_64 memory operands have a size - other memory operands can use bits + //! that represent size as an additional payload. This means that memory size is architecture specific and should + //! be accessed via \ref x86::Mem::size(). Sometimes when the user knows that the operand is either a register or + //! memory operand this function can be helpful as it avoids casting, but it only works when it targets X86 and X86_64. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t x86RmSize() const noexcept { return _signature.size(); } //! \} }; @@ -700,19 +1014,19 @@ public: //! \{ //! Creates `kOpNone` operand having all members initialized to zero. - ASMJIT_INLINE_NODEBUG constexpr Operand() noexcept + ASMJIT_INLINE_CONSTEXPR Operand() noexcept : Operand_{ Signature::fromOpType(OperandType::kNone), 0u, { 0u, 0u }} {} //! Creates a cloned `other` operand. - ASMJIT_INLINE_NODEBUG constexpr Operand(const Operand& other) noexcept = default; + ASMJIT_INLINE_CONSTEXPR Operand(const Operand& other) noexcept = default; //! Creates a cloned `other` operand. - ASMJIT_INLINE_NODEBUG constexpr explicit Operand(const Operand_& other) + ASMJIT_INLINE_CONSTEXPR explicit Operand(const Operand_& other) : Operand_(other) {} //! Creates an operand initialized to raw `[u0, u1, u2, u3]` values. - ASMJIT_INLINE_NODEBUG constexpr Operand(Globals::Init_, const Signature& u0, uint32_t u1, uint32_t u2, uint32_t u3) noexcept - : Operand_{ u0, u1, { u2, u3 }} {} + ASMJIT_INLINE_CONSTEXPR Operand(Globals::Init_, const Signature& u0, uint32_t u1, uint32_t u2, uint32_t u3) noexcept + : Operand_{{u0._bits}, u1, {u2, u3}} {} //! Creates an uninitialized operand (dangerous). ASMJIT_INLINE_NODEBUG explicit Operand(Globals::NoInit_) noexcept {} @@ -722,8 +1036,17 @@ public: //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_NODEBUG Operand& operator=(const Operand& other) noexcept = default; - ASMJIT_INLINE_NODEBUG Operand& operator=(const Operand_& other) noexcept { return operator=(static_cast(other)); } + ASMJIT_INLINE_CONSTEXPR Operand& operator=(const Operand& other) noexcept { + // Defaulted copy operator cannot be marked as constexpr in C++17, thus we have to implement it. + copyFrom(other); + return *this; + } + + ASMJIT_INLINE_CONSTEXPR Operand& operator=(const Operand_& other) noexcept { + // Defaulted copy operator cannot be marked as constexpr in C++17, thus we have to implement it. + copyFrom(other); + return *this; + } //! \} @@ -731,7 +1054,8 @@ public: //! \{ //! Clones this operand and returns its copy. - ASMJIT_INLINE_NODEBUG constexpr Operand clone() const noexcept { return Operand(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Operand clone() const noexcept { return Operand(*this); } //! \} }; @@ -768,22 +1092,22 @@ public: //! \{ //! Creates a label operand without ID (you must set the ID to make it valid). - ASMJIT_INLINE_NODEBUG constexpr Label() noexcept + ASMJIT_INLINE_CONSTEXPR Label() noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kLabel), Globals::kInvalidId, 0, 0) {} //! Creates a cloned label operand of `other`. - ASMJIT_INLINE_NODEBUG constexpr Label(const Label& other) noexcept + ASMJIT_INLINE_CONSTEXPR Label(const Label& other) noexcept : Operand(other) {} //! Creates a label operand of the given `id`. - ASMJIT_INLINE_NODEBUG constexpr explicit Label(uint32_t id) noexcept + ASMJIT_INLINE_CONSTEXPR explicit Label(uint32_t id) noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kLabel), id, 0, 0) {} ASMJIT_INLINE_NODEBUG explicit Label(Globals::NoInit_) noexcept : Operand(Globals::NoInit) {} //! Resets the label, will reset all properties and set its ID to `Globals::kInvalidId`. - ASMJIT_INLINE_NODEBUG void reset() noexcept { + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { _signature = Signature::fromOpType(OperandType::kLabel); _baseId = Globals::kInvalidId; _data[0] = 0; @@ -795,7 +1119,10 @@ public: //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_NODEBUG Label& operator=(const Label& other) noexcept = default; + ASMJIT_INLINE_CONSTEXPR Label& operator=(const Label& other) noexcept { + copyFrom(other); + return *this; + } //! \} @@ -803,56 +1130,165 @@ public: //! \{ //! Tests whether the label was created by CodeHolder and/or an attached emitter. - ASMJIT_INLINE_NODEBUG constexpr bool isValid() const noexcept { return _baseId != Globals::kInvalidId; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return _baseId != Globals::kInvalidId; } + //! Sets the label `id`. - ASMJIT_INLINE_NODEBUG void setId(uint32_t id) noexcept { _baseId = id; } + ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _baseId = id; } //! \} }; -//! \cond INTERNAL -//! Default register traits. -struct BaseRegTraits { - enum : uint32_t { - //! \ref TypeId representing this register type, could be \ref TypeId::kVoid if such type doesn't exist. - kTypeId = uint32_t(TypeId::kVoid), - //! RegType is not valid by default. - kValid = 0, - //! Count of registers (0 if none). - kCount = 0, +//! Register traits. +//! +//! Register traits contain metadata about a particular register type. The metadata is used by AsmJit to setup +//! register information on-the-fly and to populate tables that contain register information (this way it's possible +//! to change register types and groups without having to reorder these tables). +template +struct RegTraits { + //! \ref TypeId representing this register type, could be \ref TypeId::kVoid if such type doesn't exist. + static inline constexpr TypeId kTypeId = TypeId::kVoid; + //! RegType is not valid by default. + static inline constexpr uint32_t kValid = 0; - //! Zero type by default (defeaults to None). - kType = uint32_t(RegType::kNone), - //! Zero group by default (defaults to GP). - kGroup = uint32_t(RegGroup::kGp), - //! No size by default. - kSize = 0, + //! Zero type by default (defaults to None). + static inline constexpr RegType kType = RegType::kNone; + //! Zero group by default (defaults to GP). + static inline constexpr RegGroup kGroup = RegGroup::kGp; + //! No size by default. + static inline constexpr uint32_t kSize = 0u; - //! Empty signature by default (not even having operand type set to register). - kSignature = 0 - }; + //! Empty signature by default (not even having operand type set to register). + static inline constexpr uint32_t kSignature = 0; }; + +//! \cond + +//! Adds a template specialization for `REG_TYPE` into the local `RegTraits`. +#define ASMJIT_DEFINE_REG_TRAITS(REG_TYPE, GROUP, SIZE, TYPE_ID) \ +template<> \ +struct RegTraits { \ + static inline constexpr uint32_t kValid = 1; \ + static inline constexpr RegType kType = REG_TYPE; \ + static inline constexpr RegGroup kGroup = GROUP; \ + static inline constexpr uint32_t kSize = SIZE; \ + static inline constexpr TypeId kTypeId = TYPE_ID; \ + \ + static inline constexpr uint32_t kSignature = \ + (OperandSignature::fromOpType(OperandType::kReg) | \ + OperandSignature::fromRegType(kType) | \ + OperandSignature::fromRegGroup(kGroup) | \ + OperandSignature::fromSize(kSize)).bits(); \ + \ +} + +// <--------------------+------------------------+------------------------+----+------------------+ +---+---+---+---+ +// | Reg-Type | Reg-Group |Size| TypeId | |X86|X64|A32|A64| +// <--------------------+------------------------+------------------------+----+------------------+ +---+---+---+---+ +ASMJIT_DEFINE_REG_TRAITS(RegType::kPC , RegGroup::kPC , 8 , TypeId::kInt64 ); // | x | x | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp8Lo , RegGroup::kGp , 1 , TypeId::kInt8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp8Hi , RegGroup::kGp , 1 , TypeId::kInt8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp16 , RegGroup::kGp , 2 , TypeId::kInt16 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp32 , RegGroup::kGp , 4 , TypeId::kInt32 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kGp64 , RegGroup::kGp , 8 , TypeId::kInt64 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec8 , RegGroup::kVec , 1 , TypeId::kVoid ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec16 , RegGroup::kVec , 2 , TypeId::kVoid ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec32 , RegGroup::kVec , 4 , TypeId::kInt32x1 ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec64 , RegGroup::kVec , 8 , TypeId::kInt32x2 ); // | | | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec128 , RegGroup::kVec , 16 , TypeId::kInt32x4 ); // | x | x | x | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec256 , RegGroup::kVec , 32 , TypeId::kInt32x8 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVec512 , RegGroup::kVec , 64 , TypeId::kInt32x16); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kVecNLen , RegGroup::kVec , 0 , TypeId::kVoid ); // | | | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kMask , RegGroup::kMask , 0 , TypeId::kVoid ); // | x | x | | x | +ASMJIT_DEFINE_REG_TRAITS(RegType::kTile , RegGroup::kTile , 0 , TypeId::kVoid ); // | | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kSegment , RegGroup::kSegment , 2 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kControl , RegGroup::kControl , 0 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kDebug , RegGroup::kDebug , 0 , TypeId::kVoid ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_Mm , RegGroup::kX86_MM , 8 , TypeId::kMmx64 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_St , RegGroup::kX86_St , 10 , TypeId::kFloat80 ); // | x | x | | | +ASMJIT_DEFINE_REG_TRAITS(RegType::kX86_Bnd , RegGroup::kX86_Bnd , 16 , TypeId::kVoid ); // | x | x | | | + +#undef ASMJIT_DEFINE_REG_TRAITS + //! \endcond -//! Physical or virtual register operand. -class BaseReg : public Operand { +namespace RegUtils { + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR OperandSignature signatureOf(RegType regType) noexcept { + constexpr uint32_t signatureTable[] = { + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, + RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature, RegTraits::kSignature + }; + return OperandSignature{signatureTable[size_t(regType)]}; +} + +[[nodiscard]] +static ASMJIT_INLINE_NODEBUG OperandSignature signatureOfVecSize(uint32_t size) noexcept { + RegType regType = RegType(Support::ctz((size | 0x40u) & 0x0Fu) - 4u + uint32_t(RegType::kVec128)); + return signatureOf(regType); +} + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR RegGroup groupOf(RegType regType) noexcept { + constexpr RegGroup groupTable[] = { + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, + RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup, RegTraits::kGroup + }; + return groupTable[size_t(regType)]; +} + +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR TypeId typeIdOf(RegType regType) noexcept { + constexpr TypeId typeIdTable[] = { + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, + RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId, RegTraits::kTypeId + }; + return typeIdTable[size_t(regType)]; +} + +} // {RegUtils} + +//! Unified physical or virtual register operand. +class Reg : public Operand { public: //! \name Constants //! \{ - enum : uint32_t { - //! None or any register (mostly internal). - kIdBad = 0xFFu, + //! None or any register (mostly internal). + static inline constexpr uint32_t kIdBad = 0xFFu; - kBaseSignatureMask = - Signature::kOpTypeMask | - Signature::kRegTypeMask | - Signature::kRegGroupMask | - Signature::kSizeMask, + static inline constexpr uint32_t kBaseSignatureMask = + Signature::kOpTypeMask | + Signature::kRegTypeMask | + Signature::kRegGroupMask | + Signature::kSizeMask; - kTypeNone = uint32_t(RegType::kNone), - kSignature = Signature::fromOpType(OperandType::kReg).bits() - }; + static inline constexpr uint32_t kTypeNone = uint32_t(RegType::kNone); + static inline constexpr uint32_t kSignature = Signature::fromOpType(OperandType::kReg).bits(); + + template + static ASMJIT_INLINE_CONSTEXPR Signature _signatureOf() noexcept { return Signature{RegTraits::kSignature}; } + + static ASMJIT_INLINE_CONSTEXPR Signature signatureOf(RegType regType) noexcept { return RegUtils::signatureOf(regType); } //! \} @@ -860,49 +1296,74 @@ public: //! \{ //! Creates a dummy register operand. - ASMJIT_INLINE_NODEBUG constexpr BaseReg() noexcept - : Operand(Globals::Init, Signature::fromOpType(OperandType::kReg), kIdBad, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kReg), kIdBad, 0u, 0u) {} //! Creates a new register operand which is the same as `other` . - ASMJIT_INLINE_NODEBUG constexpr BaseReg(const BaseReg& other) noexcept + ASMJIT_INLINE_CONSTEXPR Reg(const Reg& other) noexcept : Operand(other) {} //! Creates a new register operand compatible with `other`, but with a different `id`. - ASMJIT_INLINE_NODEBUG constexpr BaseReg(const BaseReg& other, uint32_t id) noexcept - : Operand(Globals::Init, other._signature, id, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg(const Reg& other, uint32_t id) noexcept + : Operand(Globals::Init, other._signature, id, 0u, 0u) {} //! Creates a register initialized to the given `signature` and `id`. - ASMJIT_INLINE_NODEBUG constexpr BaseReg(const Signature& signature, uint32_t id) noexcept - : Operand(Globals::Init, signature, id, 0, 0) {} + ASMJIT_INLINE_CONSTEXPR Reg(const Signature& signature, uint32_t id) noexcept + : Operand(Globals::Init, signature, id, 0u, 0u) {} - ASMJIT_INLINE_NODEBUG explicit BaseReg(Globals::NoInit_) noexcept + ASMJIT_INLINE_NODEBUG explicit Reg(Globals::NoInit_) noexcept : Operand(Globals::NoInit) {} + //! Creates a new register from register type and id. + static ASMJIT_INLINE_CONSTEXPR Reg fromTypeAndId(RegType type, uint32_t id) noexcept { return Reg(signatureOf(type), id); } + //! \} //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_NODEBUG BaseReg& operator=(const BaseReg& other) noexcept = default; + ASMJIT_INLINE_CONSTEXPR Reg& operator=(const Reg& other) noexcept { + copyFrom(other); + return *this; + } //! \} - //! \name Accessors + //! \name Signature Accessors //! \{ //! Returns base signature of the register associated with each register type. //! //! Base signature only contains the operand type, register type, register group, and register size. It doesn't //! contain element type, predicate, or other architecture-specific data. Base signature is a signature that is - //! provided by architecture-specific `RegTraits`, like \ref x86::RegTraits. - ASMJIT_INLINE_NODEBUG constexpr OperandSignature baseSignature() const noexcept { return _signature & kBaseSignatureMask; } + //! provided by \ref RegTraits. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature baseSignature() const noexcept { return _signature & kBaseSignatureMask; } //! Tests whether the operand's base signature matches the given signature `sign`. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseSignature(uint32_t signature) const noexcept { return baseSignature() == signature; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseSignature(uint32_t signature) const noexcept { return baseSignature() == signature; } + //! Tests whether the operand's base signature matches the given signature `sign`. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseSignature(const OperandSignature& signature) const noexcept { return baseSignature() == signature; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseSignature(const OperandSignature& signature) const noexcept { return baseSignature() == signature; } + //! Tests whether the operand's base signature matches the base signature of the `other` operand. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseSignature(const BaseReg& other) const noexcept { return baseSignature() == other.baseSignature(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseSignature(const Reg& other) const noexcept { return baseSignature() == other.baseSignature(); } + + //! \} + + //! \name Register Accessors + //! \{ + + //! Returns the type of the register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return _signature.regType(); } + + //! Returns the group this register belongs to. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup regGroup() const noexcept { return _signature.regGroup(); } //! Tests whether this register is the same as `other`. //! @@ -911,231 +1372,364 @@ public: //! both \ref equals() and \ref isSame() should give the same answer, however, if any of these two contains garbage //! or other metadata in the upper 8 bytes then \ref isSame() may return `true` in cases in which \ref equals() //! returns false. - ASMJIT_INLINE_NODEBUG constexpr bool isSame(const BaseReg& other) const noexcept { - return (_signature == other._signature) & (_baseId == other._baseId); - } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSame(const Reg& other) const noexcept { return (_signature == other._signature) & (_baseId == other._baseId); } //! Tests whether the register is valid (either virtual or physical). - ASMJIT_INLINE_NODEBUG constexpr bool isValid() const noexcept { return (_signature != 0) & (_baseId != kIdBad); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return Support::bool_and(_signature != 0u, _baseId != kIdBad); } //! Tests whether this is a physical register. - ASMJIT_INLINE_NODEBUG constexpr bool isPhysReg() const noexcept { return _baseId < kIdBad; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return _baseId < kIdBad; } + //! Tests whether this is a virtual register. - ASMJIT_INLINE_NODEBUG constexpr bool isVirtReg() const noexcept { return _baseId > kIdBad; } - - //! Tests whether the register type matches `type` - same as `isReg(type)`, provided for convenience. - ASMJIT_INLINE_NODEBUG constexpr bool isType(RegType type) const noexcept { return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(type); } - //! Tests whether the register group matches `group`. - ASMJIT_INLINE_NODEBUG constexpr bool isGroup(RegGroup group) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(group); } - - //! Tests whether the register is a general purpose register (any size). - ASMJIT_INLINE_NODEBUG constexpr bool isGp() const noexcept { return isGroup(RegGroup::kGp); } - //! Tests whether the register is a vector register. - ASMJIT_INLINE_NODEBUG constexpr bool isVec() const noexcept { return isGroup(RegGroup::kVec); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return _baseId > kIdBad; } + // NOTE: A careful reader may wonder why the member functions here are repeated and basically do the same as in + // `Operand_`, however, they don't always do the same. In `Operand_` case each test function must check + // whether the operand is actually a register, whereas here we don't have to. However, in some cases it's + // actually beneficial to do that (if the check is using a constant expression on the input). Since C++ + // doesn't support mixins the implementation is basically duplicating some of the `Operand_` checks here. using Operand_::isReg; - //! Same as `isType()`, provided for convenience. - ASMJIT_INLINE_NODEBUG constexpr bool isReg(RegType rType) const noexcept { return isType(rType); } - //! Tests whether the register type matches `type` and register id matches `id`. - ASMJIT_INLINE_NODEBUG constexpr bool isReg(RegType rType, uint32_t id) const noexcept { return isType(rType) && this->id() == id; } + //! Tests whether the register is of the given register type `regType`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType) const noexcept { +#if defined(__GNUC__) + // At the moment operand type is 3 bits and register type is 5 bits, which form a value that is stored in + // the 8 least significant bits of the operand signature. Checking ALL of these 8 bits is much better than + // extracting register type at least on X86 and X86_64 targets. + if (__builtin_constant_p(regType)) { + return _signature.isReg(regType); + } +#endif - //! Returns the register type. - ASMJIT_INLINE_NODEBUG constexpr RegType type() const noexcept { return _signature.regType(); } - //! Returns the register group. - ASMJIT_INLINE_NODEBUG constexpr RegGroup group() const noexcept { return _signature.regGroup(); } + return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(regType); + } + + //! Tests whether the register is of the given register type `regType` having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegType regType, uint32_t regId) const noexcept { return Support::bool_and(isReg(regType), _baseId == regId); } + + //! Tests whether the register is of the given register group `regGroup`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(regGroup); } + + //! Tests whether the register is of the given register group `regGroup` having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg(RegGroup regGroup, uint32_t regId) const noexcept { return Support::bool_and(isReg(regGroup), _baseId == regId); } + + //! Tests whether the operand is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPc() const noexcept { return isReg(RegType::kPC); } + + //! Tests whether the register is a general purpose register of any type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isReg(RegGroup::kGp); } + + //! Tests whether the register is a general purpose register of any type having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp(uint32_t regId) const noexcept { return isReg(RegGroup::kGp, regId); } + + //! Tests whether the register is an 8-bit low or high general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8() const noexcept { return Support::bool_or(isReg(RegType::kGp8Lo), isReg(RegType::kGp8Hi)); } + + //! Tests whether the register is an 8-bit low or high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8(uint32_t regId) const noexcept { return Support::bool_and(isGp8(), id() == regId); } + + //! Tests whether the register is an 8-bit low general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo() const noexcept { return isReg(RegType::kGp8Lo); } + + //! Tests whether the register is an 8-bit low general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Lo(uint32_t regId) const noexcept { return isReg(RegType::kGp8Lo, regId); } + + //! Tests whether the register is an 8-bit high general purpose register (X86|X86_64 only - AH, BH, CH, DH). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi() const noexcept { return isReg(RegType::kGp8Hi); } + + //! Tests whether the register is an 8-bit high general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp8Hi(uint32_t regId) const noexcept { return isReg(RegType::kGp8Hi, regId); } + + //! Tests whether the register is a 16-bit general purpose register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16() const noexcept { return isReg(RegType::kGp16); } + + //! Tests whether the register is a 16-bit general purpose register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp16(uint32_t regId) const noexcept { return isReg(RegType::kGp16, regId); } + + //! Tests whether the register is a 32-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32() const noexcept { return isReg(RegType::kGp32); } + + //! Tests whether the register is a 32-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp32(uint32_t regId) const noexcept { return isReg(RegType::kGp32, regId); } + + //! Tests whether the register is a 64-bit general purpose register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64() const noexcept { return isReg(RegType::kGp64); } + + //! Tests whether the register is a 64-bit general purpose register having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp64(uint32_t regId) const noexcept { return isReg(RegType::kGp64, regId); } + + //! Tests whether the register is a vector register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isReg(RegGroup::kVec); } + + //! Tests whether the register is a vector register of any size having the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec(uint32_t regId) const noexcept { return isReg(RegGroup::kVec, regId); } + + //! Tests whether the register is an 8-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8() const noexcept { return isReg(RegType::kVec8); } + + //! Tests whether the register is an 8-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec8(uint32_t regId) const noexcept { return isReg(RegType::kVec8, regId); } + + //! Tests whether the register is a 16-bit vector register or view (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16() const noexcept { return isReg(RegType::kVec16); } + + //! Tests whether the register is a 16-bit vector register or view having the given id `regId` (AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec16(uint32_t regId) const noexcept { return isReg(RegType::kVec16, regId); } + + //! Tests whether the register is a 32-bit vector register or view (AArch32, AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32() const noexcept { return isReg(RegType::kVec32); } + + //! Tests whether the register is a 32-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec32(uint32_t regId) const noexcept { return isReg(RegType::kVec32, regId); } + + //! Tests whether the register is a 64-bit vector register or view (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64() const noexcept { return isReg(RegType::kVec64); } + + //! Tests whether the register is a 64-bit vector register or view having the given id `regId` (AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec64(uint32_t regId) const noexcept { return isReg(RegType::kVec64, regId); } + + //! Tests whether the register is a 128-bit vector register or view (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128() const noexcept { return isReg(RegType::kVec128); } + + //! Tests whether the register is a 128-bit vector register or view having the given id `regId` (X86|X86_64/AArch32/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec128(uint32_t regId) const noexcept { return isReg(RegType::kVec128, regId); } + + //! Tests whether the register is a 256-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256() const noexcept { return isReg(RegType::kVec256); } + + //! Tests whether the register is a 256-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec256(uint32_t regId) const noexcept { return isReg(RegType::kVec256, regId); } + + //! Tests whether the register is a 512-bit vector register or view (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512() const noexcept { return isReg(RegType::kVec512); } + + //! Tests whether the register is a 512-bit vector register or view having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec512(uint32_t regId) const noexcept { return isReg(RegType::kVec512, regId); } + + //! Tests whether the register is a mask register of any size. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register of any size having the given id `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMaskReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) - alias of \ref isMask(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg() const noexcept { return isReg(RegType::kMask); } + + //! Tests whether the register is a mask register (`K` register on X86|X86_64) of any size having the given id + //! `regId` (X86|X86_64/AArch64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isKReg(uint32_t regId) const noexcept { return isReg(RegType::kMask, regId); } + + //! Tests whether the register is a tile register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register of the given id `regId`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTileReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) - alias of \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg() const noexcept { return isReg(RegType::kTile); } + + //! Tests whether the register is a tile register (`TMM` register on X86_64) of the given id `regId` - alias of + //! \ref isTileReg(). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isTmmReg(uint32_t regId) const noexcept { return isReg(RegType::kTile, regId); } + + //! Tests whether the register is a segment register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg() const noexcept { return isReg(RegType::kSegment); } + + //! Tests whether the register is a segment register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isSegmentReg(uint32_t regId) const noexcept { return isReg(RegType::kSegment, regId); } + + //! Tests whether the register is a control register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg() const noexcept { return isReg(RegType::kControl); } + + //! Tests whether the register is a control register having the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isControlReg(uint32_t regId) const noexcept { return isReg(RegType::kControl, regId); } + + //! Tests whether the register is a debug register (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg() const noexcept { return isReg(RegType::kDebug); } + + //! Tests whether the register is a debug register of the given id `regId` (X86|X86_64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isDebugReg(uint32_t regId) const noexcept { return isReg(RegType::kDebug, regId); } + + //! Tests whether the register is an MMX register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg() const noexcept { return isReg(RegType::kX86_Mm); } + + //! Tests whether the register is an MMX register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isMmReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Mm, regId); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg() const noexcept { return isReg(RegType::kX86_St); } + + //! Tests whether the register is an FPU register (`ST` register on X86|X86_64) of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isStReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_St, regId); } + + //! Tests whether the register is a BND register (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg() const noexcept { return isReg(RegType::kX86_Bnd); } + + //! Tests whether the register is a BND register of the given id `regId` (X86|X64). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isBndReg(uint32_t regId) const noexcept { return isReg(RegType::kX86_Bnd, regId); } //! Tests whether the register specifies a size (i.e. the size is not zero). - ASMJIT_INLINE_NODEBUG constexpr bool hasSize() const noexcept { return _signature.hasField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSize() const noexcept { return _signature.hasField(); } + //! Tests whether the register size matches size `s`. - ASMJIT_INLINE_NODEBUG constexpr bool hasSize(uint32_t s) const noexcept { return size() == s; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasSize(uint32_t s) const noexcept { return size() == s; } //! Returns the size of the register in bytes. If the register size depends on architecture (like `x86::CReg` and //! `x86::DReg`) the size returned should be the greatest possible (so it should return 64-bit size in such case). - ASMJIT_INLINE_NODEBUG constexpr uint32_t size() const noexcept { return _signature.getField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t size() const noexcept { return _signature.getField(); } //! Returns operation predicate of the register (ARM/AArch64). //! //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp //! of the register. - ASMJIT_INLINE_NODEBUG constexpr uint32_t predicate() const noexcept { return _signature.getField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t predicate() const noexcept { return _signature.getField(); } //! Sets operation predicate of the register to `predicate` (ARM/AArch64). //! //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp //! of the register. - ASMJIT_INLINE_NODEBUG void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } + ASMJIT_INLINE_CONSTEXPR void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } //! Resets shift operation type of the register to the default value (ARM/AArch64). - ASMJIT_INLINE_NODEBUG void resetPredicate() noexcept { _signature.setField(0); } + ASMJIT_INLINE_CONSTEXPR void resetPredicate() noexcept { _signature.setField(0); } //! Clones the register operand. - ASMJIT_INLINE_NODEBUG constexpr BaseReg clone() const noexcept { return BaseReg(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Reg clone() const noexcept { return Reg(*this); } //! Casts this register to `RegT` by also changing its signature. //! //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. template - ASMJIT_INLINE_NODEBUG constexpr RegT cloneAs() const noexcept { return RegT(Signature(RegT::kSignature), id()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegT cloneAs() const noexcept { return RegT(Signature(RegT::kSignature), id()); } //! Casts this register to `other` by also changing its signature. //! //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. template - ASMJIT_INLINE_NODEBUG constexpr RegT cloneAs(const RegT& other) const noexcept { return RegT(other.signature(), id()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegT cloneAs(const RegT& other) const noexcept { return RegT(other.signature(), id()); } + + template + ASMJIT_INLINE_CONSTEXPR void setRegT(uint32_t id) noexcept { + setSignature(RegTraits::kSignature); + setId(id); + } //! Sets the register id to `id`. - ASMJIT_INLINE_NODEBUG void setId(uint32_t id) noexcept { _baseId = id; } + ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _baseId = id; } //! Sets a 32-bit operand signature based on traits of `RegT`. template - ASMJIT_INLINE_NODEBUG void setSignatureT() noexcept { _signature = RegT::kSignature; } + ASMJIT_INLINE_CONSTEXPR void setSignatureT() noexcept { _signature = RegT::kSignature; } //! Sets the register `signature` and `id`. - ASMJIT_INLINE_NODEBUG void setSignatureAndId(const OperandSignature& signature, uint32_t id) noexcept { + ASMJIT_INLINE_CONSTEXPR void setSignatureAndId(const OperandSignature& signature, uint32_t id) noexcept { _signature = signature; _baseId = id; } //! \} - //! \name Static Functions - //! \{ +#if !defined(ASMJIT_NO_DEPRECATED) + [[deprecated("Use regType() instead")]] + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } - //! Tests whether the `op` operand is a general purpose register. - static ASMJIT_INLINE_NODEBUG bool isGp(const Operand_& op) noexcept { - // Check operand type and register group. Not interested in register type and size. - return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kGp)); - } - - //! Tests whether the `op` operand is a vector register. - static ASMJIT_INLINE_NODEBUG bool isVec(const Operand_& op) noexcept { - // Check operand type and register group. Not interested in register type and size. - return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kVec)); - } - - //! Tests whether the `op` is a general purpose register of the given `id`. - static ASMJIT_INLINE_NODEBUG bool isGp(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGp(op)) & unsigned(op.id() == id)); } - //! Tests whether the `op` is a vector register of the given `id`. - static ASMJIT_INLINE_NODEBUG bool isVec(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVec(op)) & unsigned(op.id() == id)); } - - //! \} + [[deprecated("Use regGroup() instead")]] + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } +#endif // !ASMJIT_NO_DEPRECATED }; -//! RegOnly is 8-byte version of `BaseReg` that allows to store either register or nothing. -//! -//! It's designed to decrease the space consumed by an extra "operand" in \ref BaseEmitter and \ref InstNode. -struct RegOnly { - //! \name Types - //! \{ +#if !defined(ASMJIT_NO_DEPRECATED) +using BaseReg [[deprecated("Use asmjit::Reg instead of asmjit::BaseReg")]] = Reg; +#endif // !ASMJIT_NO_DEPRECATED - typedef OperandSignature Signature; - - //! \} - - //! Operand signature - only \ref OperandType::kNone and \ref OperandType::kReg are supported. - Signature _signature; - //! Physical or virtual register id. - uint32_t _id; - - //! \name Construction & Destruction - //! \{ - - //! Initializes the `RegOnly` instance to hold register `signature` and `id`. - ASMJIT_INLINE_NODEBUG void init(const OperandSignature& signature, uint32_t id) noexcept { - _signature = signature; - _id = id; - } - - ASMJIT_INLINE_NODEBUG void init(const BaseReg& reg) noexcept { init(reg.signature(), reg.id()); } - ASMJIT_INLINE_NODEBUG void init(const RegOnly& reg) noexcept { init(reg.signature(), reg.id()); } - - //! Resets the `RegOnly` members to zeros (none). - ASMJIT_INLINE_NODEBUG void reset() noexcept { init(Signature::fromBits(0), 0); } - - //! \} - - //! \name Accessors - //! \{ - - //! Tests whether this ExtraReg is none (same as calling `Operand_::isNone()`). - ASMJIT_INLINE_NODEBUG constexpr bool isNone() const noexcept { return _signature == 0; } - //! Tests whether the register is valid (either virtual or physical). - ASMJIT_INLINE_NODEBUG constexpr bool isReg() const noexcept { return _signature != 0; } - - //! Tests whether this is a physical register. - ASMJIT_INLINE_NODEBUG constexpr bool isPhysReg() const noexcept { return _id < BaseReg::kIdBad; } - //! Tests whether this is a virtual register (used by `BaseCompiler`). - ASMJIT_INLINE_NODEBUG constexpr bool isVirtReg() const noexcept { return _id > BaseReg::kIdBad; } - - //! Returns the register signature or 0 if no register is assigned. - ASMJIT_INLINE_NODEBUG constexpr OperandSignature signature() const noexcept { return _signature; } - //! Returns the register id. - //! - //! \note Always check whether the register is assigned before using the returned identifier as - //! non-assigned `RegOnly` instance would return zero id, which is still a valid register id. - ASMJIT_INLINE_NODEBUG constexpr uint32_t id() const noexcept { return _id; } - - //! Sets the register id. - ASMJIT_INLINE_NODEBUG void setId(uint32_t id) noexcept { _id = id; } - - //! Returns the register type. - ASMJIT_INLINE_NODEBUG constexpr RegType type() const noexcept { return _signature.regType(); } - //! Returns the register group. - ASMJIT_INLINE_NODEBUG constexpr RegGroup group() const noexcept { return _signature.regGroup(); } - - //! \} - - //! \name Utilities - //! \{ - - //! Converts this ExtraReg to a real `RegT` operand. - template - ASMJIT_INLINE_NODEBUG constexpr RegT toReg() const noexcept { return RegT(_signature, _id); } - - //! \} -}; - -//! \cond INTERNAL -//! Adds a template specialization for `REG_TYPE` into the local `RegTraits`. -#define ASMJIT_DEFINE_REG_TRAITS(REG, REG_TYPE, GROUP, SIZE, COUNT, TYPE_ID) \ -template<> \ -struct RegTraits { \ - typedef REG RegT; \ - \ - static constexpr uint32_t kValid = 1; \ - static constexpr uint32_t kCount = COUNT; \ - static constexpr RegType kType = REG_TYPE; \ - static constexpr RegGroup kGroup = GROUP; \ - static constexpr uint32_t kSize = SIZE; \ - static constexpr TypeId kTypeId = TYPE_ID; \ - \ - static constexpr uint32_t kSignature = \ - (OperandSignature::fromOpType(OperandType::kReg) | \ - OperandSignature::fromRegType(kType) | \ - OperandSignature::fromRegGroup(kGroup) | \ - OperandSignature::fromSize(kSize)).bits(); \ - \ -} +//! \cond //! Adds constructors and member functions to a class that implements abstract register. Abstract register is register //! that doesn't have type or signature yet, it's a base class like `x86::Reg` or `arm::Reg`. #define ASMJIT_DEFINE_ABSTRACT_REG(REG, BASE) \ public: \ /*! Default constructor that only setups basics. */ \ - ASMJIT_INLINE_NODEBUG constexpr REG() noexcept \ + ASMJIT_INLINE_CONSTEXPR REG() noexcept \ : BASE(Signature{kSignature}, kIdBad) {} \ \ /*! Makes a copy of the `other` register operand. */ \ - ASMJIT_INLINE_NODEBUG constexpr REG(const REG& other) noexcept \ + ASMJIT_INLINE_CONSTEXPR REG(const REG& other) noexcept \ : BASE(other) {} \ \ /*! Makes a copy of the `other` register having id set to `id` */ \ - ASMJIT_INLINE_NODEBUG constexpr REG(const BaseReg& other, uint32_t id) noexcept \ + ASMJIT_INLINE_CONSTEXPR REG(const Reg& other, uint32_t id) noexcept \ : BASE(other, id) {} \ \ /*! Creates a register based on `signature` and `id`. */ \ - ASMJIT_INLINE_NODEBUG constexpr REG(const OperandSignature& sgn, uint32_t id) noexcept \ + ASMJIT_INLINE_CONSTEXPR REG(const OperandSignature& sgn, uint32_t id) noexcept \ : BASE(sgn, id) {} \ \ /*! Creates a completely uninitialized REG register operand (garbage). */ \ @@ -1148,26 +1742,422 @@ public: } \ \ /*! Clones the register operand. */ \ - ASMJIT_INLINE_NODEBUG constexpr REG clone() const noexcept { return REG(*this); } \ + [[nodiscard]] \ + ASMJIT_INLINE_CONSTEXPR REG clone() const noexcept { return REG(*this); } \ \ - ASMJIT_INLINE_NODEBUG REG& operator=(const REG& other) noexcept = default; + ASMJIT_INLINE_CONSTEXPR REG& operator=(const REG& other) noexcept { \ + copyFrom(other); \ + return *this; \ + } //! Adds constructors and member functions to a class that implements final register. Final registers MUST HAVE a valid //! signature. #define ASMJIT_DEFINE_FINAL_REG(REG, BASE, TRAITS) \ public: \ - static constexpr RegType kThisType = TRAITS::kType; \ - static constexpr RegGroup kThisGroup = TRAITS::kGroup; \ - static constexpr uint32_t kThisSize = TRAITS::kSize; \ - static constexpr uint32_t kSignature = TRAITS::kSignature; \ + static inline constexpr RegType kThisType = TRAITS::kType; \ + static inline constexpr RegGroup kThisGroup = TRAITS::kGroup; \ + static inline constexpr uint32_t kThisSize = TRAITS::kSize; \ + static inline constexpr uint32_t kSignature = TRAITS::kSignature; \ \ ASMJIT_DEFINE_ABSTRACT_REG(REG, BASE) \ \ /*! Creates a register operand having its id set to `id`. */ \ - ASMJIT_INLINE_NODEBUG constexpr explicit REG(uint32_t id) noexcept \ + ASMJIT_INLINE_CONSTEXPR explicit REG(uint32_t id) noexcept \ : BASE(Signature{kSignature}, id) {} + //! \endcond +//! Unified general purpose register (also acts as a base class for architecture specific GP registers). +class UniGp : public Reg { +public: + ASMJIT_DEFINE_ABSTRACT_REG(UniGp, Reg) + + //! \name Static Constructors + //! \{ + + //! Creates a new 32-bit GP register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniGp make_r32(uint32_t regId) noexcept { return UniGp(_signatureOf(), regId); } + + //! Creates a new 64-bit GP register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniGp make_r64(uint32_t regId) noexcept { return UniGp(_signatureOf(), regId); } + + //! \} + + //! \name Unified Accessors + //! \{ + + //! Clones and casts this register to a 32-bit GP register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniGp r32() const noexcept { return UniGp(_signatureOf(), id()); } + + //! Clones and casts this register to a 64-bit GP register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniGp r64() const noexcept { return UniGp(_signatureOf(), id()); } + + //! \} +}; + +//! Unified vector register (also acts as a base class for architecture specific vector registers). +class UniVec : public Reg { +public: + ASMJIT_DEFINE_ABSTRACT_REG(UniVec, Reg) + + //! \name Static Constructors + //! \{ + + //! Creates a new 128-bit vector register having the given register id `regId`. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v128(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! Creates a new 256-bit vector register having the given register id `regId`. + //! + //! \note 256-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v256(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! Creates a new 512-bit vector register having the given register id `regId`. + //! + //! \note 512-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + static ASMJIT_INLINE_CONSTEXPR UniVec make_v512(uint32_t regId) noexcept { return UniVec(_signatureOf(), regId); } + + //! \} + + //! \name Unified Accessors + //! \{ + + //! Clones and casts this register to a 128-bit vector register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v128() const noexcept { return UniVec(_signatureOf(), id()); } + + //! Clones and casts this register to a 256-bit vector register. + //! + //! \note 256-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v256() const noexcept { return UniVec(_signatureOf(), id()); } + + //! Clones and casts this register to a 512-bit vector register. + //! + //! \note 512-bit vector registers are only supported by X86|X86_64. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR UniVec v512() const noexcept { return UniVec(_signatureOf(), id()); } + + //! \} +}; +//! RegOnly is 8-byte version of `Reg` that allows to store either register or nothing. +//! +//! It's designed to decrease the space consumed by an extra "operand" in \ref BaseEmitter and \ref InstNode. +struct RegOnly { + //! \name Types + //! \{ + + using Signature = OperandSignature; + + //! \} + + //! Operand signature - only \ref OperandType::kNone and \ref OperandType::kReg are supported. + Signature _signature; + //! Physical or virtual register id. + uint32_t _id; + + //! \name Construction & Destruction + //! \{ + + //! Initializes the `RegOnly` instance to hold register `signature` and `id`. + ASMJIT_INLINE_CONSTEXPR void init(const OperandSignature& signature, uint32_t id) noexcept { + _signature = signature; + _id = id; + } + + ASMJIT_INLINE_CONSTEXPR void init(const Reg& reg) noexcept { init(reg.signature(), reg.id()); } + ASMJIT_INLINE_CONSTEXPR void init(const RegOnly& reg) noexcept { init(reg.signature(), reg.id()); } + + //! Resets the `RegOnly` members to zeros (none). + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { init(Signature::fromBits(0), 0); } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether this ExtraReg is none (same as calling `Operand_::isNone()`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isNone() const noexcept { return _signature == 0; } + + //! Tests whether the register is valid (either virtual or physical). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isReg() const noexcept { return _signature != 0; } + + //! Tests whether this is a physical register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isPhysReg() const noexcept { return _id < Reg::kIdBad; } + + //! Tests whether this is a virtual register (used by `BaseCompiler`). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVirtReg() const noexcept { return _id > Reg::kIdBad; } + + //! Returns the register signature or 0 if no register is assigned. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR OperandSignature signature() const noexcept { return _signature; } + + //! Returns the register id. + //! + //! \note Always check whether the register is assigned before using the returned identifier as + //! non-assigned `RegOnly` instance would return zero id, which is still a valid register id. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t id() const noexcept { return _id; } + + //! Sets the register id. + ASMJIT_INLINE_CONSTEXPR void setId(uint32_t id) noexcept { _id = id; } + + //! Returns the register type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType type() const noexcept { return _signature.regType(); } + + //! Returns the register group. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup group() const noexcept { return _signature.regGroup(); } + + //! \} + + //! \name Utilities + //! \{ + + //! Converts this ExtraReg to a real `RegT` operand. + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegT toReg() const noexcept { return RegT(_signature, _id); } + + //! \} +}; + +//! List of physical registers (base). +//! +//! \note List of registers is only used by some ARM instructions at the moment. +class BaseRegList : public Operand { +public: + //! \name Constants + //! \{ + + static inline constexpr uint32_t kSignature = Signature::fromOpType(OperandType::kRegList).bits(); + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a dummy register operand. + ASMJIT_INLINE_CONSTEXPR BaseRegList() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kRegList), 0, 0, 0) {} + + //! Creates a new register operand which is the same as `other` . + ASMJIT_INLINE_CONSTEXPR BaseRegList(const BaseRegList& other) noexcept + : Operand(other) {} + + //! Creates a new register operand compatible with `other`, but with a different `id`. + ASMJIT_INLINE_CONSTEXPR BaseRegList(const BaseRegList& other, RegMask regMask) noexcept + : Operand(Globals::Init, other._signature, regMask, 0, 0) {} + + //! Creates a register initialized to the given `signature` and `id`. + ASMJIT_INLINE_CONSTEXPR BaseRegList(const Signature& signature, RegMask regMask) noexcept + : Operand(Globals::Init, signature, regMask, 0, 0) {} + + ASMJIT_INLINE_NODEBUG explicit BaseRegList(Globals::NoInit_) noexcept + : Operand(Globals::NoInit) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + ASMJIT_INLINE_CONSTEXPR BaseRegList& operator=(const BaseRegList& other) noexcept { + copyFrom(other); + return *this; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the register-list is valid, which means it has a type and at least a single register in the list. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isValid() const noexcept { return bool(unsigned(_signature != 0u) & unsigned(_baseId != 0u)); } + + //! Tests whether the register type matches `type` - same as `isReg(type)`, provided for convenience. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isType(RegType type) const noexcept { return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(type); } + + //! Tests whether the register group matches `group`. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGroup(RegGroup group) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(group); } + + //! Tests whether the register is a general purpose register (any size). + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isGp() const noexcept { return isGroup(RegGroup::kGp); } + + //! Tests whether the register is a vector register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isVec() const noexcept { return isGroup(RegGroup::kVec); } + + //! Returns the register type. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType regType() const noexcept { return _signature.regType(); } + + //! Returns the register group. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegGroup regGroup() const noexcept { return _signature.regGroup(); } + + //! Returns the size of a single register in this register-list or 0 if unspecified. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t size() const noexcept { return _signature.getField(); } + + //! Returns the register list as a mask, where each bit represents one physical register. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegMask list() const noexcept { return _baseId; } + + //! Sets the register list to `mask`. + ASMJIT_INLINE_CONSTEXPR void setList(RegMask mask) noexcept { _baseId = mask; } + + //! Remoes all registers from the register-list by making the underlying register-mask zero. + ASMJIT_INLINE_CONSTEXPR void resetList() noexcept { _baseId = 0; } + + //! Adds registers passed by a register `mask` to the register-list. + ASMJIT_INLINE_CONSTEXPR void addList(RegMask mask) noexcept { _baseId |= mask; } + + //! Removes registers passed by a register `mask` to the register-list. + ASMJIT_INLINE_CONSTEXPR void clearList(RegMask mask) noexcept { _baseId &= ~mask; } + + //! Uses AND operator to combine the current register-list with other register `mask`. + ASMJIT_INLINE_CONSTEXPR void andList(RegMask mask) noexcept { _baseId &= mask; } + + //! Uses XOR operator to combine the current register-list with other register `mask`. + ASMJIT_INLINE_CONSTEXPR void xorList(RegMask mask) noexcept { _baseId ^= mask; } + + //! Checks whether a physical register `physId` is in the register-list. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasReg(uint32_t physId) const noexcept { return physId < 32u ? (_baseId & (1u << physId)) != 0 : false; } + + //! Adds a physical register `physId` to the register-list. + ASMJIT_INLINE_CONSTEXPR void addReg(uint32_t physId) noexcept { addList(1u << physId); } + + //! Removes a physical register `physId` from the register-list. + ASMJIT_INLINE_CONSTEXPR void clearReg(uint32_t physId) noexcept { clearList(1u << physId); } + + //! Clones the register-list operand. + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR BaseRegList clone() const noexcept { return BaseRegList(*this); } + + //! Casts this register to `RegT` by also changing its signature. + //! + //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegListT cloneAs() const noexcept { return RegListT(Signature(RegListT::kSignature), list()); } + + //! Casts this register to `other` by also changing its signature. + //! + //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. + template + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegListT cloneAs(const RegListT& other) const noexcept { return RegListT(other.signature(), list()); } + + //! \} +}; + +template +class RegListT : public BaseRegList { +public: + //! \name Construction & Destruction + //! \{ + + //! Creates a dummy register operand. + ASMJIT_INLINE_CONSTEXPR RegListT() noexcept + : BaseRegList() {} + + //! Creates a new register operand which is the same as `other` . + ASMJIT_INLINE_CONSTEXPR RegListT(const RegListT& other) noexcept + : BaseRegList(other) {} + + //! Creates a new register operand compatible with `other`, but with a different `id`. + ASMJIT_INLINE_CONSTEXPR RegListT(const RegListT& other, RegMask regMask) noexcept + : BaseRegList(other, regMask) {} + + //! Creates a register initialized to the given `signature` and `id`. + ASMJIT_INLINE_CONSTEXPR RegListT(const Signature& signature, RegMask regMask) noexcept + : BaseRegList(signature, regMask) {} + + //! Creates a register initialized to the given `signature` and `regs`. + ASMJIT_INLINE_NODEBUG RegListT(const Signature& signature, std::initializer_list regs) noexcept + : BaseRegList(signature, RegMask(0)) { addRegs(regs); } + + ASMJIT_INLINE_NODEBUG explicit RegListT(Globals::NoInit_) noexcept + : BaseRegList(Globals::NoInit) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + ASMJIT_INLINE_CONSTEXPR RegListT& operator=(const RegListT& other) noexcept { + copyFrom(other); + return *this; + } + + //! \} + + //! \name Accessors + //! \{ + + using BaseRegList::addList; + using BaseRegList::clearList; + using BaseRegList::andList; + using BaseRegList::xorList; + + //! Adds registers to this register-list as provided by `other` register-list. + ASMJIT_INLINE_CONSTEXPR void addList(const RegListT& other) noexcept { addList(other.list()); } + + //! Removes registers contained in `other` register-list. + ASMJIT_INLINE_CONSTEXPR void clearList(const RegListT& other) noexcept { clearList(other.list()); } + + //! Uses AND operator to combine the current register-list with `other` register-list. + ASMJIT_INLINE_CONSTEXPR void andList(const RegListT& other) noexcept { andList(other.list()); } + + //! Uses XOR operator to combine the current register-list with `other` register-list. + ASMJIT_INLINE_CONSTEXPR void xorList(const RegListT& other) noexcept { xorList(other.list()); } + + using BaseRegList::addReg; + using BaseRegList::clearReg; + + ASMJIT_INLINE_CONSTEXPR void addReg(const RegT& reg) noexcept { + if (reg.id() < 32u) { + addReg(reg.id()); + } + } + + ASMJIT_INLINE_CONSTEXPR void addRegs(std::initializer_list regs) noexcept { + for (const RegT& reg : regs) { + addReg(reg); + } + } + + ASMJIT_INLINE_CONSTEXPR void clearReg(const RegT& reg) noexcept { + if (reg.id() < 32u) { + clearReg(reg.id()); + } + } + + ASMJIT_INLINE_CONSTEXPR void clearRegs(std::initializer_list regs) noexcept { + for (const RegT& reg : regs) { + clearReg(reg); + } + } + + //! \} +}; + //! Base class for all memory operands. //! //! The data is split into the following parts: @@ -1193,27 +2183,27 @@ public: //! \{ //! Creates a default `BaseMem` operand, that points to [0]. - ASMJIT_INLINE_NODEBUG constexpr BaseMem() noexcept + ASMJIT_INLINE_CONSTEXPR BaseMem() noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kMem), 0, 0, 0) {} //! Creates a `BaseMem` operand that is a clone of `other`. - ASMJIT_INLINE_NODEBUG constexpr BaseMem(const BaseMem& other) noexcept + ASMJIT_INLINE_CONSTEXPR BaseMem(const BaseMem& other) noexcept : Operand(other) {} //! Creates a `BaseMem` operand from `baseReg` and `offset`. //! //! \note This is an architecture independent constructor that can be used to create an architecture //! independent memory operand to be used in portable code that can handle multiple architectures. - ASMJIT_INLINE_NODEBUG constexpr explicit BaseMem(const BaseReg& baseReg, int32_t offset = 0) noexcept + ASMJIT_INLINE_CONSTEXPR explicit BaseMem(const Reg& baseReg, int32_t offset = 0) noexcept : Operand(Globals::Init, - Signature::fromOpType(OperandType::kMem) | Signature::fromMemBaseType(baseReg.type()), + Signature::fromOpType(OperandType::kMem) | Signature::fromMemBaseType(baseReg.regType()), baseReg.id(), 0, uint32_t(offset)) {} //! \cond INTERNAL //! Creates a `BaseMem` operand from 4 integers as used by `Operand_` struct. - ASMJIT_INLINE_NODEBUG constexpr BaseMem(const OperandSignature& u0, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept + ASMJIT_INLINE_CONSTEXPR BaseMem(const OperandSignature& u0, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept : Operand(Globals::Init, u0, baseId, indexId, uint32_t(offset)) {} //! \endcond @@ -1222,7 +2212,7 @@ public: : Operand(Globals::NoInit) {} //! Resets the memory operand - after the reset the memory points to [0]. - ASMJIT_INLINE_NODEBUG void reset() noexcept { + ASMJIT_INLINE_CONSTEXPR void reset() noexcept { _signature = Signature::fromOpType(OperandType::kMem); _baseId = 0; _data[0] = 0; @@ -1234,7 +2224,10 @@ public: //! \name Overloaded Operators //! \{ - ASMJIT_INLINE_NODEBUG BaseMem& operator=(const BaseMem& other) noexcept { copyFrom(other); return *this; } + ASMJIT_INLINE_CONSTEXPR BaseMem& operator=(const BaseMem& other) noexcept { + copyFrom(other); + return *this; + } //! \} @@ -1242,54 +2235,66 @@ public: //! \{ //! Clones the memory operand. - ASMJIT_INLINE_NODEBUG constexpr BaseMem clone() const noexcept { return BaseMem(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR BaseMem clone() const noexcept { return BaseMem(*this); } //! Creates a new copy of this memory operand adjusted by `off`. - ASMJIT_INLINE_NODEBUG BaseMem cloneAdjusted(int64_t off) const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR BaseMem cloneAdjusted(int64_t off) const noexcept { BaseMem result(*this); result.addOffset(off); return result; } //! Tests whether this memory operand is a register home (only used by \ref asmjit_compiler) - ASMJIT_INLINE_NODEBUG constexpr bool isRegHome() const noexcept { return _signature.hasField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isRegHome() const noexcept { return _signature.hasField(); } + //! Mark this memory operand as register home (only used by \ref asmjit_compiler). - ASMJIT_INLINE_NODEBUG void setRegHome() noexcept { _signature |= Signature::kMemRegHomeFlag; } + ASMJIT_INLINE_CONSTEXPR void setRegHome() noexcept { _signature |= Signature::kMemRegHomeFlag; } + //! Marks this operand to not be a register home (only used by \ref asmjit_compiler). - ASMJIT_INLINE_NODEBUG void clearRegHome() noexcept { _signature &= ~Signature::kMemRegHomeFlag; } + ASMJIT_INLINE_CONSTEXPR void clearRegHome() noexcept { _signature &= ~Signature::kMemRegHomeFlag; } //! Tests whether the memory operand has a BASE register or label specified. - ASMJIT_INLINE_NODEBUG constexpr bool hasBase() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBase() const noexcept { return (_signature & Signature::kMemBaseTypeMask) != 0; } //! Tests whether the memory operand has an INDEX register specified. - ASMJIT_INLINE_NODEBUG constexpr bool hasIndex() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasIndex() const noexcept { return (_signature & Signature::kMemIndexTypeMask) != 0; } //! Tests whether the memory operand has BASE or INDEX register. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseOrIndex() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseOrIndex() const noexcept { return (_signature & Signature::kMemBaseIndexMask) != 0; } //! Tests whether the memory operand has BASE and INDEX register. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseAndIndex() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseAndIndex() const noexcept { return (_signature & Signature::kMemBaseTypeMask) != 0 && (_signature & Signature::kMemIndexTypeMask) != 0; } //! Tests whether the BASE operand is a label. - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseLabel() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseLabel() const noexcept { return _signature.subset(Signature::kMemBaseTypeMask) == Signature::fromMemBaseType(RegType::kLabelTag); } //! Tests whether the BASE operand is a register (registers start after `RegType::kLabelTag`). - ASMJIT_INLINE_NODEBUG constexpr bool hasBaseReg() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasBaseReg() const noexcept { return _signature.subset(Signature::kMemBaseTypeMask).bits() > Signature::fromMemBaseType(RegType::kLabelTag).bits(); } //! Tests whether the INDEX operand is a register (registers start after `RegType::kLabelTag`). - ASMJIT_INLINE_NODEBUG constexpr bool hasIndexReg() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasIndexReg() const noexcept { return _signature.subset(Signature::kMemIndexTypeMask).bits() > Signature::fromMemIndexType(RegType::kLabelTag).bits(); } @@ -1297,83 +2302,99 @@ public: //! //! \note If the returned type is one (a value never associated to a register type) the BASE is not register, but it //! is a label. One equals to `kLabelTag`. You should always check `hasBaseLabel()` before using `baseId()` result. - ASMJIT_INLINE_NODEBUG constexpr RegType baseType() const noexcept { return _signature.memBaseType(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType baseType() const noexcept { return _signature.memBaseType(); } //! Returns the type of an INDEX register (0 if this memory operand doesn't //! use the INDEX register). - ASMJIT_INLINE_NODEBUG constexpr RegType indexType() const noexcept { return _signature.memIndexType(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR RegType indexType() const noexcept { return _signature.memIndexType(); } //! This is used internally for BASE+INDEX validation. - ASMJIT_INLINE_NODEBUG constexpr uint32_t baseAndIndexTypes() const noexcept { return _signature.getField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t baseAndIndexTypes() const noexcept { return _signature.getField(); } //! Returns both BASE (4:0 bits) and INDEX (9:5 bits) types combined into a single value. //! //! \remarks Returns id of the BASE register or label (if the BASE was specified as label). - ASMJIT_INLINE_NODEBUG constexpr uint32_t baseId() const noexcept { return _baseId; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t baseId() const noexcept { return _baseId; } //! Returns the id of the INDEX register. - ASMJIT_INLINE_NODEBUG constexpr uint32_t indexId() const noexcept { return _data[kDataMemIndexId]; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t indexId() const noexcept { return _data[kDataMemIndexId]; } //! Sets the id of the BASE register (without modifying its type). - ASMJIT_INLINE_NODEBUG void setBaseId(uint32_t id) noexcept { _baseId = id; } + ASMJIT_INLINE_CONSTEXPR void setBaseId(uint32_t id) noexcept { _baseId = id; } + + //! Sets the register type of the BASE register (without modifying its id). + ASMJIT_INLINE_CONSTEXPR void setBaseType(RegType regType) noexcept { _signature.setMemBaseType(regType); } + //! Sets the id of the INDEX register (without modifying its type). - ASMJIT_INLINE_NODEBUG void setIndexId(uint32_t id) noexcept { _data[kDataMemIndexId] = id; } + ASMJIT_INLINE_CONSTEXPR void setIndexId(uint32_t id) noexcept { _data[kDataMemIndexId] = id; } + + //! Sets the register type of the INDEX register (without modifying its id). + ASMJIT_INLINE_CONSTEXPR void setIndexType(RegType regType) noexcept { _signature.setMemIndexType(regType); } //! Sets the base register to type and id of the given `base` operand. - ASMJIT_INLINE_NODEBUG void setBase(const BaseReg& base) noexcept { return _setBase(base.type(), base.id()); } + ASMJIT_INLINE_CONSTEXPR void setBase(const Reg& base) noexcept { return _setBase(base.regType(), base.id()); } + //! Sets the index register to type and id of the given `index` operand. - ASMJIT_INLINE_NODEBUG void setIndex(const BaseReg& index) noexcept { return _setIndex(index.type(), index.id()); } + ASMJIT_INLINE_CONSTEXPR void setIndex(const Reg& index) noexcept { return _setIndex(index.regType(), index.id()); } //! \cond INTERNAL - ASMJIT_INLINE_NODEBUG void _setBase(RegType type, uint32_t id) noexcept { + ASMJIT_INLINE_CONSTEXPR void _setBase(RegType type, uint32_t id) noexcept { _signature.setField(uint32_t(type)); _baseId = id; } - ASMJIT_INLINE_NODEBUG void _setIndex(RegType type, uint32_t id) noexcept { + ASMJIT_INLINE_CONSTEXPR void _setIndex(RegType type, uint32_t id) noexcept { _signature.setField(uint32_t(type)); _data[kDataMemIndexId] = id; } //! \endcond //! Resets the memory operand's BASE register or label. - ASMJIT_INLINE_NODEBUG void resetBase() noexcept { _setBase(RegType::kNone, 0); } + ASMJIT_INLINE_CONSTEXPR void resetBase() noexcept { _setBase(RegType::kNone, 0); } //! Resets the memory operand's INDEX register. - ASMJIT_INLINE_NODEBUG void resetIndex() noexcept { _setIndex(RegType::kNone, 0); } - - //! Sets the memory operand size (in bytes). - ASMJIT_INLINE_NODEBUG void setSize(uint32_t size) noexcept { _signature.setField(size); } + ASMJIT_INLINE_CONSTEXPR void resetIndex() noexcept { _setIndex(RegType::kNone, 0); } //! Tests whether the memory operand has a 64-bit offset or absolute address. //! //! If this is true then `hasBase()` must always report false. - ASMJIT_INLINE_NODEBUG constexpr bool isOffset64Bit() const noexcept { return baseType() == RegType::kNone; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isOffset64Bit() const noexcept { return baseType() == RegType::kNone; } //! Tests whether the memory operand has a non-zero offset or absolute address. - ASMJIT_INLINE_NODEBUG constexpr bool hasOffset() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool hasOffset() const noexcept { return (_data[kDataMemOffsetLo] | uint32_t(_baseId & Support::bitMaskFromBool(isOffset64Bit()))) != 0; } //! Returns either relative offset or absolute address as 64-bit integer. - ASMJIT_INLINE_NODEBUG constexpr int64_t offset() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int64_t offset() const noexcept { return isOffset64Bit() ? int64_t(uint64_t(_data[kDataMemOffsetLo]) | (uint64_t(_baseId) << 32)) : int64_t(int32_t(_data[kDataMemOffsetLo])); // Sign extend 32-bit offset. } //! Returns a 32-bit low part of a 64-bit offset or absolute address. - ASMJIT_INLINE_NODEBUG constexpr int32_t offsetLo32() const noexcept { return int32_t(_data[kDataMemOffsetLo]); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int32_t offsetLo32() const noexcept { return int32_t(_data[kDataMemOffsetLo]); } + //! Returns a 32-but high part of a 64-bit offset or absolute address. //! //! \note This function is UNSAFE and returns garbage if `isOffset64Bit()` //! returns false. Never use it blindly without checking it first. - ASMJIT_INLINE_NODEBUG constexpr int32_t offsetHi32() const noexcept { return int32_t(_baseId); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int32_t offsetHi32() const noexcept { return int32_t(_baseId); } //! Sets a 64-bit offset or an absolute address to `offset`. //! //! \note This functions attempts to set both high and low parts of a 64-bit offset, however, if the operand has //! a BASE register it will store only the low 32 bits of the offset / address as there is no way to store both //! BASE and 64-bit offset, and there is currently no architecture that has such capability targeted by AsmJit. - inline void setOffset(int64_t offset) noexcept { + ASMJIT_INLINE_CONSTEXPR void setOffset(int64_t offset) noexcept { uint32_t lo = uint32_t(uint64_t(offset) & 0xFFFFFFFFu); uint32_t hi = uint32_t(uint64_t(offset) >> 32); uint32_t hiMsk = Support::bitMaskFromBool(isOffset64Bit()); @@ -1381,8 +2402,9 @@ public: _data[kDataMemOffsetLo] = lo; _baseId = (hi & hiMsk) | (_baseId & ~hiMsk); } + //! Sets a low 32-bit offset to `offset` (don't use without knowing how BaseMem works). - inline void setOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] = uint32_t(offset); } + ASMJIT_INLINE_CONSTEXPR void setOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] = uint32_t(offset); } //! Adjusts the offset by `offset`. //! @@ -1390,7 +2412,7 @@ public: //! there is a BASE register and the offset is only 32 bits anyway. //! Adjusts the memory operand offset by a `offset`. - inline void addOffset(int64_t offset) noexcept { + ASMJIT_INLINE_CONSTEXPR void addOffset(int64_t offset) noexcept { if (isOffset64Bit()) { int64_t result = offset + int64_t(uint64_t(_data[kDataMemOffsetLo]) | (uint64_t(_baseId) << 32)); _data[kDataMemOffsetLo] = uint32_t(uint64_t(result) & 0xFFFFFFFFu); @@ -1402,13 +2424,13 @@ public: } //! Adds `offset` to a low 32-bit offset part (don't use without knowing how BaseMem works). - ASMJIT_INLINE_NODEBUG void addOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] += uint32_t(offset); } + ASMJIT_INLINE_CONSTEXPR void addOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] += uint32_t(offset); } //! Resets the memory offset to zero. - ASMJIT_INLINE_NODEBUG void resetOffset() noexcept { setOffset(0); } + ASMJIT_INLINE_CONSTEXPR void resetOffset() noexcept { setOffset(0); } //! Resets the lo part of the memory offset to zero (don't use without knowing how BaseMem works). - ASMJIT_INLINE_NODEBUG void resetOffsetLo32() noexcept { setOffsetLo32(0); } + ASMJIT_INLINE_CONSTEXPR void resetOffsetLo32() noexcept { setOffsetLo32(0); } //! \} }; @@ -1427,30 +2449,26 @@ public: //! \cond INTERNAL template struct IsConstexprConstructibleAsImmType - : public std::integral_constant::value || - std::is_pointer::value || - std::is_integral::value || - std::is_function::value> {}; + : public std::integral_constant || std::is_pointer_v || std::is_integral_v || std::is_function_v> {}; template struct IsConvertibleToImmType - : public std::integral_constant::value || - std::is_floating_point::value> {}; + : public std::integral_constant::value || std::is_floating_point_v> {}; //! \endcond //! \name Construction & Destruction //! \{ //! Creates a new immediate value (initial value is 0). - ASMJIT_INLINE_NODEBUG constexpr Imm() noexcept + ASMJIT_INLINE_CONSTEXPR Imm() noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kImm), 0, 0, 0) {} //! Creates a new immediate value from `other`. - ASMJIT_INLINE_NODEBUG constexpr Imm(const Imm& other) noexcept + ASMJIT_INLINE_CONSTEXPR Imm(const Imm& other) noexcept : Operand(other) {} //! Creates a new immediate value from ARM/AArch64 specific `shift`. - ASMJIT_INLINE_NODEBUG constexpr Imm(const arm::Shift& shift) noexcept + ASMJIT_INLINE_CONSTEXPR Imm(const arm::Shift& shift) noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(uint32_t(shift.op())), 0, @@ -1461,8 +2479,8 @@ public: //! to `predicate`. //! //! \note Predicate is currently only used by ARM architectures. - template::type>::value>::type> - ASMJIT_INLINE_NODEBUG constexpr Imm(const T& val, const uint32_t predicate = 0) noexcept + template>::value>::type> + ASMJIT_INLINE_CONSTEXPR Imm(const T& val, const uint32_t predicate = 0) noexcept : Operand(Globals::Init, Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(predicate), 0, @@ -1492,7 +2510,10 @@ public: //! \{ //! Assigns the value of the `other` operand to this immediate. - ASMJIT_INLINE_NODEBUG Imm& operator=(const Imm& other) noexcept { copyFrom(other); return *this; } + ASMJIT_INLINE_CONSTEXPR Imm& operator=(const Imm& other) noexcept { + copyFrom(other); + return *this; + } //! \} @@ -1500,73 +2521,100 @@ public: //! \{ //! Returns immediate type. - ASMJIT_INLINE_NODEBUG constexpr ImmType type() const noexcept { return (ImmType)_signature.getField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR ImmType type() const noexcept { return (ImmType)_signature.getField(); } + //! Sets the immediate type to `type`. - ASMJIT_INLINE_NODEBUG void setType(ImmType type) noexcept { _signature.setField(uint32_t(type)); } + ASMJIT_INLINE_CONSTEXPR void setType(ImmType type) noexcept { _signature.setField(uint32_t(type)); } + //! Resets immediate type to \ref ImmType::kInt. - ASMJIT_INLINE_NODEBUG void resetType() noexcept { setType(ImmType::kInt); } + ASMJIT_INLINE_CONSTEXPR void resetType() noexcept { setType(ImmType::kInt); } //! Returns operation predicate of the immediate. //! //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp //! of the immediate. - ASMJIT_INLINE_NODEBUG constexpr uint32_t predicate() const noexcept { return _signature.getField(); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t predicate() const noexcept { return _signature.getField(); } //! Sets operation predicate of the immediate to `predicate`. //! //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp //! of the immediate. - ASMJIT_INLINE_NODEBUG void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } + ASMJIT_INLINE_CONSTEXPR void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } //! Resets the shift operation type of the immediate to the default value (no operation). - ASMJIT_INLINE_NODEBUG void resetPredicate() noexcept { _signature.setField(0); } + ASMJIT_INLINE_CONSTEXPR void resetPredicate() noexcept { _signature.setField(0); } //! Returns the immediate value as `int64_t`, which is the internal format Imm uses. - ASMJIT_INLINE_NODEBUG constexpr int64_t value() const noexcept { + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int64_t value() const noexcept { return int64_t((uint64_t(_data[kDataImmValueHi]) << 32) | _data[kDataImmValueLo]); } //! Tests whether this immediate value is integer of any size. - ASMJIT_INLINE_NODEBUG constexpr uint32_t isInt() const noexcept { return type() == ImmType::kInt; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t isInt() const noexcept { return type() == ImmType::kInt; } + //! Tests whether this immediate value is a double precision floating point value. - ASMJIT_INLINE_NODEBUG constexpr uint32_t isDouble() const noexcept { return type() == ImmType::kDouble; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t isDouble() const noexcept { return type() == ImmType::kDouble; } //! Tests whether the immediate can be casted to 8-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr bool isInt8() const noexcept { return type() == ImmType::kInt && Support::isInt8(value()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isInt8() const noexcept { return type() == ImmType::kInt && Support::isInt8(value()); } + //! Tests whether the immediate can be casted to 8-bit unsigned integer. - ASMJIT_INLINE_NODEBUG constexpr bool isUInt8() const noexcept { return type() == ImmType::kInt && Support::isUInt8(value()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isUInt8() const noexcept { return type() == ImmType::kInt && Support::isUInt8(value()); } + //! Tests whether the immediate can be casted to 16-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr bool isInt16() const noexcept { return type() == ImmType::kInt && Support::isInt16(value()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isInt16() const noexcept { return type() == ImmType::kInt && Support::isInt16(value()); } + //! Tests whether the immediate can be casted to 16-bit unsigned integer. - ASMJIT_INLINE_NODEBUG constexpr bool isUInt16() const noexcept { return type() == ImmType::kInt && Support::isUInt16(value()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isUInt16() const noexcept { return type() == ImmType::kInt && Support::isUInt16(value()); } + //! Tests whether the immediate can be casted to 32-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr bool isInt32() const noexcept { return type() == ImmType::kInt && Support::isInt32(value()); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isInt32() const noexcept { return type() == ImmType::kInt && Support::isInt32(value()); } + //! Tests whether the immediate can be casted to 32-bit unsigned integer. - ASMJIT_INLINE_NODEBUG constexpr bool isUInt32() const noexcept { return type() == ImmType::kInt && _data[kDataImmValueHi] == 0; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR bool isUInt32() const noexcept { return type() == ImmType::kInt && _data[kDataImmValueHi] == 0; } //! Returns the immediate value casted to `T`. //! //! The value is masked before it's casted to `T` so the returned value is simply the representation of `T` //! considering the original value's lowest bits. template + [[nodiscard]] ASMJIT_INLINE_NODEBUG T valueAs() const noexcept { return Support::immediateToT(value()); } //! Returns low 32-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr int32_t int32Lo() const noexcept { return int32_t(_data[kDataImmValueLo]); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int32_t int32Lo() const noexcept { return int32_t(_data[kDataImmValueLo]); } + //! Returns high 32-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr int32_t int32Hi() const noexcept { return int32_t(_data[kDataImmValueHi]); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR int32_t int32Hi() const noexcept { return int32_t(_data[kDataImmValueHi]); } + //! Returns low 32-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr uint32_t uint32Lo() const noexcept { return _data[kDataImmValueLo]; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t uint32Lo() const noexcept { return _data[kDataImmValueLo]; } + //! Returns high 32-bit signed integer. - ASMJIT_INLINE_NODEBUG constexpr uint32_t uint32Hi() const noexcept { return _data[kDataImmValueHi]; } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR uint32_t uint32Hi() const noexcept { return _data[kDataImmValueHi]; } //! Sets immediate value to `val`, the value is casted to a signed 64-bit integer. template ASMJIT_INLINE_NODEBUG void setValue(const T& val) noexcept { - _setValueInternal(Support::immediateFromT(val), std::is_floating_point::value ? ImmType::kDouble : ImmType::kInt); + _setValueInternal(Support::immediateFromT(val), std::is_floating_point_v ? ImmType::kDouble : ImmType::kInt); } - ASMJIT_INLINE_NODEBUG void _setValueInternal(int64_t val, ImmType type) noexcept { + ASMJIT_INLINE_CONSTEXPR void _setValueInternal(int64_t val, ImmType type) noexcept { setType(type); _data[kDataImmValueHi] = uint32_t(uint64_t(val) >> 32); _data[kDataImmValueLo] = uint32_t(uint64_t(val) & 0xFFFFFFFFu); @@ -1578,14 +2626,21 @@ public: //! \{ //! Clones the immediate operand. - ASMJIT_INLINE_NODEBUG constexpr Imm clone() const noexcept { return Imm(*this); } + [[nodiscard]] + ASMJIT_INLINE_CONSTEXPR Imm clone() const noexcept { return Imm(*this); } + //! Sign extend the integer immediate value from 8-bit signed integer to 64 bits. ASMJIT_INLINE_NODEBUG void signExtend8Bits() noexcept { setValue(int64_t(valueAs())); } + //! Sign extend the integer immediate value from 16-bit signed integer to 64 bits. ASMJIT_INLINE_NODEBUG void signExtend16Bits() noexcept { setValue(int64_t(valueAs())); } + //! Sign extend the integer immediate value from 32-bit signed integer to 64 bits. ASMJIT_INLINE_NODEBUG void signExtend32Bits() noexcept { setValue(int64_t(valueAs())); } + //! Zero extend the integer immediate value from 8-bit unsigned integer to 64 bits. ASMJIT_INLINE_NODEBUG void zeroExtend8Bits() noexcept { setValue(valueAs()); } + //! Zero extend the integer immediate value from 16-bit unsigned integer to 64 bits. ASMJIT_INLINE_NODEBUG void zeroExtend16Bits() noexcept { setValue(valueAs()); } + //! Zero extend the integer immediate value from 32-bit unsigned integer to 64 bits. ASMJIT_INLINE_NODEBUG void zeroExtend32Bits() noexcept { _data[kDataImmValueHi] = 0u; } //! \} @@ -1593,7 +2648,8 @@ public: //! Creates a new immediate operand. template -static ASMJIT_INLINE_NODEBUG constexpr Imm imm(const T& val) noexcept { return Imm(val); } +[[nodiscard]] +static ASMJIT_INLINE_CONSTEXPR Imm imm(const T& val) noexcept { return Imm(val); } //! \} @@ -1609,18 +2665,20 @@ namespace Support { template struct ForwardOpImpl { + [[nodiscard]] static ASMJIT_INLINE_NODEBUG const T& forward(const T& value) noexcept { return value; } }; template struct ForwardOpImpl { + [[nodiscard]] static ASMJIT_INLINE_NODEBUG Imm forward(const T& value) noexcept { return Imm(value); } }; //! Either forwards operand T or returns a new operand that wraps it if T is a type convertible to operand. -//! At the moment this is only used to convert integers, floats, and enumarations to \ref Imm operands. +//! At the moment this is only used to convert integers, floats, and enumerations to \ref Imm operands. template -struct ForwardOp : public ForwardOpImpl::type>::value> {}; +struct ForwardOp : public ForwardOpImpl>::value> {}; } // {Support} //! \endcond diff --git a/pe-packer/asmjit/core/osutils.cpp b/pe-packer/asmjit/core/osutils.cpp index 92cd7d5..f716d27 100644 --- a/pe-packer/asmjit/core/osutils.cpp +++ b/pe-packer/asmjit/core/osutils.cpp @@ -1,96 +1,27 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #include "../core/api-build_p.h" #include "../core/osutils_p.h" #include "../core/support.h" -#if defined(_WIN32) - #include -#else +#if !defined(_WIN32) #include #include - #if defined(__APPLE__) - #include - #else - #include - #endif #endif ASMJIT_BEGIN_NAMESPACE -uint32_t OSUtils::getTickCount() noexcept { -#if defined(_WIN32) - enum HiResStatus : uint32_t { - kHiResUnknown = 0, - kHiResAvailable = 1, - kHiResNotAvailable = 2 - }; - - static std::atomic _hiResStatus(kHiResUnknown); - static volatile double _hiResFreq(0); - - uint32_t status = _hiResStatus.load(); - LARGE_INTEGER now, qpf; - - if (status != kHiResNotAvailable && ::QueryPerformanceCounter(&now)) { - double freq = _hiResFreq; - if (status == kHiResUnknown) { - // Detects the availability of high resolution counter. - if (::QueryPerformanceFrequency(&qpf)) { - freq = double(qpf.QuadPart) / 1000.0; - _hiResFreq = freq; - _hiResStatus.compare_exchange_strong(status, kHiResAvailable); - status = kHiResAvailable; - } - else { - // High resolution not available. - _hiResStatus.compare_exchange_strong(status, kHiResNotAvailable); - } - } - - if (status == kHiResAvailable) - return uint32_t(uint64_t(int64_t(double(now.QuadPart) / freq)) & 0xFFFFFFFFu); - } - - // Bail to `GetTickCount()` if we cannot use high resolution. - return ::GetTickCount(); -#elif defined(__APPLE__) - // See Apple's QA1398. - static mach_timebase_info_data_t _machTime; - - uint32_t denom = _machTime.denom; - if (ASMJIT_UNLIKELY(!denom)) { - if (mach_timebase_info(&_machTime) != KERN_SUCCESS || !(denom = _machTime.denom)) - return 0; - } - - // `mach_absolute_time()` returns nanoseconds, we want milliseconds. - uint64_t t = mach_absolute_time() / 1000000u; - t = (t * _machTime.numer) / _machTime.denom; - return uint32_t(t & 0xFFFFFFFFu); -#elif defined(_POSIX_MONOTONIC_CLOCK) && _POSIX_MONOTONIC_CLOCK >= 0 - struct timespec ts; - if (ASMJIT_UNLIKELY(clock_gettime(CLOCK_MONOTONIC, &ts) != 0)) - return 0; - - uint64_t t = (uint64_t(ts.tv_sec ) * 1000u) + (uint64_t(ts.tv_nsec) / 1000000u); - return uint32_t(t & 0xFFFFFFFFu); -#else - #pragma message("[asmjit] OSUtils::getTickCount() doesn't have implementation for the target OS.") - return 0; -#endif -} - #if !defined(_WIN32) Error OSUtils::readFile(const char* name, String& dst, size_t maxSize) noexcept { char* buffer = dst.prepare(String::ModifyOp::kAssign, maxSize); - if (ASMJIT_UNLIKELY(!buffer)) + if (ASMJIT_UNLIKELY(!buffer)) { return DebugUtils::errored(kErrorOutOfMemory); + } - int fd = ::open(name, O_RDONLY); + int fd = ASMJIT_FILE64_API(::open)(name, O_RDONLY); if (fd < 0) { dst.clear(); return DebugUtils::errored(kErrorFailedToOpenFile); diff --git a/pe-packer/asmjit/core/osutils.h b/pe-packer/asmjit/core/osutils.h index 67c8ebf..8e4fdeb 100644 --- a/pe-packer/asmjit/core/osutils.h +++ b/pe-packer/asmjit/core/osutils.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OSUTILS_H_INCLUDED @@ -13,14 +13,6 @@ ASMJIT_BEGIN_NAMESPACE //! \addtogroup asmjit_utilities //! \{ -//! Operating system utilities. -namespace OSUtils { - -//! Gets the current CPU tick count, used for benchmarking (1ms resolution). -ASMJIT_API uint32_t getTickCount() noexcept; - -} // {OSUtils} - //! \cond INTERNAL //! Lock. //! @@ -43,7 +35,7 @@ public: Handle _handle; #pragma pack(pop) #elif !defined(__EMSCRIPTEN__) - typedef pthread_mutex_t Handle; + using Handle = pthread_mutex_t; Handle _handle; #endif diff --git a/pe-packer/asmjit/core/osutils_p.h b/pe-packer/asmjit/core/osutils_p.h index 0d9d369..0969fe2 100644 --- a/pe-packer/asmjit/core/osutils_p.h +++ b/pe-packer/asmjit/core/osutils_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_OSUTILS_P_H_INCLUDED diff --git a/pe-packer/asmjit/core/raassignment_p.h b/pe-packer/asmjit/core/raassignment_p.h index 0865ece..fd357b8 100644 --- a/pe-packer/asmjit/core/raassignment_p.h +++ b/pe-packer/asmjit/core/raassignment_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED @@ -66,24 +66,26 @@ public: //! PhysReg to WorkReg mapping. uint32_t workIds[1 /* ... */]; + [[nodiscard]] static ASMJIT_INLINE_NODEBUG size_t sizeOf(size_t count) noexcept { - return sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t); + return Support::alignUp(sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t), Globals::kZoneAlignment); } - inline void reset(size_t count) noexcept { + ASMJIT_INLINE void reset(size_t count) noexcept { assigned.reset(); dirty.reset(); - for (size_t i = 0; i < count; i++) + for (size_t i = 0; i < count; i++) { workIds[i] = kWorkNone; + } } - inline void copyFrom(const PhysToWorkMap* other, size_t count) noexcept { + ASMJIT_INLINE void copyFrom(const PhysToWorkMap* other, size_t count) noexcept { size_t size = sizeOf(count); memcpy(this, other, size); } - inline void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept { + ASMJIT_INLINE void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept { assigned.clear(group, Support::bitMask(physId)); dirty.clear(group, Support::bitMask(physId)); workIds[indexInWorkIds] = kWorkNone; @@ -94,19 +96,22 @@ public: //! WorkReg to PhysReg mapping uint8_t physIds[1 /* ... */]; - static inline size_t sizeOf(size_t count) noexcept { - return size_t(count) * sizeof(uint8_t); + [[nodiscard]] + static ASMJIT_INLINE_NODEBUG size_t sizeOf(size_t count) noexcept { + return Support::alignUp(size_t(count) * sizeof(uint8_t), Globals::kZoneAlignment); } - inline void reset(size_t count) noexcept { - for (size_t i = 0; i < count; i++) + ASMJIT_INLINE void reset(size_t count) noexcept { + for (size_t i = 0; i < count; i++) { physIds[i] = kPhysNone; + } } - inline void copyFrom(const WorkToPhysMap* other, size_t count) noexcept { + ASMJIT_INLINE void copyFrom(const WorkToPhysMap* other, size_t count) noexcept { size_t size = sizeOf(count); - if (ASMJIT_LIKELY(size)) + if (ASMJIT_LIKELY(size)) { memcpy(this, other, size); + } } }; @@ -132,7 +137,7 @@ public: resetMaps(); } - ASMJIT_FORCE_INLINE void initLayout(const RARegCount& physCount, const RAWorkRegs& workRegs) noexcept { + ASMJIT_INLINE void initLayout(const RARegCount& physCount, const RAWorkRegs& workRegs) noexcept { // Layout must be initialized before data. ASMJIT_ASSERT(_physToWorkMap == nullptr); ASMJIT_ASSERT(_workToPhysMap == nullptr); @@ -145,14 +150,15 @@ public: _layout.workRegs = &workRegs; } - ASMJIT_FORCE_INLINE void initMaps(PhysToWorkMap* physToWorkMap, WorkToPhysMap* workToPhysMap) noexcept { + ASMJIT_INLINE void initMaps(PhysToWorkMap* physToWorkMap, WorkToPhysMap* workToPhysMap) noexcept { _physToWorkMap = physToWorkMap; _workToPhysMap = workToPhysMap; - for (RegGroup group : RegGroupVirtValues{}) + for (RegGroup group : RegGroupVirtValues{}) { _physToWorkIds[group] = physToWorkMap->workIds + _layout.physIndex.get(group); + } } - ASMJIT_FORCE_INLINE void resetMaps() noexcept { + ASMJIT_INLINE void resetMaps() noexcept { _physToWorkMap = nullptr; _workToPhysMap = nullptr; _physToWorkIds.fill(nullptr); @@ -163,17 +169,31 @@ public: //! \name Accessors //! \{ + [[nodiscard]] ASMJIT_INLINE_NODEBUG PhysToWorkMap* physToWorkMap() const noexcept { return _physToWorkMap; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG WorkToPhysMap* workToPhysMap() const noexcept { return _workToPhysMap; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG RARegMask& assigned() noexcept { return _physToWorkMap->assigned; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const RARegMask& assigned() const noexcept { return _physToWorkMap->assigned; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG uint32_t assigned(RegGroup group) const noexcept { return _physToWorkMap->assigned[group]; } + [[nodiscard]] ASMJIT_INLINE_NODEBUG RARegMask& dirty() noexcept { return _physToWorkMap->dirty; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG const RARegMask& dirty() const noexcept { return _physToWorkMap->dirty; } + + [[nodiscard]] ASMJIT_INLINE_NODEBUG RegMask dirty(RegGroup group) const noexcept { return _physToWorkMap->dirty[group]; } + [[nodiscard]] inline uint32_t workToPhysId(RegGroup group, uint32_t workId) const noexcept { DebugUtils::unused(group); ASMJIT_ASSERT(workId != kWorkNone); @@ -181,16 +201,19 @@ public: return _workToPhysMap->physIds[workId]; } + [[nodiscard]] inline uint32_t physToWorkId(RegGroup group, uint32_t physId) const noexcept { ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); return _physToWorkIds[group][physId]; } + [[nodiscard]] inline bool isPhysAssigned(RegGroup group, uint32_t physId) const noexcept { ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); return Support::bitTest(_physToWorkMap->assigned[group], physId); } + [[nodiscard]] inline bool isPhysDirty(RegGroup group, uint32_t physId) const noexcept { ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); return Support::bitTest(_physToWorkMap->dirty[group], physId); @@ -304,14 +327,14 @@ public: //! \name Utilities //! \{ - ASMJIT_FORCE_INLINE void swap(RAAssignment& other) noexcept { + ASMJIT_INLINE void swap(RAAssignment& other) noexcept { std::swap(_workToPhysMap, other._workToPhysMap); std::swap(_physToWorkMap, other._physToWorkMap); _physToWorkIds.swap(other._physToWorkIds); } inline void assignWorkIdsFromPhysIds() noexcept { - memset(_workToPhysMap, uint8_t(BaseReg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount)); + memset(_workToPhysMap, uint8_t(Reg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount)); for (RegGroup group : RegGroupVirtValues{}) { uint32_t physBaseIndex = _layout.physIndex[group]; @@ -342,6 +365,7 @@ public: } // Not really useful outside of debugging. + [[nodiscard]] bool equals(const RAAssignment& other) const noexcept { // Layout should always match. if (_layout.physIndex != other._layout.physIndex || @@ -357,15 +381,17 @@ public: for (uint32_t physId = 0; physId < physTotal; physId++) { uint32_t thisWorkId = _physToWorkMap->workIds[physId]; uint32_t otherWorkId = other._physToWorkMap->workIds[physId]; - if (thisWorkId != otherWorkId) + if (thisWorkId != otherWorkId) { return false; + } } for (uint32_t workId = 0; workId < workCount; workId++) { uint32_t thisPhysId = _workToPhysMap->physIds[workId]; uint32_t otherPhysId = other._workToPhysMap->physIds[workId]; - if (thisPhysId != otherPhysId) + if (thisPhysId != otherPhysId) { return false; + } } if (_physToWorkMap->assigned != other._physToWorkMap->assigned || diff --git a/pe-packer/asmjit/core/rabuilders_p.h b/pe-packer/asmjit/core/rabuilders_p.h index 9d9b428..e4f520e 100644 --- a/pe-packer/asmjit/core/rabuilders_p.h +++ b/pe-packer/asmjit/core/rabuilders_p.h @@ -1,6 +1,6 @@ // This file is part of AsmJit project // -// See asmjit.h or LICENSE.md for license and copyright information +// See or LICENSE.md for license and copyright information // SPDX-License-Identifier: Zlib #ifndef ASMJIT_CORE_RABUILDERS_P_H_INCLUDED @@ -21,15 +21,18 @@ ASMJIT_BEGIN_NAMESPACE template class RACFGBuilderT { public: - enum : uint32_t { - kRootIndentation = 2, - kCodeIndentation = 4, + //! \name Constants + //! \{ - // NOTE: This is a bit hacky. There are some nodes which are processed twice (see `onBeforeInvoke()` and - // `onBeforeRet()`) as they can insert some nodes around them. Since we don't have any flags to mark these - // we just use their position that is [at that time] unassigned. - kNodePositionDidOnBefore = 0xFFFFFFFFu - }; + static inline constexpr uint32_t kRootIndentation = 2; + static inline constexpr uint32_t kCodeIndentation = 4; + + // NOTE: This is a bit hacky. There are some nodes which are processed twice (see `onBeforeInvoke()` and + // `onBeforeRet()`) as they can insert some nodes around them. Since we don't have any flags to mark these + // we just use their position that is [at that time] unassigned. + static inline constexpr uint32_t kNodePositionDidOnBefore = 0xFFFFFFFFu; + + //! \} //! \name Members //! \{ @@ -60,17 +63,20 @@ public: _cc(pass->cc()) { #ifndef ASMJIT_NO_LOGGING _logger = _pass->hasDiagnosticOption(DiagnosticOptions::kRADebugCFG) ? _pass->logger() : nullptr; - if (_logger) + if (_logger) { _formatOptions = _logger->options(); + } #endif } - inline BaseCompiler* cc() const noexcept { return _cc; } + [[nodiscard]] + ASMJIT_INLINE_NODEBUG BaseCompiler* cc() const noexcept { return _cc; } //! \name Run //! \{ //! Called per function by an architecture-specific CFG builder. + [[nodiscard]] Error run() noexcept { log("[BuildCFG]\n"); ASMJIT_PROPAGATE(prepare()); @@ -80,8 +86,10 @@ public: RABlock* entryBlock = _curBlock; BaseNode* node = _funcNode->next(); - if (ASMJIT_UNLIKELY(!node)) + + if (ASMJIT_UNLIKELY(!node)) { return DebugUtils::errored(kErrorInvalidState); + } _curBlock->setFirst(_funcNode); _curBlock->setLast(_funcNode); @@ -117,16 +125,19 @@ public: // the first possible inserted node by `onBeforeInvoke()` or `onBeforeRet()`. BaseNode* prev = node->prev(); - if (node->type() == NodeType::kInvoke) + if (node->type() == NodeType::kInvoke) { ASMJIT_PROPAGATE(static_cast(this)->onBeforeInvoke(node->as())); - else + } + else { ASMJIT_PROPAGATE(static_cast(this)->onBeforeRet(node->as())); + } if (prev != node->prev()) { // If this was the first node in the block and something was // inserted before it then we have to update the first block. - if (_curBlock->first() == node) + if (_curBlock->first() == node) { _curBlock->setFirst(prev->next()); + } node->setPosition(kNodePositionDidOnBefore); node = prev->next(); @@ -148,7 +159,7 @@ public: logNode(inst, kCodeIndentation); InstControlFlow cf = InstControlFlow::kRegular; - ib.reset(); + ib.reset(_curBlock->blockId()); ASMJIT_PROPAGATE(static_cast(this)->onInst(inst, cf, ib)); if (node->isInvoke()) { @@ -166,7 +177,7 @@ public: RAWorkReg* workReg = _pass->workRegById(tiedReg.workId()); if (workReg->group() == RegGroup::kGp) { uint32_t useId = tiedReg.useId(); - if (useId == BaseReg::kIdBad) { + if (useId == Reg::kIdBad) { useId = _pass->_scratchRegIndexes[fixedRegCount++]; tiedReg.setUseId(useId); } @@ -191,8 +202,9 @@ public: const Operand* opArray = inst->operands(); // Cannot jump anywhere without operands. - if (ASMJIT_UNLIKELY(!opCount)) + if (ASMJIT_UNLIKELY(!opCount)) { return DebugUtils::errored(kErrorInvalidState); + } if (opArray[opCount - 1].isLabel()) { // Labels are easy for constructing the control flow. @@ -200,8 +212,9 @@ public: ASMJIT_PROPAGATE(cc()->labelNodeOf(&labelNode, opArray[opCount - 1].as