diff --git a/CMakeSettings.json b/CMakeSettings.json index 3ece806..c85b402 100644 --- a/CMakeSettings.json +++ b/CMakeSettings.json @@ -65,7 +65,7 @@ { "name": "x86-Linux-GCC-Release", "generator": "Unix Makefiles", - "configurationType": "RelWithDebInfo", + "configurationType": "MinSizeRel", "cmakeExecutable": "cmake", "remoteCopySourcesExclusionList": [ ".vs", ".git", "out" ], "cmakeCommandArgs": "", @@ -79,8 +79,7 @@ "remoteCopySources": true, "rsyncCommandArgs": "-t --delete", "remoteCopyBuildOutput": false, - "remoteCopySourcesMethod": "rsync", - "variables": [] + "remoteCopySourcesMethod": "rsync" } ] } \ No newline at end of file diff --git a/src/Assembler.h b/src/Assembler.h new file mode 100644 index 0000000..f2ae189 --- /dev/null +++ b/src/Assembler.h @@ -0,0 +1,60 @@ +#pragma once +#ifndef __ASSEMBLER_H__ +#define __ASSEMBLER_H__ +#include +#include "asmjit/asmjit.h" + +using namespace asmjit; +using namespace x86; +class CAssembler +{ +public: + CAssembler(Arch arch = Arch::kX86) : + m_Environment(arch) + { + Prepare(); + } + + ~CAssembler() {} +public: + void Prepare() + { + m_CodeHolder.reset(); + m_CodeHolder.init(m_Environment, 0); + m_CodeHolder.attach(&m_Assembler); + //m_Assembler.addDiagnosticOptions(asmjit::DiagnosticOptions::kValidateAssembler); + } + + Assembler* GetAssembler() + { + return &m_Assembler; + } + + Section* GetEmitCode() + { + return m_CodeHolder.textSection(); + } + + std::vector GetBytes(bool clear = true) + { + Section* pSection = this->GetEmitCode(); + std::vector AsmCode; + AsmCode.resize(pSection->bufferSize()); + memcpy(AsmCode.data(), pSection->data(), pSection->bufferSize()); + if (clear) this->Prepare(); + return AsmCode; + } + + +protected: + + +private: + CodeHolder m_CodeHolder; + Environment m_Environment; + Assembler m_Assembler; +}; + + +#endif // __ASSEMBLER_H__ + diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 71272f6..eea4199 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -6,8 +6,24 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32 -fpic") +file(GLOB_RECURSE DIR_ASMJIT_LIB_SRCS + "asmjit/core/*.c" + "asmjit/core/*.cc" + "asmjit/core/*.cpp" + "asmjit/core/*.cxx" + "asmjit/arm/*.c" + "asmjit/arm/*.cc" + "asmjit/arm/*.cpp" + "asmjit/arm/*.cxx" + "asmjit/x86/*.c" + "asmjit/x86/*.cc" + "asmjit/x86/*.cpp" + "asmjit/x86/*.cxx" +) + #添加头文件目录 include_directories(${PROJECT_SOURCE_DIR}/include) +add_compile_definitions(ASMJIT_STATIC) # 添加库源文件 set(LIB_SOURCE @@ -18,7 +34,7 @@ set(LIB_SOURCE ) # 添加动态库，关键词为shared -ADD_LIBRARY(hook_shared SHARED ${LIB_SOURCE}) +ADD_LIBRARY(hook_shared SHARED ${DIR_ASMJIT_LIB_SRCS} ${LIB_SOURCE}) # 指定动态库的输出名称 SET_TARGET_PROPERTIES(hook_shared PROPERTIES OUTPUT_NAME "hook") diff --git a/src/GameDataManager.h b/src/GameDataManager.h new file mode 100644 index 0000000..3a10a40 --- /dev/null +++ b/src/GameDataManager.h @@ -0,0 +1,61 @@ +#pragma once +#ifndef __GAMEDATAMANAGER_H__ +#define __GAMEDATAMANAGER_H__ + +struct AWARD_ITEM +{ + int item_id; + int item_num; +}; + +class CGameDataManager +{ +public: + SINGLETON_DEFINE_S(CGameDataManager); + + CGameDataManager() : + cera_award_begin_id(0), + cera_award_end_id(0) + { + + } + ~CGameDataManager() {}; +public: + void set_cera_award_begin_id(int id) + { + cera_award_begin_id = id; + } + void set_cera_award_end_id(int id) + { + cera_award_end_id = id; + } + int get_cera_award_begin_id() + { + return cera_award_begin_id; + } + int get_cera_award_end_id() + { + return cera_award_end_id; + } + bool add_cera_awarw_item(int count_, int item_id, int item_num) + { + cera_award_item.Push(count_, { item_id ,item_num }); + } + + Utils::TMap* get_cera_award_item_map() + { + return &cera_award_item; + } + +protected: + + +private: + int cera_award_begin_id; //商城奖励开始id + int cera_award_end_id; //商城奖励结束id + Utils::TMap cera_award_item; // 次数 + 数据 +}; + + +#endif // __GAMEDATAMANAGER_H__ + diff --git a/src/asmjit/CMakeLists.txt b/src/asmjit/CMakeLists.txt new file mode 100644 index 0000000..9d772e6 --- /dev/null +++ b/src/asmjit/CMakeLists.txt @@ -0,0 +1,29 @@ +cmake_minimum_required(VERSION 3.2) + + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32 -fpic") + +add_compile_definitions(ASMJIT_STATIC) + +#并将名称保存到 DIR_LIB_SRCS 变量 +file(GLOB_RECURSE DIR_ASMJIT_CPPLIB_SRCS + "core/*.c" + "core/*.cc" + "core/*.cpp" + "core/*.cxx" + "arm/*.c" + "arm/*.cc" + "arm/*.cpp" + "arm/*.cxx" + "x86/*.c" + "x86/*.cc" + "x86/*.cpp" + "x86/*.cxx" +) +#aux_source_directory(src DIR_LIB_SRCS) +# 生成库 +add_library (ASMJIT STATIC ${DIR_ASMJIT_CPPLIB_SRCS}) + +#target_compile_definitions(YAML_CPP PUBLIC YAML_CPP_COMPILED_LIB) +#target_include_directories(ASMJIT PUBLIC "include") diff --git a/src/asmjit/a64.h b/src/asmjit/a64.h new file mode 100644 index 0000000..ea4d304 --- /dev/null +++ b/src/asmjit/a64.h @@ -0,0 +1,62 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_A64_H_INCLUDED +#define ASMJIT_A64_H_INCLUDED + +//! \addtogroup asmjit_a64 +//! +//! ### Emitters +//! +//! - \ref a64::Assembler - AArch64 assembler (must read, provides examples). +//! - \ref a64::Builder - AArch64 builder. +//! - \ref a64::Compiler - AArch64 compiler. +//! - \ref a64::Emitter - AArch64 emitter (abstract). +//! +//! ### Supported Instructions +//! +//! - Emitters: +//! - \ref a64::EmitterExplicitT - Provides all instructions that use explicit +//! operands, provides also utility functions. The member functions provided +//! are part of all ARM/AArch64 emitters. +//! +//! - Instruction representation: +//! - \ref a64::Inst::Id - instruction identifiers. +//! +//! ### Register Operands +//! +//! - \ref arm::Reg - Base class for any AArch32/AArch64 register. +//! - \ref arm::Gp - General purpose register: +//! - \ref arm::GpW - 32-bit register. +//! - \ref arm::GpX - 64-bit register. +//! - \ref arm::Vec - Vector (SIMD) register: +//! - \ref arm::VecB - 8-bit SIMD register (AArch64 only). +//! - \ref arm::VecH - 16-bit SIMD register (AArch64 only). +//! - \ref arm::VecS - 32-bit SIMD register. +//! - \ref arm::VecD - 64-bit SIMD register. +//! - \ref arm::VecV - 128-bit SIMD register. +//! +//! ### Memory Operands +//! +//! - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features +//! including base, index, pre/post increment, and ARM-specific shift addressing and index extending. +//! +//! ### Other +//! +//! - \ref arm::Shift - Shift operation and value. +//! - \ref a64::Utils - Utilities that can help during code generation for AArch64. + +#include "./arm.h" +#include "./arm/a64assembler.h" +#include "./arm/a64builder.h" +#include "./arm/a64compiler.h" +#include "./arm/a64emitter.h" +#include "./arm/a64globals.h" +#include "./arm/a64instdb.h" +#include "./arm/a64operand.h" +#include "./arm/a64utils.h" + +#endif // ASMJIT_A64_H_INCLUDED + diff --git a/src/asmjit/arm.h b/src/asmjit/arm.h new file mode 100644 index 0000000..57ffa81 --- /dev/null +++ b/src/asmjit/arm.h @@ -0,0 +1,62 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_ARM_H_INCLUDED +#define ASMJIT_ARM_H_INCLUDED + +//! \addtogroup asmjit_arm +//! +//! ### Namespaces +//! +//! - \ref arm - arm namespace provides common functionality for both AArch32 and AArch64 backends. +//! - \ref a64 - a64 namespace provides support for AArch64 architecture. In addition it includes +//! \ref arm namespace, so you can only use a single namespace when targeting AArch64 architecture. +//! +//! ### Emitters +//! +//! - AArch64 +//! - \ref a64::Assembler - AArch64 assembler (must read, provides examples). +//! - \ref a64::Builder - AArch64 builder. +//! - \ref a64::Compiler - AArch64 compiler. +//! - \ref a64::Emitter - AArch64 emitter (abstract). +//! +//! ### Supported Instructions +//! +//! - AArch64: +//! - Emitters: +//! - \ref a64::EmitterExplicitT - Provides all instructions that use explicit operands, provides also +//! utility functions. The member functions provided are part of all AArch64 emitters. +//! - Instruction representation: +//! - \ref a64::Inst::Id - instruction identifiers. +//! +//! ### Register Operands +//! +//! - \ref arm::Reg - Base class for any AArch32/AArch64 register. +//! - \ref arm::Gp - General purpose register: +//! - \ref arm::GpW - 32-bit register. +//! - \ref arm::GpX - 64-bit register. +//! - \ref arm::Vec - Vector (SIMD) register: +//! - \ref arm::VecB - 8-bit SIMD register (AArch64 only). +//! - \ref arm::VecH - 16-bit SIMD register (AArch64 only). +//! - \ref arm::VecS - 32-bit SIMD register. +//! - \ref arm::VecD - 64-bit SIMD register. +//! - \ref arm::VecV - 128-bit SIMD register. +//! +//! ### Memory Operands +//! +//! - \ref arm::Mem - AArch32/AArch64 memory operand that provides support for all ARM addressing features +//! including base, index, pre/post increment, and ARM-specific shift addressing and index extending. +//! +//! ### Other +//! +//! - \ref arm::Shift - Shift operation and value (both AArch32 and AArch64). +//! - \ref arm::DataType - Data type that is part of an instruction in AArch32 mode. +//! - \ref a64::Utils - Utilities that can help during code generation for AArch64. + +#include "./core.h" +#include "./arm/armglobals.h" +#include "./arm/armoperand.h" + +#endif // ASMJIT_ARM_H_INCLUDED diff --git a/src/asmjit/asmjit-scope-begin.h b/src/asmjit/asmjit-scope-begin.h new file mode 100644 index 0000000..93397b5 --- /dev/null +++ b/src/asmjit/asmjit-scope-begin.h @@ -0,0 +1,17 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifdef _WIN32 + #pragma push_macro("min") + #pragma push_macro("max") + + #ifdef min + #undef min + #endif + + #ifdef max + #undef max + #endif +#endif diff --git a/src/asmjit/asmjit-scope-end.h b/src/asmjit/asmjit-scope-end.h new file mode 100644 index 0000000..702cef4 --- /dev/null +++ b/src/asmjit/asmjit-scope-end.h @@ -0,0 +1,9 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifdef _WIN32 + #pragma pop_macro("min") + #pragma pop_macro("max") +#endif diff --git a/src/asmjit/asmjit.h b/src/asmjit/asmjit.h new file mode 100644 index 0000000..1cd0651 --- /dev/null +++ b/src/asmjit/asmjit.h @@ -0,0 +1,33 @@ +// This file is part of AsmJit project +// +// SPDX-License-Identifier: Zlib +// Official GitHub Repository: https://github.com/asmjit/asmjit +// +// Copyright (c) 2008-2021 The AsmJit Authors +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// 3. This notice may not be removed or altered from any source distribution. + +#ifndef ASMJIT_ASMJIT_H_INCLUDED +#define ASMJIT_ASMJIT_H_INCLUDED + +#include "./core.h" + +#ifndef ASMJIT_NO_X86 + #include "./x86.h" +#endif + +#endif // ASMJIT_ASMJIT_H_INCLUDED diff --git a/src/asmjit/core.h b/src/asmjit/core.h new file mode 100644 index 0000000..5a07269 --- /dev/null +++ b/src/asmjit/core.h @@ -0,0 +1,1861 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_H_INCLUDED +#define ASMJIT_CORE_H_INCLUDED + +//! Root namespace used by AsmJit. +namespace asmjit { + +//! \mainpage API Reference +//! +//! AsmJit C++ API reference documentation generated by Doxygen. +//! +//! AsmJit library uses one global namespace called \ref asmjit, which provides the whole functionality. Core +//! functionality is within \ref asmjit namespace and architecture specific functionality is always in its own +//! namespace. For example \ref asmjit::x86 provides both 32-bit and 64-bit X86 code generation. +//! +//! \section main_groups Documentation Groups +//! +//! AsmJit documentation is structured into groups. Groups can be followed in order to learn AsmJit, but knowledge +//! from multiple groups is required to use AsmJit properly: +//! +//! $$DOCS_GROUP_OVERVIEW$$ +//! +//! \note It's important to understand that in order to learn AsmJit all groups are important. Some groups can be +//! omitted if a particular tool is out of interest - for example \ref asmjit_assembler users don't need to know +//! about \ref asmjit_builder, but it's not the opposite. \ref asmjit_builder users should know about \ref +//! asmjit_assembler as it also uses operands, labels, and other concepts. Similarly \ref asmjit_compiler users +//! should know how both \ref asmjit_assembler and \ref asmjit_builder tools work. +//! +//! \section where_to_start Where To Start +//! +//! AsmJit \ref asmjit_core provides the following two classes that are essential from the code generation perspective: +//! +//! - \ref CodeHolder provides functionality to temporarily hold the generated code. It stores all the necessary +//! information about the code - code buffers, sections, labels, symbols, and information about relocations. +//! +//! - \ref BaseEmitter provides interface used by emitter implementations. The interface provides basic building +//! blocks that are then implemented by \ref BaseAssembler, \ref BaseBuilder, and \ref BaseCompiler. +//! +//! Code emitters: +//! +//! - \ref asmjit_assembler - provides direct machine code generation. +//! +//! - \ref asmjit_builder - provides intermediate code generation that can be processed before it's serialized to +//! \ref BaseAssembler. +//! +//! - \ref asmjit_compiler - provides high-level code generation with built-in register allocation. +//! +//! - \ref FuncNode - provides insight into how function looks from the Compiler perspective and how it's stored in +//! a node-list. +//! +//! \section main_recommendations Recommendations +//! +//! The following steps are recommended for all AsmJit users: +//! +//! - Make sure that you use \ref Logger, see \ref asmjit_logging. +//! +//! - Make sure that you use \ref ErrorHandler, see \ref asmjit_error_handling. +//! +//! - Instruction validation in your debug builds can reveal problems too. AsmJit provides validation at instruction +//! level that can be enabled via \ref BaseEmitter::addDiagnosticOptions(). See \ref DiagnosticOptions for more +//! details. +//! +//! - If you are a Compiler user, use diagnostic options and read carefully if anything suspicious pops out. +//! Diagnostic options can be enabled via \ref BaseEmitter::addDiagnosticOptions(). If unsure which ones to use, +//! enable annotations and all debug options: `DiagnosticOptions::kRAAnnotate | DiagnosticOptions::kRADebugAll`. +//! +//! - Make sure you put a breakpoint into \ref DebugUtils::errored() function if you have a problem with AsmJit +//! returning errors during instruction encoding or register allocation. Having an active breakpoint there can +//! help to reveal the origin of the error, to inspect variables and other conditions that caused it. +//! +//! The reason for using \ref Logger and \ref ErrorHandler is that they provide a very useful information about what's +//! happening inside emitters. In many cases the information provided by these two is crucial to quickly identify and +//! fix issues that happen during development (for example wrong instruction, address, or register used). In addition, +//! output from \ref Logger is always necessary when filling bug reports. In other words, using logging and proper error +//! handling can save a lot of time during the development and can also save users from submitting issues. +//! +//! \section main_other Other Pages +//! +//! - Class List - List of classes sorted alphabetically +//! - AsmJit Namespace - List of symbols provided by `asmjit` namespace + + +//! \defgroup asmjit_build Build Instructions +//! \brief Build instructions, supported environments, and feature selection. +//! +//! ### Overview +//! +//! AsmJit is designed to be easy embeddable in any project. However, it depends on some compile-time definitions that +//! can be used to enable or disable features to decrease the resulting binary size. A typical way of building AsmJit +//! is to use [cmake](https://www.cmake.org), but it's also possible to just include AsmJit source code in your project +//! and to just build it. The easiest way to include AsmJit in your project is to just include **src** directory in +//! your project and to define \ref ASMJIT_STATIC. AsmJit can be just updated from time to time without any changes to +//! this integration process. Do not embed AsmJit's `test` files in such case as these are used exclusively for testing. +//! +//! ### Supported C++ Compilers +//! +//! - Requirements: +//! +//! - AsmJit won't build without C++11 enabled. If you use older GCC or Clang you would have to enable at least +//! C++11 standard through compiler flags. +//! +//! - Tested: +//! +//! - **Clang** - Tested by GitHub Actions - Clang 3.9+ (with C++11 enabled) is officially supported (older Clang +//! versions having C++11 support are probably fine, but are not regularly tested). +//! +//! - **GNU** - Tested by GitHub Actions - GCC 4.8+ (with C++11 enabled) is officially supported. +//! +//! - **MINGW** - Should work, but it's not tested in our CI environment. +//! +//! - **MSVC** - Tested by GitHub Actions - VS2017+ is officially supported, VS2015 is reported to work. +//! +//! - Untested: +//! +//! - **Intel** - No maintainers and no CI environment to regularly test this compiler. +//! +//! - **Other** C++ compilers would require basic support in +//! [core/api-config.h](https://github.com/asmjit/asmjit/tree/master/src/asmjit/core/api-config.h). +//! +//! ### Supported Operating Systems and Platforms +//! +//! - Tested: +//! +//! - **Linux** - Tested by GitHub Actions (any distribution is generally supported). +//! +//! - **Mac OS** - Tested by GitHub Actions (any version is supported). +//! +//! - **Windows** - Tested by GitHub Actions - (Windows 7+ is officially supported). +//! +//! - **Emscripten** - Works if compiled with \ref ASMJIT_NO_JIT. AsmJit cannot generate WASM code, but can be +//! used to generate X86/X64 code within a browser, for example. +//! +//! - Untested: +//! +//! - **BSDs** - No maintainers, no CI environment to regularly test BSDs, but they should work out of box. +//! +//! - **Haiku** - Not regularly tested, but reported to work. +//! +//! - **Other** operating systems would require some testing and support in the following files: +//! - [core/api-config.h](https://github.com/asmjit/asmjit/tree/master/src/asmjit/core/api-config.h) +//! - [core/osutils.cpp](https://github.com/asmjit/asmjit/tree/master/src/asmjit/core/osutils.cpp) +//! - [core/virtmem.cpp](https://github.com/asmjit/asmjit/tree/master/src/asmjit/core/virtmem.cpp) +//! +//! ### Supported Backends / Architectures +//! +//! - **X86** and **X86_64** - Both 32-bit and 64-bit backends tested on CI. +//! - **AArch64** - AArch64 backend is currently only partially tested (there is no native AArch64 runner to test +//! AsmJit Builder/Compiler) +//! +//! ### Static Builds and Embedding +//! +//! These definitions can be used to enable static library build. Embed is used when AsmJit's source code is embedded +//! directly in another project, implies static build as well. +//! +//! - \ref ASMJIT_EMBED - Asmjit is embedded, implies \ref ASMJIT_STATIC. +//! - \ref ASMJIT_STATIC - Enable static-library build. +//! +//! \note Projects that use AsmJit statically must define \ref ASMJIT_STATIC in all compilation units that use AsmJit, +//! otherwise AsmJit would use dynamic library imports in \ref ASMJIT_API decorator. The recommendation is to define +//! this macro across the whole project that uses AsmJit this way. +//! +//! ### Build Configuration +//! +//! These definitions control whether asserts are active or not. By default AsmJit would autodetect build configuration +//! from existing pre-processor definitions, but this behavior can be overridden, for example to enable debug asserts +//! in release configuration. +//! +//! - \ref ASMJIT_BUILD_DEBUG - Overrides build configuration to debug, asserts will be enabled in this case. +//! - \ref ASMJIT_BUILD_RELEASE - Overrides build configuration to release, asserts will be disabled in this case. +//! +//! \note There is usually no need to override the build configuration. AsmJit detects the build configuration by +//! checking whether `NDEBUG` is defined and automatically defines \ref ASMJIT_BUILD_RELEASE if configuration overrides +//! were not used. We only recommend using build configuration overrides in special situations, like using AsmJit in +//! release configuration with asserts enabled for whatever reason. +//! +//! ### AsmJit Backends +//! +//! AsmJit currently supports only X86/X64 backend, but the plan is to add more backends in the future. By default +//! AsmJit builds only the host backend, which is autodetected at compile-time, but this can be overridden. +//! +//! - \ref ASMJIT_NO_X86 - Disable X86/X64 backends. +//! - \ref ASMJIT_NO_FOREIGN - Disables the support for foreign architectures. +//! +//! ### Features Selection +//! +//! AsmJit builds by defaults all supported features, which includes all emitters, logging, instruction validation and +//! introspection, and JIT memory allocation. Features can be disabled at compile time by using `ASMJIT_NO_...` +//! definitions. +//! +//! - \ref ASMJIT_NO_DEPRECATED - Disables deprecated API at compile time so it won't be available and the +//! compilation will fail if there is attempt to use such API. This includes deprecated classes, namespaces, +//! enumerations, and functions. +//! +//! - \ref ASMJIT_NO_BUILDER - Disables \ref asmjit_builder functionality completely. This implies \ref +//! ASMJIT_NO_COMPILER as \ref asmjit_compiler cannot be used without \ref asmjit_builder. +//! +//! - \ref ASMJIT_NO_COMPILER - Disables \ref asmjit_compiler functionality completely. +//! +//! - \ref ASMJIT_NO_JIT - Disables JIT memory management and \ref JitRuntime. +//! +//! - \ref ASMJIT_NO_LOGGING - Disables \ref Logger and \ref Formatter. +//! +//! - \ref ASMJIT_NO_TEXT - Disables everything that contains string representation of AsmJit constants, should +//! be used together with \ref ASMJIT_NO_LOGGING as logging doesn't make sense without the ability to query +//! instruction names, register names, etc... +//! +//! - \ref ASMJIT_NO_VALIDATION - Disables validation API. +//! +//! - \ref ASMJIT_NO_INTROSPECTION - Disables instruction introspection API, must be used together with \ref +//! ASMJIT_NO_COMPILER as \ref asmjit_compiler requires introspection for its liveness analysis and register +//! allocation. +//! +//! \note It's not recommended to disable features if you plan to build AsmJit as a shared library that will be +//! used by multiple projects that you don't control how AsmJit was built (for example AsmJit in a Linux distribution). +//! The possibility to disable certain features exists mainly for customized AsmJit builds. + + +//! \defgroup asmjit_breaking_changes Breaking Changes +//! \brief Documentation of breaking changes +//! +//! ### Overview +//! +//! AsmJit is a live project that is being actively developed. Deprecating the existing API in favor of a new +//! one is preferred, but it's not always possible if the changes are significant. AsmJit authors prefer to do +//! accumulated breaking changes at once instead of breaking the API often. This page documents deprecated and +//! removed APIs and should serve as a how-to guide for people that want to port existing code to work with the +//! newest AsmJit. +//! +//! ### Tips +//! +//! Useful tips before you start: +//! +//! - Visit our [Public Gitter Channel](https://gitter.im/asmjit/asmjit) if you need a quick help. +//! +//! - Build AsmJit with `ASMJIT_NO_DEPRECATED` macro defined to make sure that you are not using deprecated +//! functionality at all. Deprecated functions are decorated with `ASMJIT_DEPRECATED()` macro, but sometimes +//! it's not possible to decorate everything like classes, which are used by deprecated functions as well, +//! because some compilers would warn about that. If your project compiles fine with `ASMJIT_NO_DEPRECATED` +//! it's not using anything, which was deprecated. +//! +//! ### Changes committed at 2021-12-13 +//! +//! Core changes: +//! +//! - Removed old deprecated API. +//! +//! - Many enumerations were changed to enum class, and many public APIs were changed to use such enums instead +//! of uint32_t. This change makes some APIs backward incompatible - there are no deprecations this time. +//! +//! - Extracted operand signature manipulation to `OperandSignature`. +//! - Setting function arguments through `Compiler::setArg()` was deprecated, use FuncNode::setArg() instead. +//! - Moved `{arch}::Features::k` to `CpuFeatures::{arch}::k`. +//! - Moved `BaseEmitter::kEncodingOption` to `EncodingOptions::k`. +//! - Moved `BaseEmitter::kFlag` to `EmitterFlags::k`. +//! - Moved `BaseEmitter::kType` to `EmitterType::k`. +//! - Moved `BaseEmitter::kValidationOption` to `DiagnosticOptions::kValidate`. +//! - Moved `BaseFeatures` to `CpuFeatures`. +//! - Moved `BaseInst::kControl` to `InstControlFlow::k`. +//! - Moved `BaseInst::kOption` and `x86::Inst::kOption` to `InstOptions::k`. +//! - Moved `BaseNode::kNode` to `NodeType::k`. +//! - Moved `BaseReg::kGroup` and `x86::Reg::kGroup` to `RegGroup::k`. +//! - Moved `BaseReg::kType` and `x86::Reg::kType` to `RegType::k`. +//! - Moved `CallConv::kFlag` to `CallConvFlags::k`. +//! - Moved `CallConv::kId` to `CallConvId::k`. +//! - Moved `CallConv::kStrategy` to `CallConvStrategy::k`. +//! - Moved `CodeBuffer::kFlag` to `CodeBufferFlags`. +//! - Moved `ConstPool::kScope` to `ConstPoolScope::k`. +//! - Moved `Environment::kArch` to `Arch::k`. +//! - Moved `Environment::kSubArch` to `SubArch::k`. +//! - Moved `Environment::kFormat` to `OjectFormat::k`. +//! - Moved `Environment::kPlatform` to `Platform::k`. +//! - Moved `Environment::kAbi` to `PlatformABI::k`. +//! - Moved `Environment::kVendor` to `Vendor::k`. +//! - Moved `FormatOptions::kFlag` to `FormatFlags::k` and `DiagnosticOptions::k` (Compiler diagnostics flags). +//! - Moved `FormatOptions::kIndentation` to `FormatIndentationGroup::k`. +//! - Moved `FuncFrame::kAttr` to `FuncAttributes::k`. +//! - Moved `Globals::kReset` to `ResetPolicy::k`. +//! - Moved `InstDB::kAvx512Flag` to `InstDB::Avx512Flags::k`. +//! - Moved `InstDB::kFlag` to `InstDB::InstFlags::k`. +//! - Moved `InstDB::kMemFlag` to `InstDB::OpFlags::kMem`. +//! - Moved `InstDB::kMode` to `InstDB::Mode::k`. +//! - Moved `InstDB::kOpFlag` to `InstDB::OpFlags::k{OpType}...`. +//! - Moved `JitAllocator::kOption` to `JitAllocatorOptions::k`. +//! - Moved `Label::kType` to `LabelType::k`. +//! - Moved `Operand::kOpType` to `OperandType::k`. +//! - Moved `OpRWInfo::kFlag` to `OpRWFlags::k`. +//! - Moved `Type::kId` to `TypeId::k`. +//! - Moved `VirtMem::k` to `VirtMem::MemoryFlags::k`. +//! +//! ### Changes committed at 2020-05-30 +//! +//! AsmJit has been cleaned up significantly, many todo items have been fixed and many functions and classes have +//! been redesigned, some in an incompatible way. +//! +//! Core changes: +//! +//! - `Imm` operand has now only `Imm::value()` and `Imm::valueAs()` functions that return its value content, +//! and `Imm::setValue()` function that sets the content. Functions like `setI8()`, `setU8()` were deprecated. +//! +//! Old functions were deprecated, but code using them should still compile. +//! +//! - `ArchInfo` has been replaced with `Environment`. Environment provides more details about the architecture, +//! but drops some properties that were used by arch info - `gpSize(`) and `gpCount()`. `gpSize()` can be replaced +//! with `registerSize()` getter, which returns a native register size of the architecture the environment uses. +//! However, `gpCount()` was removed - at the moment `ArchTraits` can be used to access such properties. +//! +//! Some other functions were renamed, like `ArchInfo::isX86Family()` is now `Environment::isFamilyX86()`, etc. +//! The reason for changing the order was support for more propertries and all the accessors now start with the +//! type of the property, like `Environment::isPlatformWindows()`. +//! +//! This function causes many other classes to provide `environment()` getter instead of `archInfo()` getter. +//! In addition, AsmJit now uses `arch()` to get an architecture instead of `archId()`. `ArchInfo::kIdXXX` was +//! renamed to `Environment::kArchXXX`. +//! +//! Some functions were deprecated, some removed... +//! +//! - `CodeInfo` has been removed in favor of `Environment`. If you used `CodeInfo` to set architecture and base +//! address, this is now possible with `Environment` and setting base address explicitly by `CodeHolder::init()` +//! - the first argument is `Environment`, and the second argument is base address, which defaults to +//! `Globals::kNoBaseAddress`. +//! +//! CodeInfo class was deprecated, but the code using it should still compile with warnings. +//! +//! - `CallConv` has been updated to offer a more unified way of representing calling conventions - many calling +//! conventions were abstracted to follow standard naming like `CallConvId::kCDecl` or `CallConvId::kStdCall`. +//! +//! This change means that other APIs like `FuncDetail::init()` now require both, calling convention and target +//! `Environment`. +//! +//! - `Logging` namespace has been renamed to `Formatter`, which now provides general functionality for formatting +//! in AsmJit. +//! +//! Logging namespace should still work, but its use is deprecated. Unfortunately this will be without deprecation +//! warnings, so make sure you don't use it. +//! +//! - `Data64`, `Data128`, and `Data256` structs were deprecated and should no longer be used. There is no replacement, +//! AsmJit users should simply create their own structures if they need them or use the new repeated embed API in +//! emitters, see `BaseEmitter::embedDataArray()`. +//! +//! Emitter changes: +//! +//! - `BaseEmitter::emit()` function signature has been changed to accept 3 operands by reference and the rest 3 +//! operands as a continuous array. This change is purely cosmetic and shouldn't affect users as emit() has many +//! overloads that dispatch to the right function. +//! +//! - `x86::Emitter` (Assembler, Builder, Compiler) deprecates embed utilities like `dint8()`, `duint8()`, `duint16()`, +//! `dxmm()`, etc... in favor of a new and more powerful `BaseEmitter::embedDataArray()`. This function also allows +//! emitting repeated values and/or patterns, which is used by helpers `BaseEmitter::embedUInt8()`, and others... +//! +//! - Validation is now available through `BaseEmitter::DiagnosticOptions`, which can be enabled/disabled through +//! `BaseEmitter::addDiagnosticOptions()` and `BaseEmitter::clearDiagnosticOptions()`, respectively. Validation +//! options now separate between encoding and Builder/Compiler so it's possible to choose the granularity required. +//! +//! Builder changes: +//! +//! - Internal functions for creating nodes were redesigned. They now accept a pointer to the node created as +//! a first parameter. These changes should not affect AsmJit users as these functions were used internally. +//! +//! Compiler changes: +//! +//! - `FuncCallNode` has been renamed to `InvokeNode`. Additionally, function calls should now use +//! `x86::Compiler::invoke()` instead of `call()`. The reason behind this is to remove the confusion between a +//! `call` instruction and AsmJit's `call()` intrinsic, which is now `invoke()`. +//! +//! - Creating new nodes also changed. Now the preferred way of invoking a function is to call +//! `x86::Compiler::invoke()` where the first argument is `InvokeNode**`. The function now returns an error and +//! would call `ErrorHandler` in case of a failure. Error handling was unspecified in the past - the function was +//! marked noexcept, but called error handler, which could throw. +//! +//! The reason behind this change is to make the API consistent with other changes and to also make it possible +//! to inspect the possible error. In the previous API it returned a new node or `nullptr` in case of error, +//! which the user couldn't inspect unless there was an attached `ErrorHandler`. +//! +//! Samples: +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! // The basic setup of JitRuntime and CodeHolder changed, use environment() +//! // instead of codeInfo(). +//! void basicSetup() { +//! JitRuntime rt; +//! CodeHolder code(rt.environment()); +//! } +//! +//! // Calling a function (Compiler) changed - use invoke() instead of call(). +//! void functionInvocation(x86::Compiler& cc) { +//! InvokeNode* invokeNode; +//! cc.invoke(&invokeNode, targetOperand, FuncSignatureT<...>(...)); +//! } +//! ``` + + +//! \defgroup asmjit_core Core +//! \brief Globals, code storage, and emitter interface. +//! +//! ### Overview +//! +//! AsmJit library uses \ref CodeHolder to hold code during code generation and emitters inheriting from \ref +//! BaseEmitter to emit code. CodeHolder uses containers to manage its data: +//! +//! - \ref Section - stores information about a code or data section. +//! - \ref CodeBuffer - stores actual code or data, part of \ref Section. +//! - \ref LabelEntry - stores information about a label - its name, offset, section where it belongs to, and +//! other bits. +//! - \ref LabelLink - stores information about yet unbound label, which was already used by the assembler. +//! - \ref RelocEntry - stores information about a relocation. +//! - \ref AddressTableEntry - stores information about an address, which was used in a jump or call. Such +//! address may need relocation. +//! +//! To generate code you would need to instantiate at least the following classes: +//! +//! - \ref CodeHolder - to hold code during code generation. +//! - \ref BaseEmitter - to emit code into \ref CodeHolder. +//! - \ref Target (optional) - most likely \ref JitRuntime to keep the generated code in executable memory. \ref +//! Target can be customized by inheriting from it. +//! +//! There are also other core classes that are important: +//! +//! - \ref Environment - describes where the code will run. Environment brings the concept of target triples or +//! tuples into AsmJit, which means that users can specify target architecture, platform, and ABI. +//! - \ref TypeId - encapsulates lightweight type functionality that can be used to describe primitive and vector +//! types. Types are used by higher level utilities, for example by \ref asmjit_function and \ref asmjit_compiler. +//! - \ref CpuInfo - encapsulates CPU information - stores both CPU information and CPU features described by \ref +//! CpuFeatures. +//! +//! AsmJit also provides global constants: +//! +//! - \ref Globals - namespace that provides global constants. +//! - \ref ByteOrder - byte-order constants and functionality. +//! +//! \note CodeHolder examples use \ref x86::Assembler as abstract interfaces cannot be used to generate code. +//! +//! ### CodeHolder & Emitters +//! +//! The example below shows how the mentioned classes interact to generate X86 code: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! // Signature of the generated function. +//! typedef int (*Func)(void); +//! +//! int main() { +//! JitRuntime rt; // Runtime specialized for JIT code execution. +//! +//! CodeHolder code; // Holds code and relocation information. +//! code.init(rt.environment()); // Initialize code to match the JIT environment. +//! +//! x86::Assembler a(&code); // Create and attach x86::Assembler to code. +//! a.mov(x86::eax, 1); // Move one to eax register. +//! a.ret(); // Return from function. +//! // ===== x86::Assembler is no longer needed from here and can be destroyed ===== +//! +//! Func fn; // Holds address to the generated function. +//! Error err = rt.add(&fn, &code); // Add the generated code to the runtime. +//! if (err) return 1; // Handle a possible error returned by AsmJit. +//! // ===== CodeHolder is no longer needed from here and can be destroyed ===== +//! +//! int result = fn(); // Execute the generated code. +//! printf("%d\n", result); // Print the resulting "1". +//! +//! // All classes use RAII, all resources will be released before `main()` returns, +//! // the generated function can be, however, released explicitly if you intend to +//! // reuse or keep the runtime alive, which you should in a production-ready code. +//! rt.release(fn); +//! +//! return 0; +//! } +//! ``` +//! +//! The example above used \ref x86::Assembler as an emitter. AsmJit provides the following emitters that offer various +//! levels of abstraction: +//! +//! - \ref asmjit_assembler - Low-level emitter that emits directly to \ref CodeBuffer. +//! - \ref asmjit_builder - Low-level emitter that emits to a \ref BaseNode list. +//! - \ref asmjit_compiler - High-level emitter that provides register allocation. +//! +//! ### Targets and JitRuntime +//! +//! AsmJit's \ref Target is an interface that provides basic target abstraction. At the moment AsmJit provides only +//! one implementation called \ref JitRuntime, which as the name suggests provides JIT code target and execution +//! runtime. \ref JitRuntime provides all the necessary stuff to implement a simple JIT compiler with basic memory +//! management. It only provides \ref JitRuntime::add() and \ref JitRuntime::release() functions that are used to +//! either add code to the runtime or release it. \ref JitRuntime doesn't do any decisions on when the code should be +//! released, the decision is up to the developer. +//! +//! See more at \ref asmjit_virtual_memory group. +//! +//! ### More About Environment +//! +//! In the previous example the \ref Environment is retrieved from \ref JitRuntime. It's logical as \ref JitRuntime +//! always returns an \ref Environment that is compatible with the host. For example if your application runs on X86_64 +//! CPU the \ref Environment returned will use \ref Arch::kX64 architecture in contrast to \ref Arch::kX86, which will +//! be used in 32-bit mode on an X86 target. +//! +//! AsmJit allows to setup the \ref Environment manually and to select a different architecture and ABI when necessary. +//! So let's do something else this time, let's always generate a 32-bit code and print its binary representation. To +//! do that, we can create our own \ref Environment and initialize it to \ref Arch::kX86. +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! int main(int argc, char* argv[]) { +//! using namespace asmjit::x86; +//! +//! // Create a custom environment initialized to 32-bit X86 architecture. +//! Environment env; +//! env.setArch(Arch::kX86); +//! +//! CodeHolder code; // Create a CodeHolder. +//! code.init(env); // Initialize CodeHolder with custom environment. +//! +//! // Generate a 32-bit function that sums 4 floats and looks like: +//! // void func(float* dst, const float* a, const float* b) +//! x86::Assembler a(&code); // Create and attach x86::Assembler to `code`. +//! +//! a.mov(eax, dword_ptr(esp, 4)); // Load the destination pointer. +//! a.mov(ecx, dword_ptr(esp, 8)); // Load the first source pointer. +//! a.mov(edx, dword_ptr(esp, 12)); // Load the second source pointer. +//! +//! a.movups(xmm0, ptr(ecx)); // Load 4 floats from [ecx] to XMM0. +//! a.movups(xmm1, ptr(edx)); // Load 4 floats from [edx] to XMM1. +//! a.addps(xmm0, xmm1); // Add 4 floats in XMM1 to XMM0. +//! a.movups(ptr(eax), xmm0); // Store the result to [eax]. +//! a.ret(); // Return from function. +//! +//! // We have no Runtime this time, it's on us what we do with the code. +//! // CodeHolder stores code in Section, which provides some basic properties +//! // and CodeBuffer structure. We are interested in section's CodeBuffer. +//! // +//! // NOTE: The first section is always '.text', it can be retrieved by +//! // code.sectionById(0) or simply by code.textSection(). +//! CodeBuffer& buffer = code.textSection()->buffer(); +//! +//! // Print the machine-code generated or do something else with it... +//! // 8B4424048B4C24048B5424040F28010F58010F2900C3 +//! for (size_t i = 0; i < buffer.length; i++) +//! printf("%02X", buffer.data[i]); +//! +//! return 0; +//! } +//! ``` +//! +//! ### Explicit Code Relocation +//! +//! In addition to \ref Environment, \ref CodeHolder can be configured to specify a base-address (or a virtual base +//! address in a linker terminology), which could be static (useful when you know the location where the target's +//! machine code will be) or dynamic. AsmJit assumes dynamic base-address by default and relocates the code held by +//! \ref CodeHolder to a user provided address on-demand. To be able to relocate to a user provided address it needs +//! to store some information about relocations, which is represented by \ref RelocEntry. Relocation entries are only +//! required if you call external functions from the generated code that cannot be encoded by using a 32-bit +//! displacement (64-bit displacements are not provided by aby supported architecture). +//! +//! There is also a concept called \ref LabelLink - label link is a lightweight data structure that doesn't have any +//! identifier and is stored in \ref LabelEntry as a single-linked list. Label link represents either unbound yet used +//! label and cross-sections links (only relevant to code that uses multiple sections). Since crossing sections is +//! something that cannot be resolved immediately these links persist until offsets of these sections are assigned and +//! until \ref CodeHolder::resolveUnresolvedLinks() is called. It's an error if you end up with code that has +//! unresolved label links after flattening. You can verify it by calling \ref CodeHolder::hasUnresolvedLinks(), which +//! inspects the value returned by \ref CodeHolder::unresolvedLinkCount(). +//! +//! AsmJit can flatten code that uses multiple sections by assigning each section an incrementing offset that respects +//! its alignment. Use \ref CodeHolder::flatten() to do that. After the sections are flattened their offsets and +//! virtual sizes are adjusted to respect each section's buffer size and alignment. The \ref +//! CodeHolder::resolveUnresolvedLinks() function must be called before relocating the code held by \ref CodeHolder. +//! You can also flatten your code manually by iterating over all sections and calculating their offsets (relative to +//! base) by your own algorithm. In that case \ref CodeHolder::flatten() should not be called, however, +//! \ref CodeHolder::resolveUnresolvedLinks() should be. +//! +//! The example below shows how to use a built-in virtual memory allocator \ref JitAllocator instead of using \ref +//! JitRuntime (just in case you want to use your own memory management) and how to relocate the generated code +//! into your own memory block - you can use your own virtual memory allocator if you prefer that, but that's OS +//! specific and not covered by the documentation. +//! +//! The following code is similar to the previous one, but implements a function working in both 32-bit and 64-bit +//! environments: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! typedef void (*SumIntsFunc)(int* dst, const int* a, const int* b); +//! +//! int main() { +//! // Create a custom environment that matches the current host environment. +//! Environment env = Environment::host(); +//! +//! CodeHolder code; // Create a CodeHolder. +//! code.init(env); // Initialize CodeHolder with environment. +//! +//! x86::Assembler a(&code); // Create and attach x86::Assembler to `code`. +//! +//! // Signature: 'void func(int* dst, const int* a, const int* b)'. +//! x86::Gp dst; +//! x86::Gp src_a; +//! x86::Gp src_b; +//! +//! // Handle the difference between 32-bit and 64-bit calling conventions +//! // (arguments passed through stack vs. arguments passed by registers). +//! if (env.is32Bit()) { +//! dst = x86::eax; +//! src_a = x86::ecx; +//! src_b = x86::edx; +//! a.mov(dst , x86::dword_ptr(x86::esp, 4)); +//! a.mov(src_a, x86::dword_ptr(x86::esp, 8)); +//! a.mov(src_b, x86::dword_ptr(x86::esp, 12)); +//! } +//! else { +//! if (env.isPlatformWindows()) { +//! dst = x86::rcx; // First argument (destination pointer). +//! src_a = x86::rdx; // Second argument (source 'a' pointer). +//! src_b = x86::r8; // Third argument (source 'b' pointer). +//! } +//! else { +//! dst = x86::rdi; // First argument (destination pointer). +//! src_a = x86::rsi; // Second argument (source 'a' pointer). +//! src_b = x86::rdx; // Third argument (source 'b' pointer). +//! } +//! } +//! +//! a.movdqu(x86::xmm0, x86::ptr(src_a)); // Load 4 ints from [src_a] to XMM0. +//! a.movdqu(x86::xmm1, x86::ptr(src_b)); // Load 4 ints from [src_b] to XMM1. +//! a.paddd(x86::xmm0, x86::xmm1); // Add 4 ints in XMM1 to XMM0. +//! a.movdqu(x86::ptr(dst), x86::xmm0); // Store the result to [dst]. +//! a.ret(); // Return from function. +//! +//! // Even when we didn't use multiple sections AsmJit could insert one section +//! // called '.addrtab' (address table section), which would be filled by data +//! // required by relocations (absolute jumps and calls). You can omit this code +//! // if you are 100% sure your code doesn't contain multiple sections and +//! // such relocations. You can use `CodeHolder::hasAddressTable()` to verify +//! // whether the address table section does exist. +//! code.flatten(); +//! code.resolveUnresolvedLinks(); +//! +//! // After the code was generated it can be relocated manually to any memory +//! // location, however, we need to know it's size before we perform memory +//! // allocation. `CodeHolder::codeSize()` returns the worst estimated code +//! // size in case that relocations are not possible without trampolines (in +//! // that case some extra code at the end of the current code buffer is +//! // generated during relocation). +//! size_t estimatedSize = code.codeSize(); +//! +//! // Instead of rolling up our own memory allocator we can use the one AsmJit +//! // provides. It's decoupled so you don't need to use `JitRuntime` for that. +//! JitAllocator allocator; +//! +//! // Allocate an executable virtual memory and handle a possible failure. +//! void* p = allocator.alloc(estimatedSize); +//! if (!p) +//! return 0; +//! +//! // Now relocate the code to the address provided by the memory allocator. +//! // Please note that this DOESN'T COPY anything to `p`. This function will +//! // store the address in CodeHolder and use relocation entries to patch the +//! // existing code in all sections to respect the base address provided. +//! code.relocateToBase((uint64_t)p); +//! +//! // This is purely optional. There are cases in which the relocation can omit +//! // unneeded data, which would shrink the size of address table. If that +//! // happened the codeSize returned after relocateToBase() would be smaller +//! // than the originally `estimatedSize`. +//! size_t codeSize = code.codeSize(); +//! +//! // This will copy code from all sections to `p`. Iterating over all sections +//! // and calling `memcpy()` would work as well, however, this function supports +//! // additional options that can be used to also zero pad sections' virtual +//! // size, etc. +//! // +//! // With some additional features, copyFlattenData() does roughly this: +//! // for (Section* section : code.sections()) +//! // memcpy((uint8_t*)p + section->offset(), +//! // section->data(), +//! // section->bufferSize()); +//! code.copyFlattenedData(p, codeSize, CopySectionFlags::kPadSectionBuffer); +//! +//! // Execute the generated function. +//! int inA[4] = { 4, 3, 2, 1 }; +//! int inB[4] = { 1, 5, 2, 8 }; +//! int out[4]; +//! +//! // This code uses AsmJit's ptr_as_func<> to cast between void* and SumIntsFunc. +//! ptr_as_func(p)(out, inA, inB); +//! +//! // Prints {5 8 4 9} +//! printf("{%d %d %d %d}\n", out[0], out[1], out[2], out[3]); +//! +//! // Release 'p' is it's no longer needed. It will be destroyed with 'vm' +//! // instance anyway, but it's a good practice to release it explicitly +//! // when you know that the function will not be needed anymore. +//! allocator.release(p); +//! +//! return 0; +//! } +//! ``` +//! +//! If you know the base-address in advance (before the code generation) it can be passed as a second argument to +//! \ref CodeHolder::init(). In that case the Assembler will know the absolute position of each instruction and +//! would be able to use it during instruction encoding to prevent relocations where possible. The following example +//! shows how to configure the base address: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void initializeCodeHolder(CodeHolder& code) { +//! Environment env = Environment::host(); +//! uint64_t baseAddress = uint64_t(0x1234); +//! +//! // initialize CodeHolder with environment and custom base address. +//! code.init(env, baseAddress); +//! } +//! ``` +//! +//! ### Label Offsets and Links +//! +//! When a label that is not yet bound is used by the Assembler, it creates a \ref LabelLink, which is then added to +//! a \ref LabelEntry. These links are also created if a label is used in a different section than in which it was +//! bound. Let's examine some functions that can be used to check whether there are any unresolved links. +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void labelLinksExample(CodeHolder& code, const Label& label) { +//! // Tests whether the `label` is bound. +//! bool isBound = code.isLabelBound(label); +//! printf("Label %u is %s\n", label.id(), isBound ? "bound" : "not bound"); +//! +//! // Returns true if the code contains either referenced, but unbound +//! // labels, or cross-section label links that are not resolved yet. +//! bool hasUnresolved = code.hasUnresolvedLinks(); // Boolean answer. +//! size_t nUnresolved = code.unresolvedLinkCount(); // Count of unresolved links. +//! +//! printf("Number of unresolved links: %zu\n", nUnresolved); +//! } +//! ``` +//! +//! There is no function that would return the number of unbound labels as this is completely unimportant from +//! CodeHolder's perspective. If a label is not used then it doesn't matter whether it's bound or not, only actually +//! used labels matter. After a Label is bound it's possible to query its offset offset relative to the start of the +//! section where it was bound: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void labelOffsetExample(CodeHolder& code, const Label& label) { +//! // Label offset is known after it's bound. The offset provided is relative +//! // to the start of the section, see below for alternative. If the given +//! // label is not bound the offset returned will be zero. It's recommended +//! // to always check whether the label is bound before using its offset. +//! uint64_t sectionOffset = code.labelOffset(label); +//! printf("Label offset relative to section: %llu\n", (unsigned long long)sectionOffset); +//! +//! // If you use multiple sections and want the offset relative to the base. +//! // NOTE: This function expects that the section has already an offset and +//! // the label-link was resolved (if this is not true you will still get an +//! // offset relative to the start of the section). +//! uint64_t baseOffset = code.labelOffsetFromBase(label); +//! printf("Label offset relative to base: %llu\n", (unsigned long long)baseOffset); +//! } +//! ``` +//! +//! ### Sections +//! +//! AsmJit allows to create multiple sections within the same \ref CodeHolder. A test-case +//! [asmjit_test_x86_sections.cpp](https://github.com/asmjit/asmjit/blob/master/test/asmjit_test_x86_sections.cpp) +//! can be used as a reference point although the following example should also provide a useful insight: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void sectionsExample(CodeHolder& code) { +//! // Text section is always provided as the first section. +//! Section* text = code.textSection(); // or code.sectionById(0); +//! +//! // To create another section use CodeHolder::newSection(). +//! Section* data; +//! Error err = code.newSection(&data, +//! ".data", // Section name +//! SIZE_MAX, // Name length if the name is not null terminated (or SIZE_MAX). +//! SectionFlags::kNone, // Section flags, see SectionFlags. +//! 8, // Section alignment, must be power of 2. +//! 0); // Section order value (optional, default 0). +//! +//! // When you switch sections in Assembler, Builder, or Compiler the cursor +//! // will always move to the end of that section. When you create an Assembler +//! // the cursor would be placed at the end of the first (.text) section, which +//! // is initially empty. +//! x86::Assembler a(&code); +//! Label L_Data = a.newLabel(); +//! +//! a.mov(x86::eax, x86::ebx); // Emits in .text section. +//! +//! a.section(data); // Switches to the end of .data section. +//! a.bind(L_Data); // Binds label in this .data section +//! a.db(0x01); // Emits byte in .data section. +//! +//! a.section(text); // Switches to the end of .text section. +//! a.add(x86::ebx, x86::eax); // Emits in .text section. +//! +//! // References a label in .text section, which was bound in .data section. +//! // This would create a LabelLink even when the L_Data is already bound, +//! // because the reference crosses sections. See below... +//! a.lea(x86::rsi, x86::ptr(L_Data)); +//! } +//! ``` +//! +//! The last line in the example above shows that a LabelLink would be created even for bound labels that cross +//! sections. In this case a referenced label was bound in another section, which means that the link couldn't be +//! resolved at that moment. If your code uses sections, but you wish AsmJit to flatten these sections (you don't +//! plan to flatten them manually) then there is an API for that. +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! // ... (continuing the previous example) ... +//! void sectionsExampleContinued(CodeHolder& code) { +//! // Suppose we have some code that contains multiple sections and +//! // we would like to flatten it by using AsmJit's built-in API: +//! Error err = code.flatten(); +//! if (err) { +//! // There are many reasons it can fail, so always handle a possible error. +//! printf("Failed to flatten the code: %s\n", DebugUtils::errorAsString(err)); +//! exit(1); +//! } +//! +//! // After flattening all sections would contain assigned offsets +//! // relative to base. Offsets are 64-bit unsigned integers so we +//! // cast them to `size_t` for simplicity. On 32-bit targets it's +//! // guaranteed that the offset cannot be greater than `2^32 - 1`. +//! printf("Data section offset %zu", size_t(data->offset())); +//! +//! // The flattening doesn't resolve unresolved label links, this +//! // has to be done manually as flattening can be done separately. +//! err = code.resolveUnresolvedLinks(); +//! if (err) { +//! // This is the kind of error that should always be handled... +//! printf("Failed to resolve label links: %s\n", DebugUtils::errorAsString(err)); +//! exit(1); +//! } +//! +//! if (code.hasUnresolvedLinks()) { +//! // This would mean either unbound label or some other issue. +//! printf("The code has %zu unbound labels\n", code.unresovedLinkCount()); +//! exit(1); +//! } +//! } +//! ``` + + +//! \defgroup asmjit_assembler Assembler +//! \brief Assembler interface and operands. +//! +//! ### Overview +//! +//! AsmJit's Assembler is used to emit machine code directly into a \ref CodeBuffer. In general, code generation +//! with assembler requires the knowledge of the following: +//! +//! - \ref BaseAssembler and architecture-specific assemblers: +//! - \ref x86::Assembler - Assembler specific to X86 architecture +//! - \ref Operand and its variations: +//! - \ref BaseReg - Base class for a register operand, inherited by: +//! - \ref x86::Reg - Register operand specific to X86 architecture. +//! - \ref BaseMem - Base class for a memory operand, inherited by: +//! - \ref x86::Mem - Memory operand specific to X86 architecture. +//! - \ref Imm - Immediate (value) operand. +//! - \ref Label - Label operand. +//! +//! \note Assembler examples use \ref x86::Assembler as abstract interfaces cannot be used to generate code. +//! +//! ### Operand Basics +//! +//! Let's start with operands. \ref Operand is a data structure that defines a data layout of any operand. It can be +//! inherited, but any class inheriting it cannot add any members to it, only the existing layout can be reused. +//! AsmJit allows to construct operands dynamically, to store them, and to query a complete information about them +//! at run-time. Operands are small (always 16 bytes per \ref Operand) and can be copied and passed by value. Please +//! never allocate individual operands dynamically by using a `new` keyword - it would work, but then you would have +//! to be responsible for deleting such operands. In AsmJit operands are always part of some other data structures +//! like \ref InstNode, which is part of \ref asmjit_builder tool. +//! +//! Operands contain only identifiers, but not pointers to any code-generation data. For example \ref Label operand +//! only provides label identifier, but not a pointer to \ref LabelEntry structure. In AsmJit such IDs are used to +//! link stuff together without having to deal with pointers. +//! +//! AsmJit's operands all inherit from a base class called \ref Operand. Operands have the following properties that +//! are commonly accessible by getters and setters: +//! +//! - \ref Operand - Base operand, which only provides accessors that are common to all operand types. +//! - \ref BaseReg - Describes either physical or virtual register. Physical registers have id that matches the +//! target's machine id directly whereas virtual registers must be allocated into physical registers by a register +//! allocator pass. Register operand provides: +//! - Register Type (\ref RegType) - Unique id that describes each possible register provided by the target +//! architecture - for example X86 backend provides general purpose registers (GPB-LO, GPB-HI, GPW, GPD, and GPQ) +//! and all types of other registers like K, MM, BND, XMM, YMM, ZMM, and TMM. +//! - Register Group (\ref RegGroup) - Groups multiple register types under a single group - for example all +//! general-purpose registers (of all sizes) on X86 are part of \ref RegGroup::kGp and all SIMD registers +//! (XMM, YMM, ZMM) are part of \ref RegGroup::kVec. +//! - Register Size - Contains the size of the register in bytes. If the size depends on the mode (32-bit vs +//! 64-bit) then generally the higher size is used (for example RIP register has size 8 by default). +//! - Register Id - Contains physical or virtual id of the register. +//! - \ref BaseMem - Used to reference a memory location. Memory operand provides: +//! - Base Register - A base register type and id (physical or virtual). +//! - Index Register - An index register type and id (physical or virtual). +//! - Offset - Displacement or absolute address to be referenced (32-bit if base register is used and 64-bit if +//! base register is not used). +//! - Flags that can describe various architecture dependent information (like scale and segment-override on X86). +//! - \ref Imm - Immediate values are usually part of instructions (encoded within the instruction itself) or data. +//! - \ref Label - used to reference a location in code or data. Labels must be created by the \ref BaseEmitter or +//! by \ref CodeHolder. Each label has its unique id per \ref CodeHolder instance. +//! +//! ### Operand Manipulation +//! +//! AsmJit allows to construct operands dynamically, to store them, and to query a complete information about them at +//! run-time. Operands are small (always 16 bytes per `Operand`) and should be always copied (by value) if you intend +//! to store them (don't create operands by using `new` keyword, it's not recommended). Operands are safe to be passed +//! to `memcpy()` and `memset()`, which becomes handy when working with arrays of operands. If you set all members of +//! an \ref Operand to zero the operand would become NONE operand, which is the same as a default constructed Operand. +//! +//! The example below illustrates how operands can be used and modified even without using any other code generation +//! classes. The example uses X86 architecture-specific operands. +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! // Registers can be copied, it's a common practice. +//! x86::Gp dstRegByValue() { return x86::ecx; } +//! +//! void usingOperandsExample(x86::Assembler& a) { +//! // Gets `ecx` register returned by a function. +//! x86::Gp dst = dstRegByValue(); +//! // Gets `rax` register directly from the provided `x86` namespace. +//! x86::Gp src = x86::rax; +//! // Constructs `r10` dynamically. +//! x86::Gp idx = x86::gpq(10); +//! // Constructs [src + idx] memory address - referencing [rax + r10]. +//! x86::Mem m = x86::ptr(src, idx); +//! +//! // Examine `m`: Returns `RegType::kX86_Gpq`. +//! m.indexType(); +//! // Examine `m`: Returns 10 (`r10`). +//! m.indexId(); +//! +//! // Reconstruct `idx` stored in mem: +//! x86::Gp idx_2 = x86::Gp::fromTypeAndId(m.indexType(), m.indexId()); +//! +//! // True, `idx` and idx_2` are identical. +//! idx == idx_2; +//! +//! // Possible - op will still be the same as `m`. +//! Operand op = m; +//! // True (can be casted to BaseMem or architecture-specific Mem). +//! op.isMem(); +//! +//! // True, `op` is just a copy of `m`. +//! m == op; +//! +//! // Static cast is fine and valid here. +//! static_cast(op).addOffset(1); +//! // However, using `as()` to cast to a derived type is preferred. +//! op.as().addOffset(1); +//! // False, `op` now points to [rax + r10 + 2], which is not [rax + r10]. +//! m == op; +//! +//! // Emitting 'mov' - type safe way. +//! a.mov(dst, m); +//! // Not possible, `mov` doesn't provide mov(x86::Gp, Operand) overload. +//! a.mov(dst, op); +//! +//! // Type-unsafe, but possible. +//! a.emit(x86::Inst::kIdMov, dst, m); +//! // Also possible, `emit()` is typeless and can be used with raw Operand. +//! a.emit(x86::Inst::kIdMov, dst, op); +//! } +//! ``` +//! +//! Some operands have to be created explicitly by emitters. For example labels must be created by \ref +//! BaseEmitter::newLabel(), which creates a label entry and returns a \ref Label operand with the id that refers +//! to it. Such label then can be used by emitters. +//! +//! ### Memory Operands +//! +//! Some architectures like X86 provide a complex memory addressing model that allows to encode addresses having a +//! BASE register, INDEX register with a possible scale (left shift), and displacement (called offset in AsmJit). +//! Memory address on X86 can also specify memory segment (segment-override in X86 terminology) and some instructions +//! (gather / scatter) require INDEX to be a \ref x86::Vec register instead of a general-purpose register. +//! +//! AsmJit allows to encode and work with all forms of addresses mentioned and implemented by X86. In addition, it +//! also allows to construct absolute 64-bit memory address operands, which is only allowed in one form of 'mov' +//! instruction. +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! void testX86Mem() { +//! // Makes it easier to access x86 stuff... +//! using namespace asmjit::x86; +//! +//! // BASE + OFFSET. +//! Mem a = ptr(rax); // a = [rax] +//! Mem b = ptr(rax, 15); // b = [rax + 15] +//! +//! // BASE + INDEX << SHIFT - Shift is in BITS as used by X86! +//! Mem c = ptr(rax, rbx); // c = [rax + rbx] +//! Mem d = ptr(rax, rbx, 2); // d = [rax + rbx << 2] +//! Mem e = ptr(rax, rbx, 2, 15); // e = [rax + rbx << 2 + 15] +//! +//! // BASE + VM (Vector Index) (encoded as MOD+VSIB). +//! Mem f = ptr(rax, xmm1); // f = [rax + xmm1] +//! Mem g = ptr(rax, xmm1, 2); // g = [rax + xmm1 << 2] +//! Mem h = ptr(rax, xmm1, 2, 15); // h = [rax + xmm1 << 2 + 15] +//! +//! // Absolute address: +//! uint64_t addr = (uint64_t)0x1234; +//! Mem i = ptr(addr); // i = [0x1234] +//! Mem j = ptr(addr, rbx); // j = [0x1234 + rbx] +//! Mem k = ptr(addr, rbx, 2); // k = [0x1234 + rbx << 2] +//! +//! // LABEL - Will be encoded as RIP (64-bit) or absolute address (32-bit). +//! Label L = ...; +//! Mem m = ptr(L); // m = [L] +//! Mem n = ptr(L, rbx); // n = [L + rbx] +//! Mem o = ptr(L, rbx, 2); // o = [L + rbx << 2] +//! Mem p = ptr(L, rbx, 2, 15); // p = [L + rbx << 2 + 15] +//! +//! // RIP - 64-bit only (RIP can't use INDEX). +//! Mem q = ptr(rip, 24); // q = [rip + 24] +//! } +//! ``` +//! +//! Memory operands can optionally contain memory size. This is required by instructions where the memory size cannot +//! be deduced from other operands, like `inc` and `dec` on X86: +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! void testX86Mem() { +//! // The same as: dword ptr [rax + rbx]. +//! x86::Mem a = x86::dword_ptr(rax, rbx); +//! +//! // The same as: qword ptr [rdx + rsi << 0 + 1]. +//! x86::Mem b = x86::qword_ptr(rdx, rsi, 0, 1); +//! } +//! ``` +//! +//! Memory operands provide API that can be used to access its properties: +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! void testX86Mem() { +//! // The same as: dword ptr [rax + 12]. +//! x86::Mem mem = x86::dword_ptr(rax, 12); +//! +//! mem.hasBase(); // true. +//! mem.hasIndex(); // false. +//! mem.size(); // 4. +//! mem.offset(); // 12. +//! +//! mem.setSize(0); // Sets the size to 0 (makes it sizeless). +//! mem.addOffset(-1); // Adds -1 to the offset and makes it 11. +//! mem.setOffset(0); // Sets the offset to 0. +//! mem.setBase(rcx); // Changes BASE to RCX. +//! mem.setIndex(rax); // Changes INDEX to RAX. +//! mem.hasIndex(); // true. +//! } +//! // ... +//! ``` +//! +//! Making changes to memory operand is very comfortable when emitting loads +//! and stores: +//! +//! ``` +//! #include +//! +//! using namespace asmjit; +//! +//! void testX86Mem(CodeHolder& code) { +//! x86::Assembler a(code); // Your initialized x86::Assembler. +//! x86::Mem mSrc = x86::ptr(eax); // Construct [eax] memory operand. +//! +//! // One way of emitting bunch of loads is to use `mem.adjusted()`, which +//! // returns a new memory operand and keeps the source operand unchanged. +//! a.movaps(x86::xmm0, mSrc); // No adjustment needed to load [eax]. +//! a.movaps(x86::xmm1, mSrc.adjusted(16)); // Loads from [eax + 16]. +//! a.movaps(x86::xmm2, mSrc.adjusted(32)); // Loads from [eax + 32]. +//! a.movaps(x86::xmm3, mSrc.adjusted(48)); // Loads from [eax + 48]. +//! +//! // ... do something with xmm0-3 ... +//! +//! // Another way of adjusting memory is to change the operand in-place. +//! // If you want to keep the original operand you can simply clone it. +//! x86::Mem mDst = mSrc.clone(); // Clone mSrc. +//! +//! a.movaps(mDst, x86::xmm0); // Stores xmm0 to [eax]. +//! mDst.addOffset(16); // Adds 16 to `mDst`. +//! +//! a.movaps(mDst, x86::xmm1); // Stores to [eax + 16] . +//! mDst.addOffset(16); // Adds 16 to `mDst`. +//! +//! a.movaps(mDst, x86::xmm2); // Stores to [eax + 32]. +//! mDst.addOffset(16); // Adds 16 to `mDst`. +//! +//! a.movaps(mDst, x86::xmm3); // Stores to [eax + 48]. +//! } +//! ``` +//! +//! ### Assembler Examples +//! +//! - \ref x86::Assembler provides many X86/X64 examples. + + +//! \defgroup asmjit_builder Builder +//! \brief Builder interface, nodes, and passes. +//! +//! ### Overview +//! +//! Both \ref BaseBuilder and \ref BaseCompiler interfaces describe emitters that emit into a representation that +//! allows further processing. The code stored in such representation is completely safe to be patched, simplified, +//! reordered, obfuscated, removed, injected, analyzed, or processed some other way. Each instruction, label, +//! directive, or other building block is stored as \ref BaseNode (or derived class like \ref InstNode or \ref +//! LabelNode) and contains all the information necessary to pass that node later to the assembler. +//! +//! \ref BaseBuilder is an emitter that inherits from \ref BaseEmitter interface. It was designed to provide a maximum +//! compatibility with the existing \ref BaseAssembler emitter so users can move from assembler to builder when needed, +//! for example to implement post-processing, which is not possible with Assembler. +//! +//! ### Builder Nodes +//! +//! \ref BaseBuilder doesn't generate machine code directly, it uses an intermediate representation based on nodes, +//! however, it allows to serialize to \ref BaseAssembler when the code is ready to be encoded. +//! +//! There are multiple node types used by both \ref BaseBuilder and \ref BaseCompiler : +//! +//! - Basic nodes: +//! - \ref BaseNode - Base class for all nodes. +//! - \ref InstNode - Represents an instruction node. +//! - \ref AlignNode - Represents an alignment directive (.align). +//! - \ref LabelNode - Represents a location where to bound a \ref Label. +//! +//! - Data nodes: +//! - \ref EmbedDataNode - Represents data. +//! - \ref EmbedLabelNode - Represents \ref Label address embedded as data. +//! - \ref EmbedLabelDeltaNode - Represents a difference of two labels embedded in data. +//! - \ref ConstPoolNode - Represents a constant pool data embedded as data. +//! +//! - Informative nodes: +//! - \ref CommentNode - Represents a comment string, doesn't affect code generation. +//! - \ref SentinelNode - A marker that can be used to remember certain position in code or data, doesn't affect +//! code generation. Used by \ref FuncNode to mark the end of a function. +//! +//! - Other nodes are provided by \ref asmjit_compiler infrastructure. +//! +//! ### Builder Examples +//! +//! - \ref x86::Builder provides many X86/X64 examples. + + +//! \defgroup asmjit_compiler Compiler +//! \brief Compiler interface. +//! +//! ### Overview +//! +//! \ref BaseCompiler is a high-level interface, which provides register allocation and support for defining and +//! invoking functions, built on top of \ref BaseBuilder interface At the moment it's the easiest way of generating +//! code in AsmJit as most architecture and OS specifics is properly abstracted and handled by AsmJit automatically. +//! However, abstractions also mean restrictions, which means that \ref BaseCompiler has more limitations than \ref +//! BaseAssembler or \ref BaseBuilder. +//! +//! Since \ref BaseCompiler provides register allocation it also establishes the concept of functions - a function +//! in Compiler sense is a unit in which virtual registers are allocated into physical registers by the register +//! allocator. In addition, it enables to use such virtual registers in function invocations. +//! +//! \ref BaseCompiler automatically handles function calling conventions. It's still architecture dependent, but +//! makes the code generation much easies. Functions are essential; the first-step to generate some code is to define +//! a signature of the function to be generated (before generating the function body itself). Function arguments and +//! return value(s) are handled by assigning virtual registers to them. Similarly, function calls are handled the same +//! way. +//! +//! ### Compiler Nodes +//! +//! \ref BaseCompiler adds some nodes that are required for function generation and invocation: +//! +//! - \ref FuncNode - Represents a function definition. +//! - \ref FuncRetNode - Represents a function return. +//! - \ref InvokeNode - Represents a function invocation. +//! +//! \ref BaseCompiler also makes the use of passes (\ref Pass) and automatically adds an architecture-dependent +//! register allocator pass to the list of passes when attached to \ref CodeHolder. +//! +//! ### Compiler Examples +//! +//! - \ref x86::Compiler provides many X86/X64 examples. +//! +//! ### Compiler Tips +//! +//! Users of AsmJit have done mistakes in the past, this section should provide some useful tips for beginners: +//! +//! - Virtual registers in compiler are bound to a single function. At the moment the implementation doesn't +//! care whether a single virtual register is used in multiple functions, but it sees it as two independent +//! virtual registers in that case. This means that virtual registers cannot be used to implement global +//! variables. Global variables are basically memory addresses which functions can read from and write to, +//! and they have to be implemented in the same way. +//! +//! - Compiler provides a useful debugging functionality, which can be turned on through \ref FormatFlags. Use +//! \ref Logger::addFlags() to turn on additional logging features when using Compiler. + + +//! \defgroup asmjit_function Function +//! \brief Function definitions. +//! +//! ### Overview +//! +//! AsmJit provides functionality that can be used to define function signatures and to calculate automatically +//! optimal function frame that can be used directly by a prolog and epilog insertion. This feature was exclusive +//! to AsmJit's Compiler for a very long time, but was abstracted out and is now available for all users regardless +//! of the emitter they use. The following use cases are possible: +//! +//! - Calculate function frame before the function is generated - this is the only way available to \ref +//! BaseAssembler users and it will be described in this section. +//! +//! - Calculate function frame after the function is generated - this way is generally used by \ref BaseBuilder +//! and \ref BaseCompiler emitters and this way is generally described in \ref asmjit_compiler section. +//! +//! The following concepts are used to describe and create functions in AsmJit: +//! +//! - \ref TypeId - Type-id is an 8-bit value that describes a platform independent type as we know from C/C++. +//! It provides abstractions for most common types like `int8_t`, `uint32_t`, `uintptr_t`, `float`, `double`, +//! and all possible vector types to match ISAs up to AVX512. \ref TypeId was introduced originally for \ref +//! asmjit_compiler, but it's now used by \ref FuncSignature as well. +//! +//! - \ref CallConv - Describes a calling convention - this class contains instructions to assign registers and +//! stack addresses to function arguments and return value(s), but doesn't specify any function signature itself. +//! Calling conventions are architecture and OS dependent. +//! +//! - \ref FuncSignature - Describes a function signature, for example `int func(int, int)`. FuncSignature contains +//! a function calling convention id, return value type, and function arguments. The signature itself is platform +//! independent and uses \ref TypeId to describe types of function arguments and function return value(s). +//! +//! - \ref FuncDetail - Architecture and ABI dependent information that describes \ref CallConv and expanded \ref +//! FuncSignature. Each function argument and return value is represented as \ref FuncValue that contains the +//! original \ref TypeId enriched with additional information that specifies whether the value is passed or +//! returned by register (and which register) or by stack. Each value also contains some other metadata that +//! provide additional information required to handle it properly (for example whether a vector is passed +//! indirectly by a pointer as required by WIN64 calling convention). +//! +//! - \ref FuncFrame - Contains information about the function frame that can be used by prolog/epilog inserter +//! (PEI). Holds call stack size size and alignment, local stack size and alignment, and various attributes that +//! describe how prolog and epilog should be constructed. `FuncFrame` doesn't know anything about function's +//! arguments or return values, it hold only information necessary to create a valid and ABI conforming function +//! prologs and epilogs. +//! +//! - \ref FuncArgsAssignment - A helper class that can be used to reassign function arguments into user specified +//! registers. It's architecture and ABI dependent mapping from function arguments described by \ref CallConv +//! and \ref FuncDetail into registers specified by the user. +//! +//! It's a lot of concepts where each represents one step in a function frame calculation. It can be used to create +//! function prologs, epilogs, and also to calculate information necessary to perform function calls. + + +//! \defgroup asmjit_logging Logging +//! \brief Logging and formatting. +//! +//! ### Overview +//! +//! The initial phase of a project that generates machine code is not always smooth. Failure cases are common not just +//! at the beginning phase, but also during the development or refactoring. AsmJit provides logging functionality to +//! address this issue. AsmJit does already a good job with function overloading to prevent from emitting unencodable +//! instructions, but it can't prevent from emitting machine code that is correct at instruction level, but doesn't +//! work when it's executed asa whole. Logging has always been an important part of AsmJit's infrastructure and looking +//! at logs can sometimes reveal code generation issues quickly. +//! +//! AsmJit provides API for logging and formatting: +//! +//! - \ref Logger - A logger that you can pass to \ref CodeHolder and all emitters that inherit from \ref BaseEmitter. +//! +//! - \ref FormatOptions - Formatting options that can change how instructions and operands are formatted. +//! +//! - \ref Formatter - A namespace that provides functions that can format input data like \ref Operand, \ref BaseReg, +//! \ref Label, and \ref BaseNode into \ref String. +//! +//! AsmJit's \ref Logger serves the following purposes: +//! +//! - Provides a basic foundation for logging. +//! +//! - Abstract class leaving the implementation on users. The following built-in implementations are provided for +//! simplicity: +//! +//! - \ref FileLogger implements logging into a standard `FILE` stream. +//! - \ref StringLogger serializes all logs into a \ref String instance. +//! +//! AsmJit's \ref FormatOptions provides the following to customize the formatting of instructions and operands through: +//! +//! - \ref FormatFlags +//! - \ref FormatIndentationGroup +//! +//! ### Logging +//! +//! A \ref Logger is typically attached to a \ref CodeHolder, which propagates it to all attached emitters +//! automatically. The example below illustrates how to use \ref FileLogger that outputs to standard output: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! int main() { +//! JitRuntime rt; // Runtime specialized for JIT code execution. +//! FileLogger logger(stdout); // Logger should always survive CodeHolder. +//! +//! CodeHolder code; // Holds code and relocation information. +//! code.init(rt.environment()); // Initialize to the same arch as JIT runtime. +//! code.setLogger(&logger); // Attach the `logger` to `code` holder. +//! +//! // ... code as usual, everything emitted will be logged to `stdout` ... +//! return 0; +//! } +//! ``` +//! +//! If output to FILE stream is not desired it's possible to use \ref StringLogger, which concatenates everything +//! into a multi-line string: +//! +//! ``` +//! #include +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! int main() { +//! JitRuntime rt; // Runtime specialized for JIT code execution. +//! StringLogger logger; // Logger should always survive CodeHolder. +//! +//! CodeHolder code; // Holds code and relocation information. +//! code.init(rt.environment()); // Initialize to the same arch as JIT runtime. +//! code.setLogger(&logger); // Attach the `logger` to `code` holder. +//! +//! // ... code as usual, logging will be concatenated to logger string ... +//! +//! // You can either use the string from StringLogger directly or you can +//! // move it. Logger::data() returns its content as null terminated char[]. +//! printf("Logger content: %s\n", logger.data()); +//! +//! // It can be moved into your own string like this: +//! String content = std::move(logger.content()); +//! printf("The same content: %s\n", content.data()); +//! +//! return 0; +//! } +//! ``` +//! +//! ### Formatting +//! +//! AsmJit uses \ref Formatter to format inputs that are then passed to \ref Logger. Formatting is public and can be +//! used by AsmJit users as well. The most important thing to know regarding formatting is that \ref Formatter always +//! appends to the output string, so it can be used to build complex strings without having to concatenate +//! intermediate strings. +//! +//! The first example illustrates how to format operands: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void logOperand(Arch arch, const Operand_& op) { +//! // The emitter is optional (named labels and virtual registers need it). +//! BaseEmitter* emitter = nullptr; +//! +//! // No flags by default. +//! FormatFlags formatFlags = FormatFlags::kNone; +//! +//! StringTmp<128> sb; +//! Formatter::formatOperand(sb, formatFlags, emitter, arch, op); +//! printf("%s\n", sb.data()); +//! } +//! +//! void formattingExample() { +//! using namespace x86; +//! +//! // Architecture is not part of operand, it must be passed explicitly. +//! // Format flags. We pass it explicitly also to 'logOperand' to make +//! // compatible with what AsmJit normally does. +//! Arch arch = Arch::kX64; +//! +//! log(arch, rax); // Prints 'rax'. +//! log(arch, ptr(rax, rbx, 2)); // Prints '[rax + rbx * 4]`. +//! log(arch, dword_ptr(rax, rbx, 2)); // Prints 'dword [rax + rbx * 4]`. +//! log(arch, imm(42)); // Prints '42'. +//! } +//! ``` +//! +//! Next example illustrates how to format whole instructions: +//! +//! ``` +//! #include +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! template +//! void logInstruction(Arch arch, const BaseInst& inst, Args&&... args) { +//! // The emitter is optional (named labels and virtual registers need it). +//! BaseEmitter* emitter = nullptr; +//! +//! // No flags by default. +//! FormatFlags formatFlags = FormatFlags::kNone; +//! +//! // The formatter expects operands in an array. +//! Operand_ operands { std::forward(args)... }; +//! +//! StringTmp<128> sb; +//! Formatter::formatInstruction( +//! sb, formatFlags, emitter, arch, inst, operands, sizeof...(args)); +//! printf("%s\n", sb.data()); +//! } +//! +//! void formattingExample() { +//! using namespace x86; +//! +//! // Architecture is not part of operand, it must be passed explicitly. +//! // Format flags. We pass it explicitly also to 'logOperand' to make +//! // compatible with what AsmJit normally does. +//! Arch arch = Arch::kX64; +//! +//! // Prints 'mov rax, rcx'. +//! logInstruction(arch, BaseInst(Inst::kIdMov), rax, rcx); +//! +//! // Prints 'vaddpd zmm0, zmm1, [rax] {1to8}'. +//! logInstruction(arch, +//! BaseInst(Inst::kIdVaddpd), +//! zmm0, zmm1, ptr(rax)._1toN()); +//! +//! // BaseInst abstracts instruction id, instruction options, and extraReg. +//! // Prints 'lock add [rax], rcx'. +//! logInstruction(arch, +//! BaseInst(Inst::kIdAdd, InstOptions::kX86_Lock), +//! x86::ptr(rax), rcx); +//! +//! // Similarly an extra register (like AVX-512 selector) can be used. +//! // Prints 'vaddpd zmm0 {k2} {z}, zmm1, [rax]'. +//! logInstruction(arch, +//! BaseInst(Inst::kIdAdd, InstOptions::kX86_ZMask, k2), +//! zmm0, zmm1, ptr(rax)); +//! } +//! ``` +//! +//! And finally, the example below illustrates how to use a built-in function to format the content of +//! \ref BaseBuilder, which consists of nodes: +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! void formattingExample(BaseBuilder* builder) { +//! FormatOptions formatOptions {}; +//! +//! // This also shows how temporary strings can be used. +//! StringTmp<512> sb; +//! +//! // FormatNodeList requires the String for output, formatting flags, which +//! // were zero (no extra flags), and the builder instance, which we have +//! // provided. An overloaded version also exists, which accepts begin and +//! // and end nodes, which can be used to only format a range of nodes. +//! Formatter::formatNodeList(sb, formatOptions, builder); +//! +//! // You can do whatever else with the string, it's always null terminated, +//! // so it can be passed to C functions like printf(). +//! printf("%s\n", sb.data()); +//! } +//! ``` + + +//! \defgroup asmjit_error_handling Error Handling +//! \brief Error handling. +//! +//! ### Overview +//! +//! AsmJit uses error codes to represent and return errors. Every function that can fail returns an \ref Error code. +//! Exceptions are never thrown by AsmJit itself even in extreme conditions like out-of-memory, but it's possible to +//! override \ref ErrorHandler::handleError() to throw, in that case no error will be returned and exception will be +//! thrown instead. All functions where this can happen are not marked `noexcept`. +//! +//! Errors should never be ignored, however, checking errors after each AsmJit API call would simply overcomplicate +//! the whole code generation experience. \ref ErrorHandler exists to make the use of AsmJit API simpler as it allows +//! to customize how errors can be handled: +//! +//! - Record the error and continue (the way how the error is user-implemented). +//! - Throw an exception. AsmJit doesn't use exceptions and is completely exception-safe, but it's perfectly legal +//! to throw an exception from the error handler. +//! - Use plain old C's `setjmp()` and `longjmp()`. Asmjit always puts Assembler, Builder and Compiler to a +//! consistent state before calling \ref ErrorHandler::handleError(), so `longjmp()` can be used without issues +//! to cancel the code-generation if an error occurred. This method can be used if exception handling in your +//! project is turned off and you still want some comfort. In most cases it should be safe as AsmJit uses \ref +//! Zone memory and the ownership of memory it allocates always ends with the instance that allocated it. If +//! using this approach please never jump outside the life-time of \ref CodeHolder and \ref BaseEmitter. +//! +//! ### Using ErrorHandler +//! +//! An example of attaching \ref ErrorHandler to \ref CodeHolder. +//! +//! ``` +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! // A simple error handler implementation, extend according to your needs. +//! class MyErrorHandler : public ErrorHandler { +//! public: +//! void handleError(Error err, const char* message, BaseEmitter* origin) override { +//! printf("AsmJit error: %s\n", message); +//! } +//! }; +//! +//! int main() { +//! JitRuntime rt; +//! +//! MyErrorHandler myErrorHandler; +//! CodeHolder code; +//! +//! code.init(rt.environment()); +//! code.setErrorHandler(&myErrorHandler); +//! +//! x86::Assembler a(&code); +//! // ... code generation ... +//! +//! return 0; +//! } +//! ``` +//! +//! Useful classes in error handling group: +//! +//! - See \ref DebugUtils that provides utilities useful for debugging. +//! - See \ref Error that lists error codes that AsmJit uses. +//! - See \ref ErrorHandler for more details about error handling. + + +//! \defgroup asmjit_instruction_db Instruction DB +//! \brief Instruction database (introspection, read/write, validation, ...). +//! +//! ### Overview +//! +//! AsmJit provides a public instruction database that can be used to query information about a complete instruction. +//! The instruction database requires the knowledge of the following: +//! +//! - \ref BaseInst - Base instruction that contains instruction id, options, and a possible extra-register that +//! represents either REP prefix counter or AVX-512 selector (mask). +//! +//! - \ref Operand - Represents operands of an instruction. +//! +//! Each instruction can be then queried for the following information: +//! +//! - \ref InstRWInfo - Read/write information of instruction and its oprands (includes \ref OpRWInfo). +//! +//! - \ref CpuFeatures - CPU features required to execute the instruction. +//! +//! In addition to query functionality AsmJit is also able to validate whether an instruction and its operands are +//! valid. This is useful for making sure that what user tries to emit is correct and it can be also used by other +//! projects that parse user input, like AsmTK project. +//! +//! ### Query API +//! +//! The instruction query API is provided by \ref InstAPI namespace. The following queries are possible: +//! +//! - \ref InstAPI::queryRWInfo() - queries read/write information of the given instruction and its operands. +//! Includes also CPU flags read/written. +//! +//! - \ref InstAPI::queryFeatures() - queries CPU features that are required to execute the given instruction. A full +//! instruction with operands must be given as some architectures like X86 may require different features for the +//! same instruction based on its operands. +//! +//! - asmjit_test_instinfo.cpp +//! can be also used as a reference about accessing instruction information. +//! +//! ### Validation API +//! +//! The instruction validation API is provided by \ref InstAPI namespace in the similar fashion like the Query API, +//! however, validation can also be turned on at \ref BaseEmitter level. The following is possible: +//! +//! - \ref InstAPI::validate() - low-level instruction validation function that is used internally by emitters +//! if strict validation is enabled. +//! +//! - \ref BaseEmitter::addDiagnosticOptions() - can be used to enable validation at emitter level, see \ref +//! DiagnosticOptions. + + +//! \defgroup asmjit_virtual_memory Virtual Memory +//! \brief Virtual memory management. +//! +//! ### Overview +//! +//! AsmJit's virtual memory management is divided into two main categories: +//! +//! - Low level API that provides cross-platform abstractions for virtual memory allocation. Implemented in +//! \ref VirtMem namespace. +//! +//! - High level API that makes it very easy to store generated code for execution. See \ref JitRuntime, which is +//! used by many examples for its simplicity and easy integration with \ref CodeHolder. There is also \ref +//! JitAllocator, which lays somewhere between RAW memory allocation and \ref JitRuntime. + + +//! \defgroup asmjit_zone Zone Memory +//! \brief Zone memory allocator and containers. +//! +//! ### Overview +//! +//! AsmJit uses zone memory allocation (also known as Arena allocation) to allocate most of the data it uses. It's a +//! fast allocator that allows AsmJit to allocate a lot of small data structures fast and without `malloc()` overhead. +//! Since code generators and all related classes are usually short-lived this approach decreases memory usage and +//! fragmentation as arena-based allocators always allocate larger blocks of memory, which are then split into smaller +//! chunks. +//! +//! Another advantage of zone memory allocation is that since the whole library uses this strategy it's very easy to +//! deallocate everything that a particular instance is holding by simply releasing the memory the allocator holds. +//! This improves destruction time of such objects as there is no destruction at all. Long-lived objects just reset +//! its data in destructor or in their reset() member function for a future reuse. For this purpose all containers in +//! AsmJit are also zone allocated. +//! +//! ### Zone Allocation +//! +//! - \ref Zone - Incremental zone memory allocator with minimum features. It can only allocate memory without the +//! possibility to return it back to the allocator. +//! +//! - \ref ZoneTmp - A temporary \ref Zone with some initial static storage. If the allocation requests fit the +//! static storage allocated then there will be no dynamic memory allocation during the lifetime of \ref ZoneTmp, +//! otherwise it would act as \ref Zone with one preallocated block on the stack. +//! +//! - \ref ZoneAllocator - A wrapper of \ref Zone that provides the capability of returning memory to the allocator. +//! Such memory is stored in a pool for later reuse. +//! +//! ### Zone Allocated Containers +//! +//! - \ref ZoneString - Zone allocated string. +//! - \ref ZoneHash - Zone allocated hash table. +//! - \ref ZoneTree - Zone allocated red-black tree. +//! - \ref ZoneList - Zone allocated double-linked list. +//! - \ref ZoneStack - Zone allocated stack. +//! - \ref ZoneVector - Zone allocated vector. +//! - \ref ZoneBitVector - Zone allocated vector of bits. +//! +//! ### Using Zone Allocated Containers +//! +//! The most common data structure exposed by AsmJit is \ref ZoneVector. It's very similar to `std::vector`, but the +//! implementation doesn't use exceptions and uses the mentioned \ref ZoneAllocator for performance reasons. You don't +//! have to worry about allocations as you should not need to add items to AsmJit's data structures directly as there +//! should be API for all required operations. +//! +//! The following APIs in \ref CodeHolder returns \ref ZoneVector reference: +//! +//! ``` +//! using namespace asmjit; +//! +//! void example(CodeHolder& code) { +//! // Contains all emitters attached to CodeHolder. +//! const ZoneVector& emitters = code.emitters(); +//! +//! // Contains all section entries managed by CodeHolder. +//! const ZoneVector& sections = code.sections(); +//! +//! // Contains all label entries managed by CodeHolder. +//! const ZoneVector& labelEntries = code.labelEntries(); +//! +//! // Contains all relocation entries managed by CodeHolder. +//! const ZoneVector& relocEntries = code.relocEntries(); +//! } +//! ``` +//! +//! \ref ZoneVector has overloaded array access operator to make it possible to access its elements through operator[]. +//! Some standard functions like \ref ZoneVector::empty(), \ref ZoneVector::size(), and \ref ZoneVector::data() are +//! provided as well. Vectors are also iterable through a range-based for loop: +//! +//! ``` +//! using namespace asmjit; +//! +//! void example(CodeHolder& code) { +//! for (LabelEntry* le : code.labelEntries()) { +//! printf("Label #%u {Bound=%s Offset=%llu}", +//! le->id(), +//! le->isBound() ? "true" : "false", +//! (unsigned long long)le->offset()); +//! } +//! } +//! ``` +//! +//! ### Design Considerations +//! +//! Zone-allocated containers do not store the allocator within the container. This decision was made to reduce the +//! footprint of such containers as AsmJit tooling, especially Compiler's register allocation, may use many instances +//! of such containers to perform code analysis and register allocation. +//! +//! For example to append an item into a \ref ZoneVector it's required to pass the allocator as the first argument, +//! so it can be used in case that the vector needs a reallocation. Such function also returns an error, which must +//! be propagated to the caller. +//! +//! ``` +//! using namespace asmjit +//! +//! Error example(ZoneAllocator* allocator) { +//! ZoneVector vector; +//! +//! // Unfortunately, allocator must be provided to all functions that mutate +//! // the vector. However, AsmJit users should never need to do this as all +//! // manipulation should be done through public API, which takes care of +//! // that. +//! for (int i = 0; i < 100; i++) { +//! ASMJIT_PROPAGATE(vector.append(allocator, i)); +//! } +//! +//! // By default vector's destructor doesn't release anything as it knows +//! // that its content is zone allocated. However, \ref ZoneVector::release +//! // can be used to explicitly release the vector data to the allocator if +//! // necessary +//! vector.release(allocator); +//! } +//! ``` +//! +//! Containers like \ref ZoneVector also provide a functionality to reserve a certain number of items before any items +//! are added to it. This approach is used internally in most places as it allows to prepare space for data that will +//! be added to some container before the data itself was created. +//! +//! ``` +//! using namespace asmjit +//! +//! Error example(ZoneAllocator* allocator) { +//! ZoneVector vector; +//! +//! ASMJIT_PROPAGATE(vector.willGrow(100)); +//! for (int i = 0; i < 100; i++) { +//! // Cannot fail. +//! vector.appendUnsafe(allocator, i); +//! } +//! +//! vector.release(allocator); +//! } +//! ``` + + +//! \defgroup asmjit_utilities Utilities +//! \brief Utility classes and functions. +//! +//! ### Overview +//! +//! AsmJit uses and provides utility classes and functions, that can be used with AsmJit. The functionality can be +//! divided into the following topics: +//! +//! ### String Functionality +//! +//! - \ref String - AsmJit's string container, which is used internally and which doesn't use exceptions and has +//! a stable layout, which is not dependent on C++ standard library. +//! +//! - \ref StringTmp - String that can have base storage allocated on stack. The amount of storage on stack can +//! be specified as a template parameter. +//! +//! - \ref FixedString - Fixed string container limited up to N characters. +//! +//! ### Code Generation Utilities +//! +//! - \ref ConstPool - Constant pool used by \ref BaseCompiler, but also available to users that may find use of it. +//! +//! ### Support Functionality Used by AsmJit +//! +//! - \ref Support namespace provides many other utility functions and classes that are used by AsmJit, and made +//! public. + + +//! \defgroup asmjit_x86 X86 Backend +//! \brief X86/X64 backend. + + +//! \defgroup asmjit_arm ARM Commons +//! \brief ARM commons shared between AArch32 and AArch64. + + +//! \defgroup asmjit_a64 AArch64 Backend +//! \brief AArch64 backend. + + +//! \cond INTERNAL +//! \defgroup asmjit_ra RA +//! \brief Register allocator internals. +//! \endcond + +} // {asmjit} + +#include "asmjit-scope-begin.h" +#include "core/archtraits.h" +#include "core/assembler.h" +#include "core/builder.h" +#include "core/codeholder.h" +#include "core/compiler.h" +#include "core/constpool.h" +#include "core/cpuinfo.h" +#include "core/emitter.h" +#include "core/environment.h" +#include "core/errorhandler.h" +#include "core/formatter.h" +#include "core/func.h" +#include "core/globals.h" +#include "core/inst.h" +#include "core/jitallocator.h" +#include "core/jitruntime.h" +#include "core/logger.h" +#include "core/operand.h" +#include "core/osutils.h" +#include "core/string.h" +#include "core/support.h" +#include "core/target.h" +#include "core/type.h" +#include "core/virtmem.h" +#include "core/zone.h" +#include "core/zonehash.h" +#include "core/zonelist.h" +#include "core/zonetree.h" +#include "core/zonestack.h" +#include "core/zonestring.h" +#include "core/zonevector.h" +#include "asmjit-scope-end.h" + +#endif // ASMJIT_CORE_H_INCLUDED diff --git a/src/asmjit/core/api-build_p.h b/src/asmjit/core/api-build_p.h new file mode 100644 index 0000000..6eca971 --- /dev/null +++ b/src/asmjit/core/api-build_p.h @@ -0,0 +1,55 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_API_BUILD_P_H_INCLUDED +#define ASMJIT_CORE_API_BUILD_P_H_INCLUDED + +#define ASMJIT_EXPORTS + +// Only turn-off these warnings when building asmjit itself. +#ifdef _MSC_VER + #ifndef _CRT_SECURE_NO_DEPRECATE + #define _CRT_SECURE_NO_DEPRECATE + #endif + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif +#endif + +// Dependencies only required for asmjit build, but never exposed through public headers. +#ifdef _WIN32 + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #ifndef NOMINMAX + #define NOMINMAX + #endif + #include +#endif + +#include "./api-config.h" + +#if !defined(ASMJIT_BUILD_DEBUG) && defined(__GNUC__) && !defined(__clang__) + #define ASMJIT_FAVOR_SIZE __attribute__((__optimize__("Os"))) + #define ASMJIT_FAVOR_SPEED __attribute__((__optimize__("O3"))) +#elif ASMJIT_CXX_HAS_ATTRIBUTE(__minsize__, 0) + #define ASMJIT_FAVOR_SIZE __attribute__((__minsize__)) + #define ASMJIT_FAVOR_SPEED +#else + #define ASMJIT_FAVOR_SIZE + #define ASMJIT_FAVOR_SPEED +#endif + +// Make sure '#ifdef'ed unit tests are properly highlighted in IDE. +#if !defined(ASMJIT_TEST) && defined(__INTELLISENSE__) + #define ASMJIT_TEST +#endif + +// Include a unit testing package if this is a `asmjit_test_unit` build. +#if defined(ASMJIT_TEST) + #include "../../../test/broken.h" +#endif + +#endif // ASMJIT_CORE_API_BUILD_P_H_INCLUDED diff --git a/src/asmjit/core/api-config.h b/src/asmjit/core/api-config.h new file mode 100644 index 0000000..a0fb979 --- /dev/null +++ b/src/asmjit/core/api-config.h @@ -0,0 +1,613 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_API_CONFIG_H_INCLUDED +#define ASMJIT_CORE_API_CONFIG_H_INCLUDED + +// AsmJit Library & ABI Version +// ============================ + +//! \addtogroup asmjit_core +//! \{ + +//! AsmJit library version in `(Major << 16) | (Minor << 8) | (Patch)` format. +#define ASMJIT_LIBRARY_VERSION 0x010900 /* 1.9.0 */ + +//! \def ASMJIT_ABI_NAMESPACE +//! +//! AsmJit ABI namespace is an inline namespace within \ref asmjit namespace. +//! +//! It's used to make sure that when user links to an incompatible version of AsmJit, it won't link. It has also some +//! additional properties as well. When `ASMJIT_ABI_NAMESPACE` is defined by the user it would override the AsmJit +//! default, which makes it possible to use use multiple AsmJit libraries within a single project, totally controlled +//! by the users. This is useful especially in cases in which some of such library comes from a third party. +#ifndef ASMJIT_ABI_NAMESPACE + #define ASMJIT_ABI_NAMESPACE _abi_1_9 +#endif + +//! \} + +// Global Dependencies +// =================== + +#include +#include +#include // We really want std types as globals, not under 'std' namespace. +#include +#include +#include + +#include +#include +#include +#include +#include + +#if !defined(_WIN32) && !defined(__EMSCRIPTEN__) + #include +#endif + +// Build Options +// ============= + +// NOTE: Doxygen cannot document macros that are not defined, that's why we have to define them and then undefine +// them immediately, so it won't use the macros with its own preprocessor. +#ifdef _DOXYGEN +namespace asmjit { + +//! \addtogroup asmjit_build +//! \{ + +//! Asmjit is embedded, implies \ref ASMJIT_STATIC. +#define ASMJIT_EMBED + +//! Enables static-library build. +#define ASMJIT_STATIC + +//! Defined when AsmJit's build configuration is 'Debug'. +//! +//! \note Can be defined explicitly to bypass autodetection. +#define ASMJIT_BUILD_DEBUG + +//! Defined when AsmJit's build configuration is 'Release'. +//! +//! \note Can be defined explicitly to bypass autodetection. +#define ASMJIT_BUILD_RELEASE + +//! Disables X86/X64 backends. +#define ASMJIT_NO_X86 + +//! Disables AArch32 backends (both ARM and Thumb). +#define ASMJIT_NO_AARCH32 + +//! Disables AArch64 backend. +#define ASMJIT_NO_AARCH64 + +//! Disables non-host backends entirely (useful for JIT compilers to minimize the library size). +#define ASMJIT_NO_FOREIGN + +//! Disables deprecated API at compile time (deprecated API won't be available). +#define ASMJIT_NO_DEPRECATED + +//! Disables \ref asmjit_builder functionality completely. +#define ASMJIT_NO_BUILDER + +//! Disables \ref asmjit_compiler functionality completely. +#define ASMJIT_NO_COMPILER + +//! Disables JIT memory management and \ref asmjit::JitRuntime. +#define ASMJIT_NO_JIT + +//! Disables \ref asmjit::Logger and \ref asmjit::Formatter. +#define ASMJIT_NO_LOGGING + +//! Disables everything that contains text. +#define ASMJIT_NO_TEXT + +//! Disables instruction validation API. +#define ASMJIT_NO_VALIDATION + +//! Disables instruction introspection API. +#define ASMJIT_NO_INTROSPECTION + +// Avoid doxygen preprocessor using feature-selection definitions. +#undef ASMJIT_BUILD_EMBNED +#undef ASMJIT_BUILD_STATIC +#undef ASMJIT_BUILD_DEBUG +#undef ASMJIT_BUILD_RELEASE +#undef ASMJIT_NO_X86 +#undef ASMJIT_NO_FOREIGN +// (keep ASMJIT_NO_DEPRECATED defined, we don't document deprecated APIs). +#undef ASMJIT_NO_BUILDER +#undef ASMJIT_NO_COMPILER +#undef ASMJIT_NO_JIT +#undef ASMJIT_NO_LOGGING +#undef ASMJIT_NO_TEXT +#undef ASMJIT_NO_VALIDATION +#undef ASMJIT_NO_INTROSPECTION + +//! \} + +} // {asmjit} +#endif // _DOXYGEN + +// ASMJIT_NO_BUILDER implies ASMJIT_NO_COMPILER. +#if defined(ASMJIT_NO_BUILDER) && !defined(ASMJIT_NO_COMPILER) + #define ASMJIT_NO_COMPILER +#endif + +// Prevent compile-time errors caused by misconfiguration. +#if defined(ASMJIT_NO_TEXT) && !defined(ASMJIT_NO_LOGGING) + #pragma message("'ASMJIT_NO_TEXT' can only be defined when 'ASMJIT_NO_LOGGING' is defined.") + #undef ASMJIT_NO_TEXT +#endif + +#if defined(ASMJIT_NO_INTROSPECTION) && !defined(ASMJIT_NO_COMPILER) + #pragma message("'ASMJIT_NO_INTROSPECTION' can only be defined when 'ASMJIT_NO_COMPILER' is defined") + #undef ASMJIT_NO_INTROSPECTION +#endif + +// Build Mode +// ========== + +// Detect ASMJIT_BUILD_DEBUG and ASMJIT_BUILD_RELEASE if not defined. +#if !defined(ASMJIT_BUILD_DEBUG) && !defined(ASMJIT_BUILD_RELEASE) + #if !defined(NDEBUG) + #define ASMJIT_BUILD_DEBUG + #else + #define ASMJIT_BUILD_RELEASE + #endif +#endif + +// Target Architecture Detection +// ============================= + +#if defined(_M_X64) || defined(__x86_64__) + #define ASMJIT_ARCH_X86 64 +#elif defined(_M_IX86) || defined(__X86__) || defined(__i386__) + #define ASMJIT_ARCH_X86 32 +#else + #define ASMJIT_ARCH_X86 0 +#endif + +#if defined(__arm64__) || defined(__aarch64__) +# define ASMJIT_ARCH_ARM 64 +#elif defined(_M_ARM) || defined(_M_ARMT) || defined(__arm__) || defined(__thumb__) || defined(__thumb2__) + #define ASMJIT_ARCH_ARM 32 +#else + #define ASMJIT_ARCH_ARM 0 +#endif + +#if defined(_MIPS_ARCH_MIPS64) || defined(__mips64) + #define ASMJIT_ARCH_MIPS 64 +#elif defined(_MIPS_ARCH_MIPS32) || defined(_M_MRX000) || defined(__mips__) + #define ASMJIT_ARCH_MIPS 32 +#else + #define ASMJIT_ARCH_MIPS 0 +#endif + +#define ASMJIT_ARCH_BITS (ASMJIT_ARCH_X86 | ASMJIT_ARCH_ARM | ASMJIT_ARCH_MIPS) +#if ASMJIT_ARCH_BITS == 0 + #undef ASMJIT_ARCH_BITS + #if defined (__LP64__) || defined(_LP64) + #define ASMJIT_ARCH_BITS 64 + #else + #define ASMJIT_ARCH_BITS 32 + #endif +#endif + +#if (defined(__ARMEB__)) || \ + (defined(__MIPSEB__)) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + #define ASMJIT_ARCH_LE 0 + #define ASMJIT_ARCH_BE 1 +#else + #define ASMJIT_ARCH_LE 1 + #define ASMJIT_ARCH_BE 0 +#endif + +#if defined(ASMJIT_NO_FOREIGN) + #if !ASMJIT_ARCH_X86 && !defined(ASMJIT_NO_X86) + #define ASMJIT_NO_X86 + #endif + + #if !ASMJIT_ARCH_ARM && !defined(ASMJIT_NO_AARCH64) + #define ASMJIT_NO_AARCH64 + #endif +#endif + + +// C++ Compiler and Features Detection +// =================================== + +#define ASMJIT_CXX_GNU 0 +#define ASMJIT_CXX_MAKE_VER(MAJOR, MINOR) ((MAJOR) * 1000 + (MINOR)) + +// Intel Compiler [pretends to be GNU or MSC, so it must be checked first]: +// - https://software.intel.com/en-us/articles/c0x-features-supported-by-intel-c-compiler +// - https://software.intel.com/en-us/articles/c14-features-supported-by-intel-c-compiler +// - https://software.intel.com/en-us/articles/c17-features-supported-by-intel-c-compiler +#if defined(__INTEL_COMPILER) + +// MSC Compiler: +// - https://msdn.microsoft.com/en-us/library/hh567368.aspx +// +// Version List: +// - 16.00.0 == VS2010 +// - 17.00.0 == VS2012 +// - 18.00.0 == VS2013 +// - 19.00.0 == VS2015 +// - 19.10.0 == VS2017 +#elif defined(_MSC_VER) && defined(_MSC_FULL_VER) + +// Clang Compiler [Pretends to be GNU, so it must be checked before]: +// - https://clang.llvm.org/cxx_status.html +#elif defined(__clang_major__) && defined(__clang_minor__) && defined(__clang_patchlevel__) + +// GNU Compiler: +// - https://gcc.gnu.org/projects/cxx-status.html +#elif defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) + + #undef ASMJIT_CXX_GNU + #define ASMJIT_CXX_GNU ASMJIT_CXX_MAKE_VER(__GNUC__, __GNUC_MINOR__) + +#endif + +// Compiler features detection macros. +#if defined(__clang__) && defined(__has_attribute) + #define ASMJIT_CXX_HAS_ATTRIBUTE(NAME, CHECK) (__has_attribute(NAME)) +#else + #define ASMJIT_CXX_HAS_ATTRIBUTE(NAME, CHECK) (!(!(CHECK))) +#endif + +// API Decorators & C++ Extensions +// =============================== + +//! \def ASMJIT_API +//! +//! A decorator that is used to decorate API that AsmJit exports when built as a shared library. + +// API (Export / Import). +#if !defined(ASMJIT_STATIC) + #if defined(_WIN32) && (defined(_MSC_VER) || defined(__MINGW32__)) + #ifdef ASMJIT_EXPORTS + #define ASMJIT_API __declspec(dllexport) + #else + #define ASMJIT_API __declspec(dllimport) + #endif + #elif defined(_WIN32) && defined(__GNUC__) + #ifdef ASMJIT_EXPORTS + #define ASMJIT_API __attribute__((__dllexport__)) + #else + #define ASMJIT_API __attribute__((__dllimport__)) + #endif + #elif defined(__GNUC__) + #define ASMJIT_API __attribute__((__visibility__("default"))) + #endif +#endif + +#if !defined(ASMJIT_API) + #define ASMJIT_API +#endif + +#if !defined(ASMJIT_VARAPI) + #define ASMJIT_VARAPI extern ASMJIT_API +#endif + +//! \def ASMJIT_VIRTAPI +//! +//! This is basically a workaround. When using MSVC and marking class as DLL export everything gets exported, which +//! is unwanted in most projects. MSVC automatically exports typeinfo and vtable if at least one symbol of the class +//! is exported. However, GCC has some strange behavior that even if one or more symbol is exported it doesn't export +//! typeinfo unless the class itself is decorated with "visibility(default)" (i.e. ASMJIT_API). +#if !defined(_WIN32) && defined(__GNUC__) + #define ASMJIT_VIRTAPI ASMJIT_API +#else + #define ASMJIT_VIRTAPI +#endif + +// Function attributes. +#if !defined(ASMJIT_BUILD_DEBUG) && defined(__GNUC__) + #define ASMJIT_FORCE_INLINE inline __attribute__((__always_inline__)) +#elif !defined(ASMJIT_BUILD_DEBUG) && defined(_MSC_VER) + #define ASMJIT_FORCE_INLINE __forceinline +#else + #define ASMJIT_FORCE_INLINE inline +#endif + +#if defined(__GNUC__) + #define ASMJIT_NOINLINE __attribute__((__noinline__)) + #define ASMJIT_NORETURN __attribute__((__noreturn__)) +#elif defined(_MSC_VER) + #define ASMJIT_NOINLINE __declspec(noinline) + #define ASMJIT_NORETURN __declspec(noreturn) +#else + #define ASMJIT_NOINLINE + #define ASMJIT_NORETURN +#endif + +// Calling conventions. +#if ASMJIT_ARCH_X86 == 32 && defined(__GNUC__) + #define ASMJIT_CDECL __attribute__((__cdecl__)) + #define ASMJIT_STDCALL __attribute__((__stdcall__)) + #define ASMJIT_FASTCALL __attribute__((__fastcall__)) + #define ASMJIT_REGPARM(N) __attribute__((__regparm__(N))) +#elif ASMJIT_ARCH_X86 == 32 && defined(_MSC_VER) + #define ASMJIT_CDECL __cdecl + #define ASMJIT_STDCALL __stdcall + #define ASMJIT_FASTCALL __fastcall + #define ASMJIT_REGPARM(N) +#else + #define ASMJIT_CDECL + #define ASMJIT_STDCALL + #define ASMJIT_FASTCALL + #define ASMJIT_REGPARM(N) +#endif + +#if ASMJIT_ARCH_X86 && defined(_WIN32) && defined(_MSC_VER) + #define ASMJIT_VECTORCALL __vectorcall +#elif ASMJIT_ARCH_X86 && defined(_WIN32) + #define ASMJIT_VECTORCALL __attribute__((__vectorcall__)) +#else + #define ASMJIT_VECTORCALL +#endif + +// Type alignment (not allowed by C++11 'alignas' keyword). +#if defined(__GNUC__) + #define ASMJIT_ALIGN_TYPE(TYPE, N) __attribute__((__aligned__(N))) TYPE +#elif defined(_MSC_VER) + #define ASMJIT_ALIGN_TYPE(TYPE, N) __declspec(align(N)) TYPE +#else + #define ASMJIT_ALIGN_TYPE(TYPE, N) TYPE +#endif + +//! \def ASMJIT_MAY_ALIAS +//! +//! Expands to `__attribute__((__may_alias__))` if supported. +#if defined(__GNUC__) + #define ASMJIT_MAY_ALIAS __attribute__((__may_alias__)) +#else + #define ASMJIT_MAY_ALIAS +#endif + +//! \def ASMJIT_MAYBE_UNUSED +//! +//! Expands to `[[maybe_unused]]` if supported or a compiler attribute instead. +#if __cplusplus >= 201703L + #define ASMJIT_MAYBE_UNUSED [[maybe_unused]] +#elif defined(__GNUC__) + #define ASMJIT_MAYBE_UNUSED __attribute__((unused)) +#else + #define ASMJIT_MAYBE_UNUSED +#endif + +#if defined(__clang_major__) && __clang_major__ >= 4 && !defined(_DOXYGEN) + // NOTE: Clang allows to apply this attribute to function arguments, which is what we want. Once GCC decides to + // support this use, we will enable it for GCC as well. However, until that, it will be clang only, which is + // what we need for static analysis. + #define ASMJIT_NONNULL(FUNCTION_ARGUMENT) FUNCTION_ARGUMENT __attribute__((__nonnull__)) +#else + #define ASMJIT_NONNULL(FUNCTION_ARGUMENT) FUNCTION_ARGUMENT +#endif + +//! \def ASMJIT_NOEXCEPT_TYPE +//! +//! Defined to `noexcept` in C++17 mode or nothing otherwise. Used by function typedefs. +#if __cplusplus >= 201703L + #define ASMJIT_NOEXCEPT_TYPE noexcept +#else + #define ASMJIT_NOEXCEPT_TYPE +#endif + +//! \def ASMJIT_ASSUME(...) +//! +//! Macro that tells the C/C++ compiler that the expression `...` evaluates to true. +//! +//! This macro has two purposes: +//! +//! 1. Enable optimizations that would not be possible without the assumption. +//! 2. Hint static analysis tools that a certain condition is true to prevent false positives. +#if defined(__clang__) + #define ASMJIT_ASSUME(...) __builtin_assume(__VA_ARGS__) +#elif defined(__GNUC__) + #define ASMJIT_ASSUME(...) do { if (!(__VA_ARGS__)) __builtin_unreachable(); } while (0) +#elif defined(_MSC_VER) + #define ASMJIT_ASSUME(...) __assume(__VA_ARGS__) +#else + #define ASMJIT_ASSUME(...) (void)0 +#endif + +//! \def ASMJIT_LIKELY(...) +//! +//! Condition is likely to be taken (mostly error handling and edge cases). + +//! \def ASMJIT_UNLIKELY(...) +//! +//! Condition is unlikely to be taken (mostly error handling and edge cases). +#if defined(__GNUC__) + #define ASMJIT_LIKELY(...) __builtin_expect(!!(__VA_ARGS__), 1) + #define ASMJIT_UNLIKELY(...) __builtin_expect(!!(__VA_ARGS__), 0) +#else + #define ASMJIT_LIKELY(...) (__VA_ARGS__) + #define ASMJIT_UNLIKELY(...) (__VA_ARGS__) +#endif + +//! \def ASMJIT_FALLTHROUGH +//! +//! Portable [[fallthrough]] attribute. +#if defined(__clang__) && __cplusplus >= 201103L + #define ASMJIT_FALLTHROUGH [[clang::fallthrough]] +#elif defined(__GNUC__) && __GNUC__ >= 7 + #define ASMJIT_FALLTHROUGH __attribute__((__fallthrough__)) +#else + #define ASMJIT_FALLTHROUGH ((void)0) /* fallthrough */ +#endif + +//! \def ASMJIT_DEPRECATED +//! +//! Marks function, class, struct, enum, or anything else as deprecated. +#if defined(__GNUC__) + #define ASMJIT_DEPRECATED(MESSAGE) __attribute__((__deprecated__(MESSAGE))) + #if defined(__clang__) + #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) __attribute__((__deprecated__(MESSAGE))) + #else + #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) /* not usable if a deprecated function uses it */ + #endif +#elif defined(_MSC_VER) + #define ASMJIT_DEPRECATED(MESSAGE) __declspec(deprecated(MESSAGE)) + #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) /* not usable if a deprecated function uses it */ +#else + #define ASMJIT_DEPRECATED(MESSAGE) + #define ASMJIT_DEPRECATED_STRUCT(MESSAGE) +#endif + +// Utilities. +#define ASMJIT_OFFSET_OF(STRUCT, MEMBER) ((int)(intptr_t)((const char*)&((const STRUCT*)0x100)->MEMBER) - 0x100) +#define ASMJIT_ARRAY_SIZE(X) uint32_t(sizeof(X) / sizeof(X[0])) + +#if ASMJIT_CXX_HAS_ATTRIBUTE(no_sanitize, 0) + #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF __attribute__((__no_sanitize__("undefined"))) +#elif ASMJIT_CXX_GNU >= ASMJIT_CXX_MAKE_VER(4, 9) + #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF __attribute__((__no_sanitize_undefined__)) +#else + #define ASMJIT_ATTRIBUTE_NO_SANITIZE_UNDEF +#endif + +// Begin-Namespace & End-Namespace Macros +// ====================================== + +#if defined _DOXYGEN + #define ASMJIT_BEGIN_NAMESPACE namespace asmjit { + #define ASMJIT_END_NAMESPACE } +#elif defined(__clang__) + #define ASMJIT_BEGIN_NAMESPACE \ + namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wconstant-logical-operand\"") \ + _Pragma("clang diagnostic ignored \"-Wunnamed-type-template-args\"") + #define ASMJIT_END_NAMESPACE \ + _Pragma("clang diagnostic pop") \ + }} +#elif defined(__GNUC__) && __GNUC__ == 4 + #define ASMJIT_BEGIN_NAMESPACE \ + namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") + #define ASMJIT_END_NAMESPACE \ + _Pragma("GCC diagnostic pop") \ + }} +#elif defined(__GNUC__) && __GNUC__ >= 8 + #define ASMJIT_BEGIN_NAMESPACE \ + namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wclass-memaccess\"") + #define ASMJIT_END_NAMESPACE \ + _Pragma("GCC diagnostic pop") \ + }} +#elif defined(_MSC_VER) && !defined(__INTEL_COMPILER) + #define ASMJIT_BEGIN_NAMESPACE \ + namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { \ + __pragma(warning(push)) \ + __pragma(warning(disable: 4127)) /* conditional expression is const */ \ + __pragma(warning(disable: 4201)) /* nameless struct/union */ + #define ASMJIT_END_NAMESPACE \ + __pragma(warning(pop)) \ + }} +#endif + +#if !defined(ASMJIT_BEGIN_NAMESPACE) && !defined(ASMJIT_END_NAMESPACE) + #define ASMJIT_BEGIN_NAMESPACE namespace asmjit { inline namespace ASMJIT_ABI_NAMESPACE { + #define ASMJIT_END_NAMESPACE }} +#endif + +#define ASMJIT_BEGIN_SUB_NAMESPACE(NAMESPACE) \ + ASMJIT_BEGIN_NAMESPACE \ + namespace NAMESPACE { + +#define ASMJIT_END_SUB_NAMESPACE \ + } \ + ASMJIT_END_NAMESPACE + +// C++ Utilities +// ============= + +#define ASMJIT_NONCOPYABLE(Type) \ + Type(const Type& other) = delete; \ + Type& operator=(const Type& other) = delete; + +#define ASMJIT_NONCONSTRUCTIBLE(Type) \ + Type() = delete; \ + Type(const Type& other) = delete; \ + Type& operator=(const Type& other) = delete; + +//! \def ASMJIT_DEFINE_ENUM_FLAGS(T) +//! +//! Defines bit operations for enumeration flags. +#ifdef _DOXYGEN + #define ASMJIT_DEFINE_ENUM_FLAGS(T) +#else + #define ASMJIT_DEFINE_ENUM_FLAGS(T) \ + static ASMJIT_FORCE_INLINE constexpr T operator~(T a) noexcept { \ + return T(~(std::underlying_type::type)(a)); \ + } \ + \ + static ASMJIT_FORCE_INLINE constexpr T operator|(T a, T b) noexcept { \ + return T((std::underlying_type::type)(a) | \ + (std::underlying_type::type)(b)); \ + } \ + static ASMJIT_FORCE_INLINE constexpr T operator&(T a, T b) noexcept { \ + return T((std::underlying_type::type)(a) & \ + (std::underlying_type::type)(b)); \ + } \ + static ASMJIT_FORCE_INLINE constexpr T operator^(T a, T b) noexcept { \ + return T((std::underlying_type::type)(a) ^ \ + (std::underlying_type::type)(b)); \ + } \ + \ + static ASMJIT_FORCE_INLINE T& operator|=(T& a, T b) noexcept { \ + a = T((std::underlying_type::type)(a) | \ + (std::underlying_type::type)(b)); \ + return a; \ + } \ + static ASMJIT_FORCE_INLINE T& operator&=(T& a, T b) noexcept { \ + a = T((std::underlying_type::type)(a) & \ + (std::underlying_type::type)(b)); \ + return a; \ + } \ + static ASMJIT_FORCE_INLINE T& operator^=(T& a, T b) noexcept { \ + a = T((std::underlying_type::type)(a) ^ \ + (std::underlying_type::type)(b)); \ + return a; \ + } +#endif + +//! \def ASMJIT_DEFINE_ENUM_COMPARE(T) +//! +//! Defines comparison operations for enumeration flags. +#ifdef _DOXYGEN + #define ASMJIT_DEFINE_ENUM_COMPARE(T) +#else + #define ASMJIT_DEFINE_ENUM_COMPARE(T) \ + static ASMJIT_FORCE_INLINE bool operator<(T a, T b) noexcept { \ + return (std::underlying_type::type)(a) < (std::underlying_type::type)(b); \ + } \ + static ASMJIT_FORCE_INLINE bool operator<=(T a, T b) noexcept { \ + return (std::underlying_type::type)(a) <= (std::underlying_type::type)(b); \ + } \ + static ASMJIT_FORCE_INLINE bool operator>(T a, T b) noexcept { \ + return (std::underlying_type::type)(a) > (std::underlying_type::type)(b); \ + } \ + static ASMJIT_FORCE_INLINE bool operator>=(T a, T b) noexcept { \ + return (std::underlying_type::type)(a) >= (std::underlying_type::type)(b); \ + } +#endif + +// Cleanup Api-Config Specific Macros +// ================================== + +#undef ASMJIT_CXX_GNU +#undef ASMJIT_CXX_MAKE_VER + +#endif // ASMJIT_CORE_API_CONFIG_H_INCLUDED diff --git a/src/asmjit/core/archcommons.h b/src/asmjit/core/archcommons.h new file mode 100644 index 0000000..e9d2c84 --- /dev/null +++ b/src/asmjit/core/archcommons.h @@ -0,0 +1,229 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED +#define ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED + +// This file provides architecture-specific classes that are required in the core library. For example Imm operand +// allows to be created from arm::Shift in a const-expr way, so the arm::Shift must be provided. So this header file +// provides everything architecture-specific that is used by the Core API. + +#include "../core/globals.h" + +ASMJIT_BEGIN_SUB_NAMESPACE(arm) + +//! \addtogroup asmjit_arm +//! \{ + +//! Condition code (both AArch32 & AArch64). +//! +//! \note This enumeration doesn't match condition code that is used in AArch32/AArch64 opcodes. In general this +//! condition code is encoded as `(cc - 2) & 0xF` so that `kAL` condition code is zero and encoded as 0xE in opcode. +//! This makes it easier to use a condition code as an instruction modifier that defaults to 'al'. +enum class CondCode : uint8_t { + kAL = 0x00u, //!< (no condition code) (always) + kNA = 0x01u, //!< (not available) (special) + kEQ = 0x02u, //!< Z==1 (any_sign ==) + kNE = 0x03u, //!< Z==0 (any_sign !=) + kCS = 0x04u, //!< C==1 (unsigned >=) + kHS = 0x04u, //!< C==1 (unsigned >=) + kCC = 0x05u, //!< C==0 (unsigned < ) + kLO = 0x05u, //!< C==0 (unsigned < ) + kMI = 0x06u, //!< N==1 (is negative) + kPL = 0x07u, //!< N==0 (is positive or zero) + kVS = 0x08u, //!< V==1 (is overflow) + kVC = 0x09u, //!< V==0 (no overflow) + kHI = 0x0Au, //!< C==1 & Z==0 (unsigned > ) + kLS = 0x0Bu, //!< C==0 | Z==1 (unsigned <=) + kGE = 0x0Cu, //!< N==V (signed >=) + kLT = 0x0Du, //!< N!=V (signed < ) + kGT = 0x0Eu, //!< Z==0 & N==V (signed > ) + kLE = 0x0Fu, //!< Z==1 | N!=V (signed <=) + + kSign = kMI, //!< Sign. + kNotSign = kPL, //!< Not sign. + + kOverflow = kVS, //!< Signed overflow. + kNotOverflow = kVC, //!< Not signed overflow. + + kEqual = kEQ, //!< Equal `a == b`. + kNotEqual = kNE, //!< Not Equal `a != b`. + + kZero = kEQ, //!< Zero (alias to equal). + kNotZero = kNE, //!< Not Zero (alias to Not Equal). + + kNegative = kMI, //!< Negative. + kPositive = kPL, //!< Positive or zero. + + kSignedLT = kLT, //!< Signed `a < b`. + kSignedLE = kLE, //!< Signed `a <= b`. + kSignedGT = kGT, //!< Signed `a > b`. + kSignedGE = kGE, //!< Signed `a >= b`. + + kUnsignedLT = kLO, //!< Unsigned `a < b`. + kUnsignedLE = kLS, //!< Unsigned `a <= b`. + kUnsignedGT = kHI, //!< Unsigned `a > b`. + kUnsignedGE = kHS, //!< Unsigned `a >= b`. + + kAlways = kAL, //!< No condition code (always). + + kMaxValue = 0x0Fu //!< Maximum value of `CondCode`. +}; + +//! Negates a condition code. +static inline constexpr CondCode negateCond(CondCode cond) noexcept { return CondCode(uint8_t(cond) ^ uint8_t(1)); } + +//! Data type that can be encoded with the instruction (AArch32 only). +enum class DataType : uint32_t { + //! No data type specified (default for all general purpose instructions). + kNone = 0, + //! 8-bit signed integer, specified as `.s8` in assembly. + kS8 = 1, + //! 16-bit signed integer, specified as `.s16` in assembly. + kS16 = 2, + //! 32-bit signed integer, specified as `.s32` in assembly. + kS32 = 3, + //! 64-bit signed integer, specified as `.s64` in assembly. + kS64 = 4, + //! 8-bit unsigned integer, specified as `.u8` in assembly. + kU8 = 5, + //! 16-bit unsigned integer, specified as `.u16` in assembly. + kU16 = 6, + //! 32-bit unsigned integer, specified as `.u32` in assembly. + kU32 = 7, + //! 64-bit unsigned integer, specified as `.u64` in assembly. + kU64 = 8, + //! 16-bit floating point (half precision), specified as `.f16` in assembly. + kF16 = 10, + //! 32-bit floating point (single precision), specified as `.f32` in assembly. + kF32 = 11, + //! 64-bit floating point (double precision), specified as `.f64` in assembly. + kF64 = 12, + //! 8-bit polynomial. + kP8 = 13, + //! 64-bit polynomial. + kP64 = 15, + + //! Maximum value of `DataType`. + kMaxValue = 15 +}; + +//! Shift operation predicate (ARM) describes either SHIFT or EXTEND operation. +//! +//! \note The constants are AsmJit specific. The first 5 values describe real constants on ARM32 and AArch64 hardware, +//! however, the addition constants that describe extend modes are specific to AsmJit and would be translated to the +//! AArch64 specific constants by the assembler. +enum class ShiftOp : uint32_t { + //! Shift left logical operation (default). + //! + //! Available to all ARM architectures. + kLSL = 0x00u, + + //! Shift right logical operation. + //! + //! Available to all ARM architectures. + kLSR = 0x01u, + + //! Shift right arithmetic operation. + //! + //! Available to all ARM architectures. + kASR = 0x02u, + + //! Rotate right operation (AArch32 only). + kROR = 0x03u, + + //! Rotate right with carry operation (encoded as `ShiftOp::kROR` with zero) (AArch32 only). + kRRX = 0x04u, + + //! Shift left by filling low order bits with ones. + kMSL = 0x05u, + + //! UXTN extend register operation (AArch64 only). + kUXTB = 0x06u, + //! UXTH extend register operation (AArch64 only). + kUXTH = 0x07u, + //! UXTW extend register operation (AArch64 only). + kUXTW = 0x08u, + //! UXTX extend register operation (AArch64 only). + kUXTX = 0x09u, + + //! SXTB extend register operation (AArch64 only). + kSXTB = 0x0Au, + //! SXTH extend register operation (AArch64 only). + kSXTH = 0x0Bu, + //! SXTW extend register operation (AArch64 only). + kSXTW = 0x0Cu, + //! SXTX extend register operation (AArch64 only). + kSXTX = 0x0Du + + // NOTE: 0xE and 0xF are used by memory operand to specify POST|PRE offset mode. +}; + +//! Represents ARM immediate shift operation type and value. +class Shift { +public: + //! Shift operation. + ShiftOp _op; + //! Shift Value. + uint32_t _value; + + //! Default constructed Shift is not initialized. + inline Shift() noexcept = default; + + //! Copy constructor (default) + constexpr Shift(const Shift& other) noexcept = default; + + //! Constructs Shift from operation `op` and shift `value`. + constexpr Shift(ShiftOp op, uint32_t value) noexcept + : _op(op), + _value(value) {} + + //! Returns the shift operation. + constexpr ShiftOp op() const noexcept { return _op; } + //! Sets shift operation to `op`. + inline void setOp(ShiftOp op) noexcept { _op = op; } + + //! Returns the shift smount. + constexpr uint32_t value() const noexcept { return _value; } + //! Sets shift amount to `value`. + inline void setValue(uint32_t value) noexcept { _value = value; } +}; + +//! Constructs a `LSL #value` shift (logical shift left). +static constexpr Shift lsl(uint32_t value) noexcept { return Shift(ShiftOp::kLSL, value); } +//! Constructs a `LSR #value` shift (logical shift right). +static constexpr Shift lsr(uint32_t value) noexcept { return Shift(ShiftOp::kLSR, value); } +//! Constructs a `ASR #value` shift (arithmetic shift right). +static constexpr Shift asr(uint32_t value) noexcept { return Shift(ShiftOp::kASR, value); } +//! Constructs a `ROR #value` shift (rotate right). +static constexpr Shift ror(uint32_t value) noexcept { return Shift(ShiftOp::kROR, value); } +//! Constructs a `RRX` shift (rotate with carry by 1). +static constexpr Shift rrx() noexcept { return Shift(ShiftOp::kRRX, 0); } +//! Constructs a `MSL #value` shift (logical shift left filling ones). +static constexpr Shift msl(uint32_t value) noexcept { return Shift(ShiftOp::kMSL, value); } + +//! Constructs a `UXTB #value` extend and shift (unsigned byte extend). +static constexpr Shift uxtb(uint32_t value) noexcept { return Shift(ShiftOp::kUXTB, value); } +//! Constructs a `UXTH #value` extend and shift (unsigned hword extend). +static constexpr Shift uxth(uint32_t value) noexcept { return Shift(ShiftOp::kUXTH, value); } +//! Constructs a `UXTW #value` extend and shift (unsigned word extend). +static constexpr Shift uxtw(uint32_t value) noexcept { return Shift(ShiftOp::kUXTW, value); } +//! Constructs a `UXTX #value` extend and shift (unsigned dword extend). +static constexpr Shift uxtx(uint32_t value) noexcept { return Shift(ShiftOp::kUXTX, value); } + +//! Constructs a `SXTB #value` extend and shift (signed byte extend). +static constexpr Shift sxtb(uint32_t value) noexcept { return Shift(ShiftOp::kSXTB, value); } +//! Constructs a `SXTH #value` extend and shift (signed hword extend). +static constexpr Shift sxth(uint32_t value) noexcept { return Shift(ShiftOp::kSXTH, value); } +//! Constructs a `SXTW #value` extend and shift (signed word extend). +static constexpr Shift sxtw(uint32_t value) noexcept { return Shift(ShiftOp::kSXTW, value); } +//! Constructs a `SXTX #value` extend and shift (signed dword extend). +static constexpr Shift sxtx(uint32_t value) noexcept { return Shift(ShiftOp::kSXTX, value); } + +//! \} + +ASMJIT_END_SUB_NAMESPACE + +#endif // ASMJIT_CORE_ARCHCOMMONS_H_INCLUDED diff --git a/src/asmjit/core/archtraits.cpp b/src/asmjit/core/archtraits.cpp new file mode 100644 index 0000000..fc825df --- /dev/null +++ b/src/asmjit/core/archtraits.cpp @@ -0,0 +1,160 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/archtraits.h" +#include "../core/misc_p.h" + +#if !defined(ASMJIT_NO_X86) + #include "../x86/x86archtraits_p.h" +#endif + +#if !defined(ASMJIT_NO_AARCH64) + #include "../arm/a64archtraits_p.h" +#endif + +ASMJIT_BEGIN_NAMESPACE + +static const constexpr ArchTraits noArchTraits = { + // SP/FP/LR/PC. + 0xFF, 0xFF, 0xFF, 0xFF, + + // Reserved, + { 0, 0, 0 }, + + // HW stack alignment. + 0, + + // Min/Max stack offset. + 0, 0, + + // ISA features [Gp, Vec, Other0, Other1]. + {{ + InstHints::kNoHints, + InstHints::kNoHints, + InstHints::kNoHints, + InstHints::kNoHints + }}, + + // RegTypeToSignature. + #define V(index) OperandSignature{0} + {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, + #undef V + + // RegTypeToTypeId. + #define V(index) TypeId::kVoid + {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, + #undef V + + // TypeIdToRegType. + #define V(index) RegType::kNone + {{ ASMJIT_LOOKUP_TABLE_32(V, 0) }}, + #undef V + + // Word names of 8-bit, 16-bit, 32-bit, and 64-bit quantities. + { + ArchTypeNameId::kByte, + ArchTypeNameId::kHalf, + ArchTypeNameId::kWord, + ArchTypeNameId::kQuad + } +}; + +ASMJIT_VARAPI const ArchTraits _archTraits[uint32_t(Arch::kMaxValue) + 1] = { + // No architecture. + noArchTraits, + + // X86/X86 architectures. +#if !defined(ASMJIT_NO_X86) + x86::x86ArchTraits, + x86::x64ArchTraits, +#else + noArchTraits, + noArchTraits, +#endif + + // RISCV32/RISCV64 architectures. + noArchTraits, + noArchTraits, + + // ARM architecture + noArchTraits, + + // AArch64 architecture. +#if !defined(ASMJIT_NO_AARCH64) + a64::a64ArchTraits, +#else + noArchTraits, +#endif + + // ARM/Thumb architecture. + noArchTraits, + + // Reserved. + noArchTraits, + + // MIPS32/MIPS64 + noArchTraits, + noArchTraits +}; + +ASMJIT_FAVOR_SIZE Error ArchUtils::typeIdToRegSignature(Arch arch, TypeId typeId, TypeId* typeIdOut, OperandSignature* regSignatureOut) noexcept { + const ArchTraits& archTraits = ArchTraits::byArch(arch); + + // TODO: Remove this, should never be used like this. + // Passed RegType instead of TypeId? + if (uint32_t(typeId) <= uint32_t(RegType::kMaxValue)) + typeId = archTraits.regTypeToTypeId(RegType(uint32_t(typeId))); + + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(typeId))) + return DebugUtils::errored(kErrorInvalidTypeId); + + // First normalize architecture dependent types. + if (TypeUtils::isAbstract(typeId)) { + bool is32Bit = Environment::is32Bit(arch); + if (typeId == TypeId::kIntPtr) + typeId = is32Bit ? TypeId::kInt32 : TypeId::kInt64; + else + typeId = is32Bit ? TypeId::kUInt32 : TypeId::kUInt64; + } + + // Type size helps to construct all groups of registers. + // TypeId is invalid if the size is zero. + uint32_t size = TypeUtils::sizeOf(typeId); + if (ASMJIT_UNLIKELY(!size)) + return DebugUtils::errored(kErrorInvalidTypeId); + + if (ASMJIT_UNLIKELY(typeId == TypeId::kFloat80)) + return DebugUtils::errored(kErrorInvalidUseOfF80); + + RegType regType = RegType::kNone; + if (TypeUtils::isBetween(typeId, TypeId::_kBaseStart, TypeId::_kVec32Start)) { + regType = archTraits._typeIdToRegType[uint32_t(typeId) - uint32_t(TypeId::_kBaseStart)]; + if (regType == RegType::kNone) { + if (typeId == TypeId::kInt64 || typeId == TypeId::kUInt64) + return DebugUtils::errored(kErrorInvalidUseOfGpq); + else + return DebugUtils::errored(kErrorInvalidTypeId); + } + } + else { + if (size <= 8 && archTraits._regSignature[RegType::kVec64].isValid()) + regType = RegType::kVec64; + else if (size <= 16 && archTraits._regSignature[RegType::kVec128].isValid()) + regType = RegType::kVec128; + else if (size == 32 && archTraits._regSignature[RegType::kVec256].isValid()) + regType = RegType::kVec256; + else if (archTraits._regSignature[RegType::kVec512].isValid()) + regType = RegType::kVec512; + else + return DebugUtils::errored(kErrorInvalidTypeId); + } + + *typeIdOut = typeId; + *regSignatureOut = archTraits.regTypeToSignature(regType); + return kErrorOk; +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/archtraits.h b/src/asmjit/core/archtraits.h new file mode 100644 index 0000000..192a826 --- /dev/null +++ b/src/asmjit/core/archtraits.h @@ -0,0 +1,290 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_ARCHTRAITS_H_INCLUDED +#define ASMJIT_CORE_ARCHTRAITS_H_INCLUDED + +#include "../core/operand.h" +#include "../core/support.h" +#include "../core/type.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! Instruction set architecture (ISA). +enum class Arch : uint8_t { + //! Unknown or uninitialized ISA. + kUnknown = 0, + + //! 32-bit X86 ISA. + kX86 = 1, + //! 64-bit X86 ISA also known as X64, X86_64, and AMD64. + kX64 = 2, + + //! 32-bit RISC-V ISA. + kRISCV32 = 3, + //! 64-bit RISC-V ISA. + kRISCV64 = 4, + + //! 32-bit ARM ISA (little endian). + kARM = 5, + //! 64-bit ARM ISA in (little endian). + kAArch64 = 6, + //! 32-bit ARM ISA in Thumb mode (little endian). + kThumb = 7, + + // 8 is not used at the moment, even numbers are 64-bit architectures. + + //! 32-bit MIPS ISA in (little endian). + kMIPS32_LE = 9, + //! 64-bit MIPS ISA in (little endian). + kMIPS64_LE = 10, + + //! 32-bit ARM ISA (big endian). + kARM_BE = 11, + //! 64-bit ARM ISA in (big endian). + kAArch64_BE = 12, + //! 32-bit ARM ISA in Thumb mode (big endian). + kThumb_BE = 13, + + // 14 is not used at the moment, even numbers are 64-bit architectures. + + //! 32-bit MIPS ISA in (big endian). + kMIPS32_BE = 15, + //! 64-bit MIPS ISA in (big endian). + kMIPS64_BE = 16, + + //! Maximum value of `Arch`. + kMaxValue = kMIPS64_BE, + + //! Mask used by 32-bit ISAs (odd are 32-bit, even are 64-bit). + k32BitMask = 0x01, + //! First big-endian architecture. + kBigEndian = kARM_BE, + + //! ISA detected at compile-time (ISA of the host). + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#else + ASMJIT_ARCH_X86 == 32 ? kX86 : + ASMJIT_ARCH_X86 == 64 ? kX64 : + + ASMJIT_ARCH_ARM == 32 && ASMJIT_ARCH_LE ? kARM : + ASMJIT_ARCH_ARM == 32 && ASMJIT_ARCH_BE ? kARM_BE : + ASMJIT_ARCH_ARM == 64 && ASMJIT_ARCH_LE ? kAArch64 : + ASMJIT_ARCH_ARM == 64 && ASMJIT_ARCH_BE ? kAArch64_BE : + + ASMJIT_ARCH_MIPS == 32 && ASMJIT_ARCH_LE ? kMIPS32_LE : + ASMJIT_ARCH_MIPS == 32 && ASMJIT_ARCH_BE ? kMIPS32_BE : + ASMJIT_ARCH_MIPS == 64 && ASMJIT_ARCH_LE ? kMIPS64_LE : + ASMJIT_ARCH_MIPS == 64 && ASMJIT_ARCH_BE ? kMIPS64_BE : + + kUnknown +#endif +}; + +//! Sub-architecture. +enum class SubArch : uint8_t { + //! Unknown or uninitialized architecture sub-type. + kUnknown = 0, + + //! Maximum value of `SubArch`. + kMaxValue = kUnknown, + + //! Sub-architecture detected at compile-time (sub-architecture of the host). + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#else + kUnknown +#endif +}; + +//! Identifier used to represent names of different data types across architectures. +enum class ArchTypeNameId : uint8_t { + //! Describes 'db' (X86/X86_64 convention, always 8-bit quantity). + kDB = 0, + //! Describes 'dw' (X86/X86_64 convention, always 16-bit word). + kDW, + //! Describes 'dd' (X86/X86_64 convention, always 32-bit word). + kDD, + //! Describes 'dq' (X86/X86_64 convention, always 64-bit word). + kDQ, + //! Describes 'byte' (always 8-bit quantity). + kByte, + //! Describes 'half' (most likely 16-bit word). + kHalf, + //! Describes 'word' (either 16-bit or 32-bit word). + kWord, + //! Describes 'hword' (most likely 16-bit word). + kHWord, + //! Describes 'dword' (either 32-bit or 64-bit word). + kDWord, + //! Describes 'qword' (64-bit word). + kQWord, + //! Describes 'xword' (64-bit word). + kXWord, + //! Describes 'short' (always 16-bit word). + kShort, + //! Describes 'long' (most likely 32-bit word). + kLong, + //! Describes 'quad' (64-bit word). + kQuad, + + //! Maximum value of `ArchTypeNameId`. + kMaxValue = kQuad +}; + +//! Instruction feature hints for each register group provided by \ref ArchTraits. +//! +//! Instruction feature hints describe miscellaneous instructions provided by the architecture that can be used by +//! register allocator to make certain things simpler - like register swaps or emitting register push/pop sequences. +//! +//! \remarks Instruction feature hints are only defined for register groups that can be used with \ref +//! asmjit_compiler infrastructure. Register groups that are not managed by Compiler are not provided by +//! \ref ArchTraits and cannot be queried. +enum class InstHints : uint8_t { + //! No feature hints. + kNoHints = 0, + + //! Architecture supports a register swap by using a single instruction. + kRegSwap = 0x01u, + //! Architecture provides push/pop instructions. + kPushPop = 0x02u +}; +ASMJIT_DEFINE_ENUM_FLAGS(InstHints) + +//! Architecture traits used by Function API and Compiler's register allocator. +struct ArchTraits { + //! \name Members + //! \{ + + //! Stack pointer register id. + uint8_t _spRegId; + //! Frame pointer register id. + uint8_t _fpRegId; + //! Link register id. + uint8_t _linkRegId; + //! Instruction pointer (or program counter) register id, if accessible. + uint8_t _ipRegId; + + // Reserved. + uint8_t _reserved[3]; + //! Hardware stack alignment requirement. + uint8_t _hwStackAlignment; + + //! Minimum addressable offset on stack guaranteed for all instructions. + uint32_t _minStackOffset; + //! Maximum addressable offset on stack depending on specific instruction. + uint32_t _maxStackOffset; + + //! Flags for each virtual register group. + Support::Array _instHints; + + //! Maps register type into a signature, that provides group, size and can be used to construct register operands. + Support::Array _regSignature; + //! Maps a register to type-id, see \ref TypeId. + Support::Array _regTypeToTypeId; + //! Maps scalar TypeId values (from TypeId::_kIdBaseStart) to register types, see \ref TypeId. + Support::Array _typeIdToRegType; + + //! Word name identifiers of 8-bit, 16-bit, 32-biit, and 64-bit quantities that appear in formatted text. + ArchTypeNameId _typeNameIdTable[4]; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns stack pointer register id. + inline uint32_t spRegId() const noexcept { return _spRegId; } + //! Returns stack frame register id. + inline uint32_t fpRegId() const noexcept { return _fpRegId; } + //! Returns link register id, if the architecture provides it. + inline uint32_t linkRegId() const noexcept { return _linkRegId; } + //! Returns instruction pointer register id, if the architecture provides it. + inline uint32_t ipRegId() const noexcept { return _ipRegId; } + + //! Returns a hardware stack alignment requirement. + //! + //! \note This is a hardware constraint. Architectures that don't constrain it would return the lowest alignment + //! (1), however, some architectures may constrain the alignment, for example AArch64 requires 16-byte alignment. + inline uint32_t hwStackAlignment() const noexcept { return _hwStackAlignment; } + + //! Tests whether the architecture provides link register, which is used across function calls. If the link + //! register is not provided then a function call pushes the return address on stack (X86/X64). + inline bool hasLinkReg() const noexcept { return _linkRegId != BaseReg::kIdBad; } + + //! Returns minimum addressable offset on stack guaranteed for all instructions. + inline uint32_t minStackOffset() const noexcept { return _minStackOffset; } + //! Returns maximum addressable offset on stack depending on specific instruction. + inline uint32_t maxStackOffset() const noexcept { return _maxStackOffset; } + + //! Returns ISA flags of the given register `group`. + inline InstHints instFeatureHints(RegGroup group) const noexcept { return _instHints[group]; } + //! Tests whether the given register `group` has the given `flag` set. + inline bool hasInstHint(RegGroup group, InstHints feature) const noexcept { return Support::test(_instHints[group], feature); } + //! Tests whether the ISA provides register swap instruction for the given register `group`. + inline bool hasInstRegSwap(RegGroup group) const noexcept { return hasInstHint(group, InstHints::kRegSwap); } + //! Tests whether the ISA provides push/pop instructions for the given register `group`. + inline bool hasInstPushPop(RegGroup group) const noexcept { return hasInstHint(group, InstHints::kPushPop); } + + inline bool hasRegType(RegType type) const noexcept { + return type <= RegType::kMaxValue && _regSignature[type].isValid(); + } + + //! Returns an operand signature from the given register `type` of this architecture. + inline OperandSignature regTypeToSignature(RegType type) const noexcept { return _regSignature[type]; } + //! Returns a register from the given register `type` of this architecture. + inline RegGroup regTypeToGroup(RegType type) const noexcept { return _regSignature[type].regGroup(); } + //! Returns a register size the given register `type` of this architecture. + inline uint32_t regTypeToSize(RegType type) const noexcept { return _regSignature[type].size(); } + //! Returns a corresponding `TypeId` from the given register `type` of this architecture. + inline TypeId regTypeToTypeId(RegType type) const noexcept { return _regTypeToTypeId[type]; } + + //! Returns a table of ISA word names that appear in formatted text. Word names are ISA dependent. + //! + //! The index of this table is log2 of the size: + //! - [0] 8-bits + //! - [1] 16-bits + //! - [2] 32-bits + //! - [3] 64-bits + inline const ArchTypeNameId* typeNameIdTable() const noexcept { return _typeNameIdTable; } + + //! Returns an ISA word name identifier of the given `index`, see \ref typeNameIdTable() for more details. + inline ArchTypeNameId typeNameIdByIndex(uint32_t index) const noexcept { return _typeNameIdTable[index]; } + + //! \} + + //! \name Statics + //! \{ + + //! Returns a const reference to `ArchTraits` for the given architecture `arch`. + static inline const ArchTraits& byArch(Arch arch) noexcept; + + //! \} +}; + +ASMJIT_VARAPI const ArchTraits _archTraits[uint32_t(Arch::kMaxValue) + 1]; + +//! \cond +inline const ArchTraits& ArchTraits::byArch(Arch arch) noexcept { return _archTraits[uint32_t(arch)]; } +//! \endcond + +//! Architecture utilities. +namespace ArchUtils { + +ASMJIT_API Error typeIdToRegSignature(Arch arch, TypeId typeId, TypeId* typeIdOut, OperandSignature* regSignatureOut) noexcept; + +} // {ArchUtils} + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_ARCHTRAITS_H_INCLUDED diff --git a/src/asmjit/core/assembler.cpp b/src/asmjit/core/assembler.cpp new file mode 100644 index 0000000..d6c8762 --- /dev/null +++ b/src/asmjit/core/assembler.cpp @@ -0,0 +1,406 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/assembler.h" +#include "../core/codewriter_p.h" +#include "../core/constpool.h" +#include "../core/emitterutils_p.h" +#include "../core/formatter.h" +#include "../core/logger.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// BaseAssembler - Construction & Destruction +// ========================================== + +BaseAssembler::BaseAssembler() noexcept + : BaseEmitter(EmitterType::kAssembler) {} + +BaseAssembler::~BaseAssembler() noexcept {} + +// BaseAssembler - Buffer Management +// ================================= + +Error BaseAssembler::setOffset(size_t offset) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + size_t size = Support::max(_section->bufferSize(), this->offset()); + if (ASMJIT_UNLIKELY(offset > size)) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + _bufferPtr = _bufferData + offset; + return kErrorOk; +} + +// BaseAssembler - Section Management +// ================================== + +static void BaseAssembler_initSection(BaseAssembler* self, Section* section) noexcept { + uint8_t* p = section->_buffer._data; + + self->_section = section; + self->_bufferData = p; + self->_bufferPtr = p + section->_buffer._size; + self->_bufferEnd = p + section->_buffer._capacity; +} + +Error BaseAssembler::section(Section* section) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + if (!_code->isSectionValid(section->id()) || _code->_sections[section->id()] != section) + return reportError(DebugUtils::errored(kErrorInvalidSection)); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) + _logger->logf(".section %s {#%u}\n", section->name(), section->id()); +#endif + + BaseAssembler_initSection(this, section); + return kErrorOk; +} + +// BaseAssembler - Label Management +// ================================ + +Label BaseAssembler::newLabel() { + uint32_t labelId = Globals::kInvalidId; + if (ASMJIT_LIKELY(_code)) { + LabelEntry* le; + Error err = _code->newLabelEntry(&le); + if (ASMJIT_UNLIKELY(err)) + reportError(err); + else + labelId = le->id(); + } + return Label(labelId); +} + +Label BaseAssembler::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { + uint32_t labelId = Globals::kInvalidId; + if (ASMJIT_LIKELY(_code)) { + LabelEntry* le; + Error err = _code->newNamedLabelEntry(&le, name, nameSize, type, parentId); + if (ASMJIT_UNLIKELY(err)) + reportError(err); + else + labelId = le->id(); + } + return Label(labelId); +} + +Error BaseAssembler::bind(const Label& label) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + Error err = _code->bindLabel(label, _section->id(), offset()); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) + EmitterUtils::logLabelBound(this, label); +#endif + + resetInlineComment(); + if (err) + return reportError(err); + + return kErrorOk; +} + +// BaseAssembler - Embed +// ===================== + +Error BaseAssembler::embed(const void* data, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + if (dataSize == 0) + return kErrorOk; + + CodeWriter writer(this); + ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); + + writer.emitData(data, dataSize); + writer.done(this); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) { + StringTmp<512> sb; + Formatter::formatData(sb, _logger->flags(), arch(), TypeId::kUInt8, data, dataSize, 1); + sb.append('\n'); + _logger->log(sb); + } +#endif + + return kErrorOk; +} + +Error BaseAssembler::embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t repeatCount) { + uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize()); + TypeId finalTypeId = TypeUtils::deabstract(typeId, deabstractDelta); + + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + if (itemCount == 0 || repeatCount == 0) + return kErrorOk; + + uint32_t typeSize = TypeUtils::sizeOf(finalTypeId); + Support::FastUInt8 of = 0; + + size_t dataSize = Support::mulOverflow(itemCount, size_t(typeSize), &of); + size_t totalSize = Support::mulOverflow(dataSize, repeatCount, &of); + + if (ASMJIT_UNLIKELY(of)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + CodeWriter writer(this); + ASMJIT_PROPAGATE(writer.ensureSpace(this, totalSize)); + + for (size_t i = 0; i < repeatCount; i++) + writer.emitData(data, dataSize); + + writer.done(this); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) { + StringTmp<512> sb; + Formatter::formatData(sb, _logger->flags(), arch(), typeId, data, itemCount, repeatCount); + sb.append('\n'); + _logger->log(sb); + } +#endif + + return kErrorOk; +} + +#ifndef ASMJIT_NO_LOGGING +static const TypeId dataTypeIdBySize[9] = { + TypeId::kVoid, // [0] (invalid) + TypeId::kUInt8, // [1] (uint8_t) + TypeId::kUInt16, // [2] (uint16_t) + TypeId::kVoid, // [3] (invalid) + TypeId::kUInt32, // [4] (uint32_t) + TypeId::kVoid, // [5] (invalid) + TypeId::kVoid, // [6] (invalid) + TypeId::kVoid, // [7] (invalid) + TypeId::kUInt64 // [8] (uint64_t) +}; +#endif + +Error BaseAssembler::embedConstPool(const Label& label, const ConstPool& pool) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + if (ASMJIT_UNLIKELY(!isLabelValid(label))) + return reportError(DebugUtils::errored(kErrorInvalidLabel)); + + ASMJIT_PROPAGATE(align(AlignMode::kData, uint32_t(pool.alignment()))); + ASMJIT_PROPAGATE(bind(label)); + + size_t size = pool.size(); + if (!size) + return kErrorOk; + + CodeWriter writer(this); + ASMJIT_PROPAGATE(writer.ensureSpace(this, size)); + +#ifndef ASMJIT_NO_LOGGING + uint8_t* data = writer.cursor(); +#endif + + pool.fill(writer.cursor()); + writer.advance(size); + writer.done(this); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) { + uint32_t dataSizeLog2 = Support::min(Support::ctz(pool.minItemSize()), 3); + uint32_t dataSize = 1 << dataSizeLog2; + + StringTmp<512> sb; + Formatter::formatData(sb, _logger->flags(), arch(), dataTypeIdBySize[dataSize], data, size >> dataSizeLog2); + sb.append('\n'); + _logger->log(sb); + } +#endif + + return kErrorOk; +} + +Error BaseAssembler::embedLabel(const Label& label, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + ASMJIT_ASSERT(_code != nullptr); + RelocEntry* re; + LabelEntry* le = _code->labelEntry(label); + + if (ASMJIT_UNLIKELY(!le)) + return reportError(DebugUtils::errored(kErrorInvalidLabel)); + + if (dataSize == 0) + dataSize = registerSize(); + + if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) + return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); + + CodeWriter writer(this); + ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) { + StringTmp<256> sb; + sb.append('.'); + Formatter::formatDataType(sb, _logger->flags(), arch(), dataTypeIdBySize[dataSize]); + sb.append(' '); + Formatter::formatLabel(sb, FormatFlags::kNone, this, label.id()); + sb.append('\n'); + _logger->log(sb); + } +#endif + + Error err = _code->newRelocEntry(&re, RelocType::kRelToAbs); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + re->_sourceSectionId = _section->id(); + re->_sourceOffset = offset(); + re->_format.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); + + if (le->isBound()) { + re->_targetSectionId = le->section()->id(); + re->_payload = le->offset(); + } + else { + OffsetFormat of; + of.resetToSimpleValue(OffsetType::kUnsignedOffset, dataSize); + + LabelLink* link = _code->newLabelLink(le, _section->id(), offset(), 0, of); + if (ASMJIT_UNLIKELY(!link)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + link->relocId = re->id(); + } + + // Emit dummy DWORD/QWORD depending on the data size. + writer.emitZeros(dataSize); + writer.done(this); + + return kErrorOk; +} + +Error BaseAssembler::embedLabelDelta(const Label& label, const Label& base, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + + LabelEntry* labelEntry = _code->labelEntry(label); + LabelEntry* baseEntry = _code->labelEntry(base); + + if (ASMJIT_UNLIKELY(!labelEntry || !baseEntry)) + return reportError(DebugUtils::errored(kErrorInvalidLabel)); + + if (dataSize == 0) + dataSize = registerSize(); + + if (ASMJIT_UNLIKELY(!Support::isPowerOf2(dataSize) || dataSize > 8)) + return reportError(DebugUtils::errored(kErrorInvalidOperandSize)); + + CodeWriter writer(this); + ASMJIT_PROPAGATE(writer.ensureSpace(this, dataSize)); + +#ifndef ASMJIT_NO_LOGGING + if (_logger) { + StringTmp<256> sb; + sb.append('.'); + Formatter::formatDataType(sb, _logger->flags(), arch(), dataTypeIdBySize[dataSize]); + sb.append(" ("); + Formatter::formatLabel(sb, FormatFlags::kNone, this, label.id()); + sb.append(" - "); + Formatter::formatLabel(sb, FormatFlags::kNone, this, base.id()); + sb.append(")\n"); + _logger->log(sb); + } +#endif + + // If both labels are bound within the same section it means the delta can be calculated now. + if (labelEntry->isBound() && baseEntry->isBound() && labelEntry->section() == baseEntry->section()) { + uint64_t delta = labelEntry->offset() - baseEntry->offset(); + writer.emitValueLE(delta, dataSize); + } + else { + RelocEntry* re; + Error err = _code->newRelocEntry(&re, RelocType::kExpression); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + Expression* exp = _code->_zone.newT(); + if (ASMJIT_UNLIKELY(!exp)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + exp->reset(); + exp->opType = ExpressionOpType::kSub; + exp->setValueAsLabel(0, labelEntry); + exp->setValueAsLabel(1, baseEntry); + + re->_format.resetToSimpleValue(OffsetType::kSignedOffset, dataSize); + re->_sourceSectionId = _section->id(); + re->_sourceOffset = offset(); + re->_payload = (uint64_t)(uintptr_t)exp; + + writer.emitZeros(dataSize); + } + + writer.done(this); + return kErrorOk; +} + +// BaseAssembler - Comment +// ======================= + +Error BaseAssembler::comment(const char* data, size_t size) { + if (!hasEmitterFlag(EmitterFlags::kLogComments)) { + if (!hasEmitterFlag(EmitterFlags::kAttached)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + return kErrorOk; + } + +#ifndef ASMJIT_NO_LOGGING + // Logger cannot be NULL if `EmitterFlags::kLogComments` is set. + ASMJIT_ASSERT(_logger != nullptr); + + _logger->log(data, size); + _logger->log("\n", 1); + return kErrorOk; +#else + DebugUtils::unused(data, size); + return kErrorOk; +#endif +} + +// BaseAssembler - Events +// ====================== + +Error BaseAssembler::onAttach(CodeHolder* code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + // Attach to the end of the .text section. + BaseAssembler_initSection(this, code->_sections[0]); + + return kErrorOk; +} + +Error BaseAssembler::onDetach(CodeHolder* code) noexcept { + _section = nullptr; + _bufferData = nullptr; + _bufferEnd = nullptr; + _bufferPtr = nullptr; + return Base::onDetach(code); +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/assembler.h b/src/asmjit/core/assembler.h new file mode 100644 index 0000000..7ea2505 --- /dev/null +++ b/src/asmjit/core/assembler.h @@ -0,0 +1,129 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_ASSEMBLER_H_INCLUDED +#define ASMJIT_CORE_ASSEMBLER_H_INCLUDED + +#include "../core/codeholder.h" +#include "../core/emitter.h" +#include "../core/operand.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_assembler +//! \{ + +//! Base assembler. +//! +//! This is a base class that provides interface used by architecture specific +//! assembler implementations. Assembler doesn't hold any data, instead it's +//! attached to \ref CodeHolder, which provides all the data that Assembler +//! needs and which can be altered by it. +//! +//! Check out architecture specific assemblers for more details and examples: +//! +//! - \ref x86::Assembler - X86/X64 assembler implementation. +class ASMJIT_VIRTAPI BaseAssembler : public BaseEmitter { +public: + ASMJIT_NONCOPYABLE(BaseAssembler) + typedef BaseEmitter Base; + + //! Current section where the assembling happens. + Section* _section = nullptr; + //! Start of the CodeBuffer of the current section. + uint8_t* _bufferData = nullptr; + //! End (first invalid byte) of the current section. + uint8_t* _bufferEnd = nullptr; + //! Pointer in the CodeBuffer of the current section. + uint8_t* _bufferPtr = nullptr; + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `BaseAssembler` instance. + ASMJIT_API BaseAssembler() noexcept; + //! Destroys the `BaseAssembler` instance. + ASMJIT_API virtual ~BaseAssembler() noexcept; + + //! \} + + //! \name Code-Buffer Management + //! \{ + + //! Returns the capacity of the current CodeBuffer. + inline size_t bufferCapacity() const noexcept { return (size_t)(_bufferEnd - _bufferData); } + //! Returns the number of remaining bytes in the current CodeBuffer. + inline size_t remainingSpace() const noexcept { return (size_t)(_bufferEnd - _bufferPtr); } + + //! Returns the current position in the CodeBuffer. + inline size_t offset() const noexcept { return (size_t)(_bufferPtr - _bufferData); } + + //! Sets the current position in the CodeBuffer to `offset`. + //! + //! \note The `offset` cannot be greater than buffer size even if it's + //! within the buffer's capacity. + ASMJIT_API Error setOffset(size_t offset); + + //! Returns the start of the CodeBuffer in the current section. + inline uint8_t* bufferData() const noexcept { return _bufferData; } + //! Returns the end (first invalid byte) in the current section. + inline uint8_t* bufferEnd() const noexcept { return _bufferEnd; } + //! Returns the current pointer in the CodeBuffer in the current section. + inline uint8_t* bufferPtr() const noexcept { return _bufferPtr; } + + //! \} + + //! \name Section Management + //! \{ + + //! Returns the current section. + inline Section* currentSection() const noexcept { return _section; } + + ASMJIT_API Error section(Section* section) override; + + //! \} + + //! \name Label Management + //! \{ + + ASMJIT_API Label newLabel() override; + ASMJIT_API Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId) override; + ASMJIT_API Error bind(const Label& label) override; + + //! \} + + //! \name Embed + //! \{ + + ASMJIT_API Error embed(const void* data, size_t dataSize) override; + ASMJIT_API Error embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1) override; + ASMJIT_API Error embedConstPool(const Label& label, const ConstPool& pool) override; + + ASMJIT_API Error embedLabel(const Label& label, size_t dataSize = 0) override; + ASMJIT_API Error embedLabelDelta(const Label& label, const Label& base, size_t dataSize = 0) override; + + //! \} + + //! \name Comment + //! \{ + + ASMJIT_API Error comment(const char* data, size_t size = SIZE_MAX) override; + + //! \} + + //! \name Events + //! \{ + + ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_ASSEMBLER_H_INCLUDED diff --git a/src/asmjit/core/builder.cpp b/src/asmjit/core/builder.cpp new file mode 100644 index 0000000..ad2cf7a --- /dev/null +++ b/src/asmjit/core/builder.cpp @@ -0,0 +1,893 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_BUILDER + +#include "../core/builder.h" +#include "../core/emitterutils_p.h" +#include "../core/errorhandler.h" +#include "../core/formatter.h" +#include "../core/logger.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// PostponedErrorHandler (Internal) +// ================================ + +//! Postponed error handler that never throws. Used as a temporal error handler +//! to run passes. If error occurs, the caller is notified and will call the +//! real error handler, that can throw. +class PostponedErrorHandler : public ErrorHandler { +public: + void handleError(Error err, const char* message, BaseEmitter* origin) override { + DebugUtils::unused(err, origin); + _message.assign(message); + } + + StringTmp<128> _message; +}; + +// BaseBuilder - Utilities +// ======================= + +static void BaseBuilder_deletePasses(BaseBuilder* self) noexcept { + for (Pass* pass : self->_passes) + pass->~Pass(); + self->_passes.reset(); +} + +// BaseBuilder - Construction & Destruction +// ======================================== + +BaseBuilder::BaseBuilder() noexcept + : BaseEmitter(EmitterType::kBuilder), + _codeZone(32768 - Zone::kBlockOverhead), + _dataZone(16384 - Zone::kBlockOverhead), + _passZone(65536 - Zone::kBlockOverhead), + _allocator(&_codeZone) {} + +BaseBuilder::~BaseBuilder() noexcept { + BaseBuilder_deletePasses(this); +} + +// BaseBuilder - Node Management +// ============================= + +Error BaseBuilder::newInstNode(InstNode** out, InstId instId, InstOptions instOptions, uint32_t opCount) { + uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); + ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); + + InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); + if (ASMJIT_UNLIKELY(!node)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + *out = new(node) InstNode(this, instId, instOptions, opCount, opCapacity); + return kErrorOk; +} + + +Error BaseBuilder::newLabelNode(LabelNode** out) { + *out = nullptr; + + ASMJIT_PROPAGATE(_newNodeT(out)); + return registerLabelNode(*out); +} + +Error BaseBuilder::newAlignNode(AlignNode** out, AlignMode alignMode, uint32_t alignment) { + *out = nullptr; + return _newNodeT(out, alignMode, alignment); +} + +Error BaseBuilder::newEmbedDataNode(EmbedDataNode** out, TypeId typeId, const void* data, size_t itemCount, size_t repeatCount) { + *out = nullptr; + + uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize()); + TypeId finalTypeId = TypeUtils::deabstract(typeId, deabstractDelta); + + if (ASMJIT_UNLIKELY(!TypeUtils::isValid(finalTypeId))) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + uint32_t typeSize = TypeUtils::sizeOf(finalTypeId); + Support::FastUInt8 of = 0; + + size_t dataSize = Support::mulOverflow(itemCount, size_t(typeSize), &of); + if (ASMJIT_UNLIKELY(of)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + EmbedDataNode* node; + ASMJIT_PROPAGATE(_newNodeT(&node)); + + node->_embed._typeId = typeId; + node->_embed._typeSize = uint8_t(typeSize); + node->_itemCount = itemCount; + node->_repeatCount = repeatCount; + + uint8_t* dstData = node->_inlineData; + if (dataSize > EmbedDataNode::kInlineBufferSize) { + dstData = static_cast(_dataZone.alloc(dataSize, 8)); + if (ASMJIT_UNLIKELY(!dstData)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + node->_externalData = dstData; + } + + if (data) + memcpy(dstData, data, dataSize); + + *out = node; + return kErrorOk; +} + +Error BaseBuilder::newConstPoolNode(ConstPoolNode** out) { + *out = nullptr; + + ASMJIT_PROPAGATE(_newNodeT(out)); + return registerLabelNode(*out); +} + +Error BaseBuilder::newCommentNode(CommentNode** out, const char* data, size_t size) { + *out = nullptr; + + if (data) { + if (size == SIZE_MAX) + size = strlen(data); + + if (size > 0) { + data = static_cast(_dataZone.dup(data, size, true)); + if (ASMJIT_UNLIKELY(!data)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + } + + return _newNodeT(out, data); +} + +BaseNode* BaseBuilder::addNode(BaseNode* node) noexcept { + ASMJIT_ASSERT(!node->_prev); + ASMJIT_ASSERT(!node->_next); + ASMJIT_ASSERT(!node->isActive()); + + if (!_cursor) { + if (!_firstNode) { + _firstNode = node; + _lastNode = node; + } + else { + node->_next = _firstNode; + _firstNode->_prev = node; + _firstNode = node; + } + } + else { + BaseNode* prev = _cursor; + BaseNode* next = _cursor->next(); + + node->_prev = prev; + node->_next = next; + + prev->_next = node; + if (next) + next->_prev = node; + else + _lastNode = node; + } + + node->addFlags(NodeFlags::kIsActive); + if (node->isSection()) + _dirtySectionLinks = true; + + _cursor = node; + return node; +} + +BaseNode* BaseBuilder::addAfter(BaseNode* node, BaseNode* ref) noexcept { + ASMJIT_ASSERT(!node->_prev); + ASMJIT_ASSERT(!node->_next); + + BaseNode* prev = ref; + BaseNode* next = ref->next(); + + node->_prev = prev; + node->_next = next; + + node->addFlags(NodeFlags::kIsActive); + if (node->isSection()) + _dirtySectionLinks = true; + + prev->_next = node; + if (next) + next->_prev = node; + else + _lastNode = node; + + return node; +} + +BaseNode* BaseBuilder::addBefore(BaseNode* node, BaseNode* ref) noexcept { + ASMJIT_ASSERT(!node->_prev); + ASMJIT_ASSERT(!node->_next); + ASMJIT_ASSERT(!node->isActive()); + ASMJIT_ASSERT(ref->isActive()); + + BaseNode* prev = ref->prev(); + BaseNode* next = ref; + + node->_prev = prev; + node->_next = next; + + node->addFlags(NodeFlags::kIsActive); + if (node->isSection()) + _dirtySectionLinks = true; + + next->_prev = node; + if (prev) + prev->_next = node; + else + _firstNode = node; + + return node; +} + +BaseNode* BaseBuilder::removeNode(BaseNode* node) noexcept { + if (!node->isActive()) + return node; + + BaseNode* prev = node->prev(); + BaseNode* next = node->next(); + + if (_firstNode == node) + _firstNode = next; + else + prev->_next = next; + + if (_lastNode == node) + _lastNode = prev; + else + next->_prev = prev; + + node->_prev = nullptr; + node->_next = nullptr; + node->clearFlags(NodeFlags::kIsActive); + if (node->isSection()) + _dirtySectionLinks = true; + + if (_cursor == node) + _cursor = prev; + + return node; +} + +void BaseBuilder::removeNodes(BaseNode* first, BaseNode* last) noexcept { + if (first == last) { + removeNode(first); + return; + } + + if (!first->isActive()) + return; + + BaseNode* prev = first->prev(); + BaseNode* next = last->next(); + + if (_firstNode == first) + _firstNode = next; + else + prev->_next = next; + + if (_lastNode == last) + _lastNode = prev; + else + next->_prev = prev; + + BaseNode* node = first; + uint32_t didRemoveSection = false; + + for (;;) { + next = node->next(); + ASMJIT_ASSERT(next != nullptr); + + node->_prev = nullptr; + node->_next = nullptr; + node->clearFlags(NodeFlags::kIsActive); + didRemoveSection |= uint32_t(node->isSection()); + + if (_cursor == node) + _cursor = prev; + + if (node == last) + break; + node = next; + } + + if (didRemoveSection) + _dirtySectionLinks = true; +} + +BaseNode* BaseBuilder::setCursor(BaseNode* node) noexcept { + BaseNode* old = _cursor; + _cursor = node; + return old; +} + +// BaseBuilder - Sections +// ====================== + +Error BaseBuilder::sectionNodeOf(SectionNode** out, uint32_t sectionId) { + *out = nullptr; + + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (ASMJIT_UNLIKELY(!_code->isSectionValid(sectionId))) + return reportError(DebugUtils::errored(kErrorInvalidSection)); + + if (sectionId >= _sectionNodes.size()) { + Error err = _sectionNodes.reserve(&_allocator, sectionId + 1); + if (ASMJIT_UNLIKELY(err != kErrorOk)) + return reportError(err); + } + + SectionNode* node = nullptr; + if (sectionId < _sectionNodes.size()) + node = _sectionNodes[sectionId]; + + if (!node) { + ASMJIT_PROPAGATE(_newNodeT(&node, sectionId)); + + // We have already reserved enough space, this cannot fail now. + if (sectionId >= _sectionNodes.size()) + _sectionNodes.resize(&_allocator, sectionId + 1); + + _sectionNodes[sectionId] = node; + } + + *out = node; + return kErrorOk; +} + +Error BaseBuilder::section(Section* section) { + SectionNode* node; + ASMJIT_PROPAGATE(sectionNodeOf(&node, section->id())); + ASMJIT_ASSUME(node != nullptr); + + if (!node->isActive()) { + // Insert the section at the end if it was not part of the code. + addAfter(node, lastNode()); + _cursor = node; + } + else { + // This is a bit tricky. We cache section links to make sure that + // switching sections doesn't involve traversal in linked-list unless + // the position of the section has changed. + if (hasDirtySectionLinks()) + updateSectionLinks(); + + if (node->_nextSection) + _cursor = node->_nextSection->_prev; + else + _cursor = _lastNode; + } + + return kErrorOk; +} + +void BaseBuilder::updateSectionLinks() noexcept { + if (!_dirtySectionLinks) + return; + + BaseNode* node_ = _firstNode; + SectionNode* currentSection = nullptr; + + while (node_) { + if (node_->isSection()) { + if (currentSection) + currentSection->_nextSection = node_->as(); + currentSection = node_->as(); + } + node_ = node_->next(); + } + + if (currentSection) + currentSection->_nextSection = nullptr; + + _dirtySectionLinks = false; +} + +// BaseBuilder - Labels +// ==================== + +Error BaseBuilder::labelNodeOf(LabelNode** out, uint32_t labelId) { + *out = nullptr; + + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + uint32_t index = labelId; + if (ASMJIT_UNLIKELY(index >= _code->labelCount())) + return DebugUtils::errored(kErrorInvalidLabel); + + if (index >= _labelNodes.size()) + ASMJIT_PROPAGATE(_labelNodes.resize(&_allocator, index + 1)); + + LabelNode* node = _labelNodes[index]; + if (!node) { + ASMJIT_PROPAGATE(_newNodeT(&node, labelId)); + _labelNodes[index] = node; + } + + *out = node; + return kErrorOk; +} + +Error BaseBuilder::registerLabelNode(LabelNode* node) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + LabelEntry* le; + ASMJIT_PROPAGATE(_code->newLabelEntry(&le)); + uint32_t labelId = le->id(); + + // We just added one label so it must be true. + ASMJIT_ASSERT(_labelNodes.size() < labelId + 1); + ASMJIT_PROPAGATE(_labelNodes.resize(&_allocator, labelId + 1)); + + _labelNodes[labelId] = node; + node->_labelId = labelId; + + return kErrorOk; +} + +static Error BaseBuilder_newLabelInternal(BaseBuilder* self, uint32_t labelId) { + ASMJIT_ASSERT(self->_labelNodes.size() < labelId + 1); + + uint32_t growBy = labelId - self->_labelNodes.size(); + Error err = self->_labelNodes.willGrow(&self->_allocator, growBy); + + if (ASMJIT_UNLIKELY(err)) + return self->reportError(err); + + LabelNode* node; + ASMJIT_PROPAGATE(self->_newNodeT(&node, labelId)); + + self->_labelNodes.resize(&self->_allocator, labelId + 1); + self->_labelNodes[labelId] = node; + node->_labelId = labelId; + return kErrorOk; +} + +Label BaseBuilder::newLabel() { + uint32_t labelId = Globals::kInvalidId; + LabelEntry* le; + + if (_code && + _code->newLabelEntry(&le) == kErrorOk && + BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { + labelId = le->id(); + } + + return Label(labelId); +} + +Label BaseBuilder::newNamedLabel(const char* name, size_t nameSize, LabelType type, uint32_t parentId) { + uint32_t labelId = Globals::kInvalidId; + LabelEntry* le; + + if (_code && + _code->newNamedLabelEntry(&le, name, nameSize, type, parentId) == kErrorOk && + BaseBuilder_newLabelInternal(this, le->id()) == kErrorOk) { + labelId = le->id(); + } + + return Label(labelId); +} + +Error BaseBuilder::bind(const Label& label) { + LabelNode* node; + ASMJIT_PROPAGATE(labelNodeOf(&node, label)); + + addNode(node); + return kErrorOk; +} + +// BaseBuilder - Passes +// ==================== + +ASMJIT_FAVOR_SIZE Pass* BaseBuilder::passByName(const char* name) const noexcept { + for (Pass* pass : _passes) + if (strcmp(pass->name(), name) == 0) + return pass; + return nullptr; +} + +ASMJIT_FAVOR_SIZE Error BaseBuilder::addPass(Pass* pass) noexcept { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (ASMJIT_UNLIKELY(pass == nullptr)) { + // Since this is directly called by `addPassT()` we treat `null` argument + // as out-of-memory condition. Otherwise it would be API misuse. + return DebugUtils::errored(kErrorOutOfMemory); + } + else if (ASMJIT_UNLIKELY(pass->_cb)) { + // Kinda weird, but okay... + if (pass->_cb == this) + return kErrorOk; + return DebugUtils::errored(kErrorInvalidState); + } + + ASMJIT_PROPAGATE(_passes.append(&_allocator, pass)); + pass->_cb = this; + return kErrorOk; +} + +ASMJIT_FAVOR_SIZE Error BaseBuilder::deletePass(Pass* pass) noexcept { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (ASMJIT_UNLIKELY(pass == nullptr)) + return DebugUtils::errored(kErrorInvalidArgument); + + if (pass->_cb != nullptr) { + if (pass->_cb != this) + return DebugUtils::errored(kErrorInvalidState); + + uint32_t index = _passes.indexOf(pass); + ASMJIT_ASSERT(index != Globals::kNotFound); + + pass->_cb = nullptr; + _passes.removeAt(index); + } + + pass->~Pass(); + return kErrorOk; +} + +Error BaseBuilder::runPasses() { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (_passes.empty()) + return kErrorOk; + + ErrorHandler* prev = errorHandler(); + PostponedErrorHandler postponed; + + Error err = kErrorOk; + setErrorHandler(&postponed); + + for (Pass* pass : _passes) { + _passZone.reset(); + err = pass->run(&_passZone, _logger); + if (err) + break; + } + _passZone.reset(); + setErrorHandler(prev); + + if (ASMJIT_UNLIKELY(err)) + return reportError(err, !postponed._message.empty() ? postponed._message.data() : nullptr); + + return kErrorOk; +} + +// BaseBuilder - Emit +// ================== + +Error BaseBuilder::_emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) { + uint32_t opCount = EmitterUtils::opCountFromEmitArgs(o0, o1, o2, opExt); + InstOptions options = instOptions() | forcedInstOptions(); + + if (Support::test(options, InstOptions::kReserved)) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + +#ifndef ASMJIT_NO_VALIDATION + // Strict validation. + if (hasDiagnosticOption(DiagnosticOptions::kValidateIntermediate)) { + Operand_ opArray[Globals::kMaxOpCount]; + EmitterUtils::opArrayFromEmitArgs(opArray, o0, o1, o2, opExt); + + ValidationFlags validationFlags = isCompiler() ? ValidationFlags::kEnableVirtRegs : ValidationFlags::kNone; + Error err = _funcs.validate(arch(), BaseInst(instId, options, _extraReg), opArray, opCount, validationFlags); + + if (ASMJIT_UNLIKELY(err)) { +#ifndef ASMJIT_NO_LOGGING + return EmitterUtils::logInstructionFailed(this, err, instId, options, o0, o1, o2, opExt); +#else + resetInstOptions(); + resetExtraReg(); + resetInlineComment(); + return reportError(err); +#endif + } + } +#endif + + // Clear instruction options that should never be part of a regular instruction. + options &= ~InstOptions::kReserved; + } + + uint32_t opCapacity = InstNode::capacityOfOpCount(opCount); + ASMJIT_ASSERT(opCapacity >= InstNode::kBaseOpCapacity); + + InstNode* node = _allocator.allocT(InstNode::nodeSizeOfOpCapacity(opCapacity)); + const char* comment = inlineComment(); + + resetInstOptions(); + resetInlineComment(); + + if (ASMJIT_UNLIKELY(!node)) { + resetExtraReg(); + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + } + + node = new(node) InstNode(this, instId, options, opCount, opCapacity); + node->setExtraReg(extraReg()); + node->setOp(0, o0); + node->setOp(1, o1); + node->setOp(2, o2); + for (uint32_t i = 3; i < opCount; i++) + node->setOp(i, opExt[i - 3]); + node->resetOpRange(opCount, opCapacity); + + if (comment) + node->setInlineComment(static_cast(_dataZone.dup(comment, strlen(comment), true))); + + addNode(node); + resetExtraReg(); + return kErrorOk; +} + +// BaseBuilder - Align +// =================== + +Error BaseBuilder::align(AlignMode alignMode, uint32_t alignment) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + AlignNode* node; + ASMJIT_PROPAGATE(newAlignNode(&node, alignMode, alignment)); + ASMJIT_ASSUME(node != nullptr); + + addNode(node); + return kErrorOk; +} + +// BaseBuilder - Embed +// =================== + +Error BaseBuilder::embed(const void* data, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + EmbedDataNode* node; + ASMJIT_PROPAGATE(newEmbedDataNode(&node, TypeId::kUInt8, data, dataSize)); + ASMJIT_ASSUME(node != nullptr); + + addNode(node); + return kErrorOk; +} + +Error BaseBuilder::embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t itemRepeat) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + EmbedDataNode* node; + ASMJIT_PROPAGATE(newEmbedDataNode(&node, typeId, data, itemCount, itemRepeat)); + ASMJIT_ASSUME(node != nullptr); + + addNode(node); + return kErrorOk; +} + +Error BaseBuilder::embedConstPool(const Label& label, const ConstPool& pool) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (!isLabelValid(label)) + return reportError(DebugUtils::errored(kErrorInvalidLabel)); + + ASMJIT_PROPAGATE(align(AlignMode::kData, uint32_t(pool.alignment()))); + ASMJIT_PROPAGATE(bind(label)); + + EmbedDataNode* node; + ASMJIT_PROPAGATE(newEmbedDataNode(&node, TypeId::kUInt8, nullptr, pool.size())); + ASMJIT_ASSUME(node != nullptr); + + pool.fill(node->data()); + addNode(node); + return kErrorOk; +} + +// BaseBuilder - EmbedLabel & EmbedLabelDelta +// ========================================== +// +// If dataSize is zero it means that the size is the same as target register width, however, +// if it's provided we really want to validate whether it's within the possible range. + +static inline bool BaseBuilder_checkDataSize(size_t dataSize) noexcept { + return !dataSize || (Support::isPowerOf2(dataSize) && dataSize <= 8); +} + +Error BaseBuilder::embedLabel(const Label& label, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (!BaseBuilder_checkDataSize(dataSize)) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + EmbedLabelNode* node; + ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), uint32_t(dataSize))); + + addNode(node); + return kErrorOk; +} + +Error BaseBuilder::embedLabelDelta(const Label& label, const Label& base, size_t dataSize) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + if (!BaseBuilder_checkDataSize(dataSize)) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + EmbedLabelDeltaNode* node; + ASMJIT_PROPAGATE(_newNodeT(&node, label.id(), base.id(), uint32_t(dataSize))); + + addNode(node); + return kErrorOk; +} + +// BaseBuilder - Comment +// ===================== + +Error BaseBuilder::comment(const char* data, size_t size) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + CommentNode* node; + ASMJIT_PROPAGATE(newCommentNode(&node, data, size)); + ASMJIT_ASSUME(node != nullptr); + + addNode(node); + return kErrorOk; +} + +// BaseBuilder - SerializeTo +// ========================= + +Error BaseBuilder::serializeTo(BaseEmitter* dst) { + Error err = kErrorOk; + BaseNode* node_ = _firstNode; + + Operand_ opArray[Globals::kMaxOpCount]; + + do { + dst->setInlineComment(node_->inlineComment()); + + if (node_->isInst()) { + InstNode* node = node_->as(); + + // NOTE: Inlined to remove one additional call per instruction. + dst->setInstOptions(node->options()); + dst->setExtraReg(node->extraReg()); + + const Operand_* op = node->operands(); + const Operand_* opExt = EmitterUtils::noExt; + + uint32_t opCount = node->opCount(); + if (opCount > 3) { + uint32_t i = 4; + opArray[3] = op[3]; + + while (i < opCount) { + opArray[i].copyFrom(op[i]); + i++; + } + while (i < Globals::kMaxOpCount) { + opArray[i].reset(); + i++; + } + opExt = opArray + 3; + } + + err = dst->_emit(node->id(), op[0], op[1], op[2], opExt); + } + else if (node_->isLabel()) { + if (node_->isConstPool()) { + ConstPoolNode* node = node_->as(); + err = dst->embedConstPool(node->label(), node->constPool()); + } + else { + LabelNode* node = node_->as(); + err = dst->bind(node->label()); + } + } + else if (node_->isAlign()) { + AlignNode* node = node_->as(); + err = dst->align(node->alignMode(), node->alignment()); + } + else if (node_->isEmbedData()) { + EmbedDataNode* node = node_->as(); + err = dst->embedDataArray(node->typeId(), node->data(), node->itemCount(), node->repeatCount()); + } + else if (node_->isEmbedLabel()) { + EmbedLabelNode* node = node_->as(); + err = dst->embedLabel(node->label(), node->dataSize()); + } + else if (node_->isEmbedLabelDelta()) { + EmbedLabelDeltaNode* node = node_->as(); + err = dst->embedLabelDelta(node->label(), node->baseLabel(), node->dataSize()); + } + else if (node_->isSection()) { + SectionNode* node = node_->as(); + err = dst->section(_code->sectionById(node->id())); + } + else if (node_->isComment()) { + CommentNode* node = node_->as(); + err = dst->comment(node->inlineComment()); + } + + if (err) break; + node_ = node_->next(); + } while (node_); + + return err; +} + +// BaseBuilder - Events +// ==================== + +Error BaseBuilder::onAttach(CodeHolder* code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + SectionNode* initialSection; + Error err = sectionNodeOf(&initialSection, 0); + + if (!err) + err = _passes.willGrow(&_allocator, 8); + + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + return err; + } + + ASMJIT_ASSUME(initialSection != nullptr); + _cursor = initialSection; + _firstNode = initialSection; + _lastNode = initialSection; + initialSection->setFlags(NodeFlags::kIsActive); + + return kErrorOk; +} + +Error BaseBuilder::onDetach(CodeHolder* code) noexcept { + BaseBuilder_deletePasses(this); + _sectionNodes.reset(); + _labelNodes.reset(); + + _allocator.reset(&_codeZone); + _codeZone.reset(); + _dataZone.reset(); + _passZone.reset(); + + _nodeFlags = NodeFlags::kNone; + _cursor = nullptr; + _firstNode = nullptr; + _lastNode = nullptr; + + return Base::onDetach(code); +} + +// Pass - Construction & Destruction +// ================================= + +Pass::Pass(const char* name) noexcept + : _name(name) {} +Pass::~Pass() noexcept {} + +ASMJIT_END_NAMESPACE + +#endif // !ASMJIT_NO_BUILDER diff --git a/src/asmjit/core/builder.h b/src/asmjit/core/builder.h new file mode 100644 index 0000000..3575de2 --- /dev/null +++ b/src/asmjit/core/builder.h @@ -0,0 +1,1391 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_BUILDER_H_INCLUDED +#define ASMJIT_CORE_BUILDER_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_BUILDER + +#include "../core/assembler.h" +#include "../core/codeholder.h" +#include "../core/constpool.h" +#include "../core/formatter.h" +#include "../core/inst.h" +#include "../core/operand.h" +#include "../core/string.h" +#include "../core/support.h" +#include "../core/type.h" +#include "../core/zone.h" +#include "../core/zonevector.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_builder +//! \{ + +class BaseBuilder; +class Pass; + +class BaseNode; +class InstNode; +class SectionNode; +class LabelNode; +class AlignNode; +class EmbedDataNode; +class EmbedLabelNode; +class ConstPoolNode; +class CommentNode; +class SentinelNode; +class LabelDeltaNode; + +//! Type of node used by \ref BaseBuilder and \ref BaseCompiler. +enum class NodeType : uint8_t { + //! Invalid node (internal, don't use). + kNone = 0, + + // [BaseBuilder] + + //! Node is \ref InstNode or \ref InstExNode. + kInst = 1, + //! Node is \ref SectionNode. + kSection = 2, + //! Node is \ref LabelNode. + kLabel = 3, + //! Node is \ref AlignNode. + kAlign = 4, + //! Node is \ref EmbedDataNode. + kEmbedData = 5, + //! Node is \ref EmbedLabelNode. + kEmbedLabel = 6, + //! Node is \ref EmbedLabelDeltaNode. + kEmbedLabelDelta = 7, + //! Node is \ref ConstPoolNode. + kConstPool = 8, + //! Node is \ref CommentNode. + kComment = 9, + //! Node is \ref SentinelNode. + kSentinel = 10, + + // [BaseCompiler] + + //! Node is \ref JumpNode (acts as InstNode). + kJump = 15, + //! Node is \ref FuncNode (acts as LabelNode). + kFunc = 16, + //! Node is \ref FuncRetNode (acts as InstNode). + kFuncRet = 17, + //! Node is \ref InvokeNode (acts as InstNode). + kInvoke = 18, + + // [UserDefined] + + //! First id of a user-defined node. + kUser = 32 +}; + +//! Node flags, specify what the node is and/or does. +enum class NodeFlags : uint8_t { + //! No flags. + kNone = 0, + //! Node is code that can be executed (instruction, label, align, etc...). + kIsCode = 0x01u, + //! Node is data that cannot be executed (data, const-pool, etc...). + kIsData = 0x02u, + //! Node is informative, can be removed and ignored. + kIsInformative = 0x04u, + //! Node can be safely removed if unreachable. + kIsRemovable = 0x08u, + //! Node does nothing when executed (label, align, explicit nop). + kHasNoEffect = 0x10u, + //! Node is an instruction or acts as it. + kActsAsInst = 0x20u, + //! Node is a label or acts as it. + kActsAsLabel = 0x40u, + //! Node is active (part of the code). + kIsActive = 0x80u +}; +ASMJIT_DEFINE_ENUM_FLAGS(NodeFlags) + +//! Type of the sentinel (purery informative purpose). +enum class SentinelType : uint8_t { + //! Type of the sentinel is not known. + kUnknown = 0u, + //! This is a sentinel used at the end of \ref FuncNode. + kFuncEnd = 1u +}; + +//! Builder interface. +//! +//! `BaseBuilder` interface was designed to be used as a \ref BaseAssembler replacement in case pre-processing or +//! post-processing of the generated code is required. The code can be modified during or after code generation. +//! Pre processing or post processing can be done manually or through a \ref Pass object. \ref BaseBuilder stores +//! the emitted code as a double-linked list of nodes, which allows O(1) insertion and removal during processing. +//! +//! Check out architecture specific builders for more details and examples: +//! +//! - \ref x86::Builder - X86/X64 builder implementation. +class ASMJIT_VIRTAPI BaseBuilder : public BaseEmitter { +public: + ASMJIT_NONCOPYABLE(BaseBuilder) + typedef BaseEmitter Base; + + //! \name Members + //! \{ + + //! Base zone used to allocate nodes and passes. + Zone _codeZone; + //! Data zone used to allocate data and names. + Zone _dataZone; + //! Pass zone, passed to `Pass::run()`. + Zone _passZone; + //! Allocator that uses `_codeZone`. + ZoneAllocator _allocator; + + //! Array of `Pass` objects. + ZoneVector _passes {}; + //! Maps section indexes to `LabelNode` nodes. + ZoneVector _sectionNodes {}; + //! Maps label indexes to `LabelNode` nodes. + ZoneVector _labelNodes {}; + + //! Current node (cursor). + BaseNode* _cursor = nullptr; + //! First node of the current section. + BaseNode* _firstNode = nullptr; + //! Last node of the current section. + BaseNode* _lastNode = nullptr; + + //! Flags assigned to each new node. + NodeFlags _nodeFlags = NodeFlags::kNone; + //! The sections links are dirty (used internally). + bool _dirtySectionLinks = false; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `BaseBuilder` instance. + ASMJIT_API BaseBuilder() noexcept; + //! Destroys the `BaseBuilder` instance. + ASMJIT_API virtual ~BaseBuilder() noexcept; + + //! \} + + //! \name Node Management + //! \{ + + //! Returns the first node. + inline BaseNode* firstNode() const noexcept { return _firstNode; } + //! Returns the last node. + inline BaseNode* lastNode() const noexcept { return _lastNode; } + + //! Allocates and instantiates a new node of type `T` and returns its instance. If the allocation fails `nullptr` + //! is returned. + //! + //! The template argument `T` must be a type that is extends \ref BaseNode. + //! + //! \remarks The pointer returned (if non-null) is owned by the Builder or Compiler. When the Builder/Compiler + //! is destroyed it destroys all nodes it created so no manual memory management is required. + template + inline Error _newNodeT(T** ASMJIT_NONNULL(out), Args&&... args) { + *out = _allocator.newT(this, std::forward(args)...); + if (ASMJIT_UNLIKELY(!*out)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + return kErrorOk; + } + + //! Creates a new \ref InstNode. + ASMJIT_API Error newInstNode(InstNode** ASMJIT_NONNULL(out), InstId instId, InstOptions instOptions, uint32_t opCount); + //! Creates a new \ref LabelNode. + ASMJIT_API Error newLabelNode(LabelNode** ASMJIT_NONNULL(out)); + //! Creates a new \ref AlignNode. + ASMJIT_API Error newAlignNode(AlignNode** ASMJIT_NONNULL(out), AlignMode alignMode, uint32_t alignment); + //! Creates a new \ref EmbedDataNode. + ASMJIT_API Error newEmbedDataNode(EmbedDataNode** ASMJIT_NONNULL(out), TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1); + //! Creates a new \ref ConstPoolNode. + ASMJIT_API Error newConstPoolNode(ConstPoolNode** ASMJIT_NONNULL(out)); + //! Creates a new \ref CommentNode. + ASMJIT_API Error newCommentNode(CommentNode** ASMJIT_NONNULL(out), const char* data, size_t size); + + //! Adds `node` after the current and sets the current node to the given `node`. + ASMJIT_API BaseNode* addNode(BaseNode* ASMJIT_NONNULL(node)) noexcept; + //! Inserts the given `node` after `ref`. + ASMJIT_API BaseNode* addAfter(BaseNode* ASMJIT_NONNULL(node), BaseNode* ASMJIT_NONNULL(ref)) noexcept; + //! Inserts the given `node` before `ref`. + ASMJIT_API BaseNode* addBefore(BaseNode* ASMJIT_NONNULL(node), BaseNode* ASMJIT_NONNULL(ref)) noexcept; + //! Removes the given `node`. + ASMJIT_API BaseNode* removeNode(BaseNode* ASMJIT_NONNULL(node)) noexcept; + //! Removes multiple nodes. + ASMJIT_API void removeNodes(BaseNode* first, BaseNode* last) noexcept; + + //! Returns the cursor. + //! + //! When the Builder/Compiler is created it automatically creates a '.text' \ref SectionNode, which will be the + //! initial one. When instructions are added they are always added after the cursor and the cursor is changed + //! to be that newly added node. Use `setCursor()` to change where new nodes are inserted. + inline BaseNode* cursor() const noexcept { return _cursor; } + + //! Sets the current node to `node` and return the previous one. + ASMJIT_API BaseNode* setCursor(BaseNode* node) noexcept; + + //! Sets the current node without returning the previous node. + //! + //! Only use this function if you are concerned about performance and want this inlined (for example if you set + //! the cursor in a loop, etc...). + inline void _setCursor(BaseNode* node) noexcept { _cursor = node; } + + //! \} + + //! \name Section Management + //! \{ + + //! Returns a vector of SectionNode objects. + //! + //! \note If a section of some id is not associated with the Builder/Compiler it would be null, so always check + //! for nulls if you iterate over the vector. + inline const ZoneVector& sectionNodes() const noexcept { + return _sectionNodes; + } + + //! Tests whether the `SectionNode` of the given `sectionId` was registered. + inline bool hasRegisteredSectionNode(uint32_t sectionId) const noexcept { + return sectionId < _sectionNodes.size() && _sectionNodes[sectionId] != nullptr; + } + + //! Returns or creates a `SectionNode` that matches the given `sectionId`. + //! + //! \remarks This function will either get the existing `SectionNode` or create it in case it wasn't created before. + //! You can check whether a section has a registered `SectionNode` by using `BaseBuilder::hasRegisteredSectionNode()`. + ASMJIT_API Error sectionNodeOf(SectionNode** ASMJIT_NONNULL(out), uint32_t sectionId); + + ASMJIT_API Error section(Section* ASMJIT_NONNULL(section)) override; + + //! Returns whether the section links of active section nodes are dirty. You can update these links by calling + //! `updateSectionLinks()` in such case. + inline bool hasDirtySectionLinks() const noexcept { return _dirtySectionLinks; } + + //! Updates links of all active section nodes. + ASMJIT_API void updateSectionLinks() noexcept; + + //! \} + + //! \name Label Management + //! \{ + + //! Returns a vector of \ref LabelNode nodes. + //! + //! \note If a label of some id is not associated with the Builder/Compiler it would be null, so always check for + //! nulls if you iterate over the vector. + inline const ZoneVector& labelNodes() const noexcept { return _labelNodes; } + + //! Tests whether the `LabelNode` of the given `labelId` was registered. + inline bool hasRegisteredLabelNode(uint32_t labelId) const noexcept { + return labelId < _labelNodes.size() && _labelNodes[labelId] != nullptr; + } + + //! \overload + inline bool hasRegisteredLabelNode(const Label& label) const noexcept { + return hasRegisteredLabelNode(label.id()); + } + + //! Gets or creates a \ref LabelNode that matches the given `labelId`. + //! + //! \remarks This function will either get the existing `LabelNode` or create it in case it wasn't created before. + //! You can check whether a label has a registered `LabelNode` by calling \ref BaseBuilder::hasRegisteredLabelNode(). + ASMJIT_API Error labelNodeOf(LabelNode** ASMJIT_NONNULL(out), uint32_t labelId); + + //! \overload + inline Error labelNodeOf(LabelNode** ASMJIT_NONNULL(out), const Label& label) { + return labelNodeOf(out, label.id()); + } + + //! Registers this \ref LabelNode (internal). + //! + //! This function is used internally to register a newly created `LabelNode` with this instance of Builder/Compiler. + //! Use \ref labelNodeOf() functions to get back \ref LabelNode from a label or its identifier. + ASMJIT_API Error registerLabelNode(LabelNode* ASMJIT_NONNULL(node)); + + ASMJIT_API Label newLabel() override; + ASMJIT_API Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId) override; + ASMJIT_API Error bind(const Label& label) override; + + //! \} + + //! \name Passes + //! \{ + + //! Returns a vector of `Pass` instances that will be executed by `runPasses()`. + inline const ZoneVector& passes() const noexcept { return _passes; } + + //! Allocates and instantiates a new pass of type `T` and returns its instance. If the allocation fails `nullptr` is + //! returned. + //! + //! The template argument `T` must be a type that is extends \ref Pass. + //! + //! \remarks The pointer returned (if non-null) is owned by the Builder or Compiler. When the Builder/Compiler is + //! destroyed it destroys all passes it created so no manual memory management is required. + template + inline T* newPassT() noexcept { return _codeZone.newT(); } + + //! \overload + template + inline T* newPassT(Args&&... args) noexcept { return _codeZone.newT(std::forward(args)...); } + + template + inline Error addPassT() { return addPass(newPassT()); } + + template + inline Error addPassT(Args&&... args) { return addPass(newPassT(std::forward(args)...)); } + + //! Returns `Pass` by name. + //! + //! If the pass having the given `name` doesn't exist `nullptr` is returned. + ASMJIT_API Pass* passByName(const char* name) const noexcept; + //! Adds `pass` to the list of passes. + ASMJIT_API Error addPass(Pass* pass) noexcept; + //! Removes `pass` from the list of passes and delete it. + ASMJIT_API Error deletePass(Pass* pass) noexcept; + + //! Runs all passes in order. + ASMJIT_API Error runPasses(); + + //! \} + + //! \name Emit + //! \{ + + ASMJIT_API Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) override; + + //! \} + + //! \name Align + //! \{ + + ASMJIT_API Error align(AlignMode alignMode, uint32_t alignment) override; + + //! \} + + //! \name Embed + //! \{ + + ASMJIT_API Error embed(const void* data, size_t dataSize) override; + ASMJIT_API Error embedDataArray(TypeId typeId, const void* data, size_t count, size_t repeat = 1) override; + ASMJIT_API Error embedConstPool(const Label& label, const ConstPool& pool) override; + + ASMJIT_API Error embedLabel(const Label& label, size_t dataSize = 0) override; + ASMJIT_API Error embedLabelDelta(const Label& label, const Label& base, size_t dataSize = 0) override; + + //! \} + + //! \name Comment + //! \{ + + ASMJIT_API Error comment(const char* data, size_t size = SIZE_MAX) override; + + //! \} + + //! \name Serialization + //! \{ + + //! Serializes everything the given emitter `dst`. + //! + //! Although not explicitly required the emitter will most probably be of Assembler type. The reason is that + //! there is no known use of serializing nodes held by Builder/Compiler into another Builder-like emitter. + ASMJIT_API Error serializeTo(BaseEmitter* dst); + + //! \} + + //! \name Events + //! \{ + + ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + + //! \} +}; + +//! Base node. +//! +//! Every node represents a building-block used by \ref BaseBuilder. It can be instruction, data, label, comment, +//! directive, or any other high-level representation that can be transformed to the building blocks mentioned. +//! Every class that inherits \ref BaseBuilder can define its own high-level nodes that can be later lowered to +//! basic nodes like instructions. +class BaseNode { +public: + ASMJIT_NONCOPYABLE(BaseNode) + + //! \name Members + //! \{ + + union { + struct { + //! Previous node. + BaseNode* _prev; + //! Next node. + BaseNode* _next; + }; + //! Links (an alternative view to previous and next nodes). + BaseNode* _links[2]; + }; + + //! Data shared between all types of nodes. + struct AnyData { + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Not used by BaseNode. + uint8_t _reserved0; + //! Not used by BaseNode. + uint8_t _reserved1; + }; + + //! Data used by \ref AlignNode. + struct AlignData { + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Align mode. + AlignMode _alignMode; + //! Not used by AlignNode. + uint8_t _reserved; + }; + + //! Data used by \ref InstNode. + struct InstData { + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Instruction operands count (used). + uint8_t _opCount; + //! Instruction operands capacity (allocated). + uint8_t _opCapacity; + }; + + //! Data used by \ref EmbedDataNode. + struct EmbedData { + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Type id. + TypeId _typeId; + //! Size of `_typeId`. + uint8_t _typeSize; + }; + + //! Data used by \ref SentinelNode. + struct SentinelData { + //! Node type. + NodeType _nodeType; + //! Node flags. + NodeFlags _nodeFlags; + //! Sentinel type. + SentinelType _sentinelType; + //! Not used by BaseNode. + uint8_t _reserved1; + }; + + //! Data that can have different meaning dependning on \ref NodeType. + union { + //! Data useful by any node type. + AnyData _any; + //! Data specific to \ref AlignNode. + AlignData _alignData; + //! Data specific to \ref InstNode. + InstData _inst; + //! Data specific to \ref EmbedDataNode. + EmbedData _embed; + //! Data specific to \ref SentinelNode. + SentinelData _sentinel; + }; + + //! Node position in code (should be unique). + uint32_t _position; + + //! Value reserved for AsmJit users never touched by AsmJit itself. + union { + //! User data as 64-bit integer. + uint64_t _userDataU64; + //! User data as pointer. + void* _userDataPtr; + }; + + //! Data used exclusively by the current `Pass`. + void* _passData; + + //! Inline comment/annotation or nullptr if not used. + const char* _inlineComment; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `BaseNode` - always use `BaseBuilder` to allocate nodes. + inline BaseNode(BaseBuilder* cb, NodeType nodeType, NodeFlags nodeFlags = NodeFlags::kNone) noexcept { + _prev = nullptr; + _next = nullptr; + _any._nodeType = nodeType; + _any._nodeFlags = nodeFlags | cb->_nodeFlags; + _any._reserved0 = 0; + _any._reserved1 = 0; + _position = 0; + _userDataU64 = 0; + _passData = nullptr; + _inlineComment = nullptr; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Casts this node to `T*`. + template + inline T* as() noexcept { return static_cast(this); } + //! Casts this node to `const T*`. + template + inline const T* as() const noexcept { return static_cast(this); } + + //! Returns previous node or `nullptr` if this node is either first or not + //! part of Builder/Compiler node-list. + inline BaseNode* prev() const noexcept { return _prev; } + //! Returns next node or `nullptr` if this node is either last or not part + //! of Builder/Compiler node-list. + inline BaseNode* next() const noexcept { return _next; } + + //! Returns the type of the node, see `NodeType`. + inline NodeType type() const noexcept { return _any._nodeType; } + + //! Sets the type of the node, see `NodeType` (internal). + //! + //! \remarks You should never set a type of a node to anything else than the initial value. This function is only + //! provided for users that use custom nodes and need to change the type either during construction or later. + inline void setType(NodeType type) noexcept { _any._nodeType = type; } + + //! Tests whether this node is either `InstNode` or extends it. + inline bool isInst() const noexcept { return hasFlag(NodeFlags::kActsAsInst); } + //! Tests whether this node is `SectionNode`. + inline bool isSection() const noexcept { return type() == NodeType::kSection; } + //! Tests whether this node is either `LabelNode` or extends it. + inline bool isLabel() const noexcept { return hasFlag(NodeFlags::kActsAsLabel); } + //! Tests whether this node is `AlignNode`. + inline bool isAlign() const noexcept { return type() == NodeType::kAlign; } + //! Tests whether this node is `EmbedDataNode`. + inline bool isEmbedData() const noexcept { return type() == NodeType::kEmbedData; } + //! Tests whether this node is `EmbedLabelNode`. + inline bool isEmbedLabel() const noexcept { return type() == NodeType::kEmbedLabel; } + //! Tests whether this node is `EmbedLabelDeltaNode`. + inline bool isEmbedLabelDelta() const noexcept { return type() == NodeType::kEmbedLabelDelta; } + //! Tests whether this node is `ConstPoolNode`. + inline bool isConstPool() const noexcept { return type() == NodeType::kConstPool; } + //! Tests whether this node is `CommentNode`. + inline bool isComment() const noexcept { return type() == NodeType::kComment; } + //! Tests whether this node is `SentinelNode`. + inline bool isSentinel() const noexcept { return type() == NodeType::kSentinel; } + + //! Tests whether this node is `FuncNode`. + inline bool isFunc() const noexcept { return type() == NodeType::kFunc; } + //! Tests whether this node is `FuncRetNode`. + inline bool isFuncRet() const noexcept { return type() == NodeType::kFuncRet; } + //! Tests whether this node is `InvokeNode`. + inline bool isInvoke() const noexcept { return type() == NodeType::kInvoke; } + + //! Returns the node flags. + inline NodeFlags flags() const noexcept { return _any._nodeFlags; } + //! Tests whether the node has the given `flag` set. + inline bool hasFlag(NodeFlags flag) const noexcept { return Support::test(_any._nodeFlags, flag); } + //! Replaces node flags with `flags`. + inline void setFlags(NodeFlags flags) noexcept { _any._nodeFlags = flags; } + //! Adds the given `flags` to node flags. + inline void addFlags(NodeFlags flags) noexcept { _any._nodeFlags |= flags; } + //! Clears the given `flags` from node flags. + inline void clearFlags(NodeFlags flags) noexcept { _any._nodeFlags &= ~flags; } + + //! Tests whether the node is code that can be executed. + inline bool isCode() const noexcept { return hasFlag(NodeFlags::kIsCode); } + //! Tests whether the node is data that cannot be executed. + inline bool isData() const noexcept { return hasFlag(NodeFlags::kIsData); } + //! Tests whether the node is informative only (is never encoded like comment, etc...). + inline bool isInformative() const noexcept { return hasFlag(NodeFlags::kIsInformative); } + //! Tests whether the node is removable if it's in an unreachable code block. + inline bool isRemovable() const noexcept { return hasFlag(NodeFlags::kIsRemovable); } + //! Tests whether the node has no effect when executed (label, .align, nop, ...). + inline bool hasNoEffect() const noexcept { return hasFlag(NodeFlags::kHasNoEffect); } + //! Tests whether the node is part of the code. + inline bool isActive() const noexcept { return hasFlag(NodeFlags::kIsActive); } + + //! Tests whether the node has a position assigned. + //! + //! \remarks Returns `true` if node position is non-zero. + inline bool hasPosition() const noexcept { return _position != 0; } + //! Returns node position. + inline uint32_t position() const noexcept { return _position; } + //! Sets node position. + //! + //! Node position is a 32-bit unsigned integer that is used by Compiler to track where the node is relatively to + //! the start of the function. It doesn't describe a byte position in a binary, instead it's just a pseudo position + //! used by liveness analysis and other tools around Compiler. + //! + //! If you don't use Compiler then you may use `position()` and `setPosition()` freely for your own purposes if + //! the 32-bit value limit is okay for you. + inline void setPosition(uint32_t position) noexcept { _position = position; } + + //! Returns user data casted to `T*`. + //! + //! User data is decicated to be used only by AsmJit users and not touched by the library. The data has a pointer + //! size so you can either store a pointer or `intptr_t` value through `setUserDataAsIntPtr()`. + template + inline T* userDataAsPtr() const noexcept { return static_cast(_userDataPtr); } + //! Returns user data casted to `int64_t`. + inline int64_t userDataAsInt64() const noexcept { return int64_t(_userDataU64); } + //! Returns user data casted to `uint64_t`. + inline uint64_t userDataAsUInt64() const noexcept { return _userDataU64; } + + //! Sets user data to `data`. + template + inline void setUserDataAsPtr(T* data) noexcept { _userDataPtr = static_cast(data); } + //! Sets used data to the given 64-bit signed `value`. + inline void setUserDataAsInt64(int64_t value) noexcept { _userDataU64 = uint64_t(value); } + //! Sets used data to the given 64-bit unsigned `value`. + inline void setUserDataAsUInt64(uint64_t value) noexcept { _userDataU64 = value; } + + //! Resets user data to zero / nullptr. + inline void resetUserData() noexcept { _userDataU64 = 0; } + + //! Tests whether the node has an associated pass data. + inline bool hasPassData() const noexcept { return _passData != nullptr; } + //! Returns the node pass data - data used during processing & transformations. + template + inline T* passData() const noexcept { return (T*)_passData; } + //! Sets the node pass data to `data`. + template + inline void setPassData(T* data) noexcept { _passData = (void*)data; } + //! Resets the node pass data to nullptr. + inline void resetPassData() noexcept { _passData = nullptr; } + + //! Tests whether the node has an inline comment/annotation. + inline bool hasInlineComment() const noexcept { return _inlineComment != nullptr; } + //! Returns an inline comment/annotation string. + inline const char* inlineComment() const noexcept { return _inlineComment; } + //! Sets an inline comment/annotation string to `s`. + inline void setInlineComment(const char* s) noexcept { _inlineComment = s; } + //! Resets an inline comment/annotation string to nullptr. + inline void resetInlineComment() noexcept { _inlineComment = nullptr; } + + //! \} +}; + +//! Instruction node. +//! +//! Wraps an instruction with its options and operands. +class InstNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(InstNode) + + //! \name Constants + //! \{ + + enum : uint32_t { + //! Count of embedded operands per `InstNode` that are always allocated as a part of the instruction. Minimum + //! embedded operands is 4, but in 32-bit more pointers are smaller and we can embed 5. The rest (up to 6 operands) + //! is always stored in `InstExNode`. + kBaseOpCapacity = uint32_t((128 - sizeof(BaseNode) - sizeof(BaseInst)) / sizeof(Operand_)) + }; + + //! \} + + //! \name Members + //! \{ + + //! Base instruction data. + BaseInst _baseInst; + //! First 4 or 5 operands (indexed from 0). + Operand_ _opArray[kBaseOpCapacity]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `InstNode` instance. + inline InstNode(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCount, uint32_t opCapacity = kBaseOpCapacity) noexcept + : BaseNode(cb, NodeType::kInst, NodeFlags::kIsCode | NodeFlags::kIsRemovable | NodeFlags::kActsAsInst), + _baseInst(instId, options) { + _inst._opCapacity = uint8_t(opCapacity); + _inst._opCount = uint8_t(opCount); + } + + //! \cond INTERNAL + //! Reset all built-in operands, including `extraReg`. + inline void _resetOps() noexcept { + _baseInst.resetExtraReg(); + resetOpRange(0, opCapacity()); + } + //! \endcond + + //! \} + + //! \name Instruction Object + //! \{ + + inline BaseInst& baseInst() noexcept { return _baseInst; } + inline const BaseInst& baseInst() const noexcept { return _baseInst; } + + //! \} + + //! \name Instruction Id + //! \{ + + //! Returns the instruction id, see `BaseInst::Id`. + inline InstId id() const noexcept { return _baseInst.id(); } + //! Returns the instruction real id, see `BaseInst::Id`. + inline InstId realId() const noexcept { return _baseInst.realId(); } + + //! Sets the instruction id to `id`, see `BaseInst::Id`. + inline void setId(InstId id) noexcept { _baseInst.setId(id); } + + //! \} + + //! \name Instruction Options + //! \{ + + inline InstOptions options() const noexcept { return _baseInst.options(); } + inline bool hasOption(InstOptions option) const noexcept { return _baseInst.hasOption(option); } + inline void setOptions(InstOptions options) noexcept { _baseInst.setOptions(options); } + inline void addOptions(InstOptions options) noexcept { _baseInst.addOptions(options); } + inline void clearOptions(InstOptions options) noexcept { _baseInst.clearOptions(options); } + inline void resetOptions() noexcept { _baseInst.resetOptions(); } + + //! \} + + //! \name Extra Register + //! \{ + + //! Tests whether the node has an extra register operand. + inline bool hasExtraReg() const noexcept { return _baseInst.hasExtraReg(); } + //! Returns extra register operand. + inline RegOnly& extraReg() noexcept { return _baseInst.extraReg(); } + //! \overload + inline const RegOnly& extraReg() const noexcept { return _baseInst.extraReg(); } + //! Sets extra register operand to `reg`. + inline void setExtraReg(const BaseReg& reg) noexcept { _baseInst.setExtraReg(reg); } + //! Sets extra register operand to `reg`. + inline void setExtraReg(const RegOnly& reg) noexcept { _baseInst.setExtraReg(reg); } + //! Resets extra register operand. + inline void resetExtraReg() noexcept { _baseInst.resetExtraReg(); } + + //! \} + + //! \name Instruction Operands + //! \{ + + //! Returns operand count. + inline uint32_t opCount() const noexcept { return _inst._opCount; } + //! Returns operand capacity. + inline uint32_t opCapacity() const noexcept { return _inst._opCapacity; } + + //! Sets operand count. + inline void setOpCount(uint32_t opCount) noexcept { _inst._opCount = uint8_t(opCount); } + + //! Returns operands array. + inline Operand* operands() noexcept { return (Operand*)_opArray; } + //! Returns operands array (const). + inline const Operand* operands() const noexcept { return (const Operand*)_opArray; } + + //! Returns operand at the given `index`. + inline Operand& op(uint32_t index) noexcept { + ASMJIT_ASSERT(index < opCapacity()); + return _opArray[index].as(); + } + + //! Returns operand at the given `index` (const). + inline const Operand& op(uint32_t index) const noexcept { + ASMJIT_ASSERT(index < opCapacity()); + return _opArray[index].as(); + } + + //! Sets operand at the given `index` to `op`. + inline void setOp(uint32_t index, const Operand_& op) noexcept { + ASMJIT_ASSERT(index < opCapacity()); + _opArray[index].copyFrom(op); + } + + //! Resets operand at the given `index` to none. + inline void resetOp(uint32_t index) noexcept { + ASMJIT_ASSERT(index < opCapacity()); + _opArray[index].reset(); + } + + //! Resets operands at `[start, end)` range. + inline void resetOpRange(uint32_t start, uint32_t end) noexcept { + for (uint32_t i = start; i < end; i++) + _opArray[i].reset(); + } + + //! \} + + //! \name Utilities + //! \{ + + inline bool hasOpType(OperandType opType) const noexcept { + for (uint32_t i = 0, count = opCount(); i < count; i++) + if (_opArray[i].opType() == opType) + return true; + return false; + } + + inline bool hasRegOp() const noexcept { return hasOpType(OperandType::kReg); } + inline bool hasMemOp() const noexcept { return hasOpType(OperandType::kMem); } + inline bool hasImmOp() const noexcept { return hasOpType(OperandType::kImm); } + inline bool hasLabelOp() const noexcept { return hasOpType(OperandType::kLabel); } + + inline uint32_t indexOfOpType(OperandType opType) const noexcept { + uint32_t i = 0; + uint32_t count = opCount(); + + while (i < count) { + if (_opArray[i].opType() == opType) + break; + i++; + } + + return i; + } + + inline uint32_t indexOfMemOp() const noexcept { return indexOfOpType(OperandType::kMem); } + inline uint32_t indexOfImmOp() const noexcept { return indexOfOpType(OperandType::kImm); } + inline uint32_t indexOfLabelOp() const noexcept { return indexOfOpType(OperandType::kLabel); } + + //! \} + + //! \name Rewriting + //! \{ + + //! \cond INTERNAL + inline uint32_t* _getRewriteArray() noexcept { return &_baseInst._extraReg._id; } + inline const uint32_t* _getRewriteArray() const noexcept { return &_baseInst._extraReg._id; } + + inline uint32_t getRewriteIndex(const uint32_t* id) const noexcept { + const uint32_t* array = _getRewriteArray(); + ASMJIT_ASSERT(array <= id); + + size_t index = (size_t)(id - array); + ASMJIT_ASSERT(index < 32); + + return uint32_t(index); + } + + inline void rewriteIdAtIndex(uint32_t index, uint32_t id) noexcept { + uint32_t* array = _getRewriteArray(); + array[index] = id; + } + //! \endcond + + //! \} + + //! \name Static Functions + //! \{ + + //! \cond INTERNAL + static inline uint32_t capacityOfOpCount(uint32_t opCount) noexcept { + return opCount <= kBaseOpCapacity ? kBaseOpCapacity : Globals::kMaxOpCount; + } + + static inline size_t nodeSizeOfOpCapacity(uint32_t opCapacity) noexcept { + size_t base = sizeof(InstNode) - kBaseOpCapacity * sizeof(Operand); + return base + opCapacity * sizeof(Operand); + } + //! \endcond + + //! \} +}; + +//! Instruction node with maximum number of operands. +//! +//! This node is created automatically by Builder/Compiler in case that the required number of operands exceeds +//! the default capacity of `InstNode`. +class InstExNode : public InstNode { +public: + ASMJIT_NONCOPYABLE(InstExNode) + + //! \name Members + //! \{ + + //! Continued `_opArray[]` to hold up to `kMaxOpCount` operands. + Operand_ _opArrayEx[Globals::kMaxOpCount - kBaseOpCapacity]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `InstExNode` instance. + inline InstExNode(BaseBuilder* cb, InstId instId, InstOptions options, uint32_t opCapacity = Globals::kMaxOpCount) noexcept + : InstNode(cb, instId, options, opCapacity) {} + + //! \} +}; + +//! Section node. +class SectionNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(SectionNode) + + //! \name Members + //! \{ + + //! Section id. + uint32_t _id; + + //! Next section node that follows this section. + //! + //! This link is only valid when the section is active (is part of the code) and when `Builder::hasDirtySectionLinks()` + //! returns `false`. If you intend to use this field you should always call `Builder::updateSectionLinks()` before you + //! do so. + SectionNode* _nextSection; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `SectionNode` instance. + inline SectionNode(BaseBuilder* cb, uint32_t secionId = 0) noexcept + : BaseNode(cb, NodeType::kSection, NodeFlags::kHasNoEffect), + _id(secionId), + _nextSection(nullptr) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the section id. + inline uint32_t id() const noexcept { return _id; } + + //! \} +}; + +//! Label node. +class LabelNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(LabelNode) + + //! \name Members + //! \{ + + //! Label identifier. + uint32_t _labelId; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `LabelNode` instance. + inline LabelNode(BaseBuilder* cb, uint32_t labelId = 0) noexcept + : BaseNode(cb, NodeType::kLabel, NodeFlags::kHasNoEffect | NodeFlags::kActsAsLabel), + _labelId(labelId) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns \ref Label representation of the \ref LabelNode. + inline Label label() const noexcept { return Label(_labelId); } + //! Returns the id of the label. + inline uint32_t labelId() const noexcept { return _labelId; } + + //! \} +}; + +//! Align directive (BaseBuilder). +//! +//! Wraps `.align` directive. +class AlignNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(AlignNode) + + //! \name Members + //! \{ + + //! Alignment (in bytes). + uint32_t _alignment; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `AlignNode` instance. + inline AlignNode(BaseBuilder* cb, AlignMode alignMode, uint32_t alignment) noexcept + : BaseNode(cb, NodeType::kAlign, NodeFlags::kIsCode | NodeFlags::kHasNoEffect) { + + _alignData._alignMode = alignMode; + _alignment = alignment; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns align mode. + inline AlignMode alignMode() const noexcept { return _alignData._alignMode; } + //! Sets align mode to `alignMode`. + inline void setAlignMode(AlignMode alignMode) noexcept { _alignData._alignMode = alignMode; } + + //! Returns align offset in bytes. + inline uint32_t alignment() const noexcept { return _alignment; } + //! Sets align offset in bytes to `offset`. + inline void setAlignment(uint32_t alignment) noexcept { _alignment = alignment; } + + //! \} +}; + +//! Embed data node. +//! +//! Wraps `.data` directive. The node contains data that will be placed at the node's position in the assembler +//! stream. The data is considered to be RAW; no analysis nor byte-order conversion is performed on RAW data. +class EmbedDataNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(EmbedDataNode) + + //! \cond INTERNAL + enum : uint32_t { + kInlineBufferSize = 128 - (sizeof(BaseNode) + sizeof(size_t) * 2) + }; + //! \endcond + + //! \name Members + //! \{ + + size_t _itemCount; + size_t _repeatCount; + + union { + uint8_t* _externalData; + uint8_t _inlineData[kInlineBufferSize]; + }; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `EmbedDataNode` instance. + inline EmbedDataNode(BaseBuilder* cb) noexcept + : BaseNode(cb, NodeType::kEmbedData, NodeFlags::kIsData), + _itemCount(0), + _repeatCount(0) { + _embed._typeId = TypeId::kUInt8; + _embed._typeSize = uint8_t(1); + memset(_inlineData, 0, kInlineBufferSize); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns data type as \ref TypeId. + inline TypeId typeId() const noexcept { return _embed._typeId; } + //! Returns the size of a single data element. + inline uint32_t typeSize() const noexcept { return _embed._typeSize; } + + //! Returns a pointer to the data casted to `uint8_t`. + inline uint8_t* data() const noexcept { + return dataSize() <= kInlineBufferSize ? const_cast(_inlineData) : _externalData; + } + + //! Returns a pointer to the data casted to `T`. + template + inline T* dataAs() const noexcept { return reinterpret_cast(data()); } + + //! Returns the number of (typed) items in the array. + inline size_t itemCount() const noexcept { return _itemCount; } + + //! Returns how many times the data is repeated (default 1). + //! + //! Repeated data is useful when defining constants for SIMD, for example. + inline size_t repeatCount() const noexcept { return _repeatCount; } + + //! Returns the size of the data, not considering the number of times it repeats. + //! + //! \note The returned value is the same as `typeSize() * itemCount()`. + inline size_t dataSize() const noexcept { return typeSize() * _itemCount; } + + //! \} +}; + +//! Label data node. +class EmbedLabelNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(EmbedLabelNode) + + //! \name Members + //! \{ + + uint32_t _labelId; + uint32_t _dataSize; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `EmbedLabelNode` instance. + inline EmbedLabelNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(cb, NodeType::kEmbedLabel, NodeFlags::kIsData), + _labelId(labelId), + _dataSize(dataSize) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the label to embed as \ref Label operand. + inline Label label() const noexcept { return Label(_labelId); } + //! Returns the id of the label. + inline uint32_t labelId() const noexcept { return _labelId; } + + //! Sets the label id from `label` operand. + inline void setLabel(const Label& label) noexcept { setLabelId(label.id()); } + //! Sets the label id (use with caution, improper use can break a lot of things). + inline void setLabelId(uint32_t labelId) noexcept { _labelId = labelId; } + + //! Returns the data size. + inline uint32_t dataSize() const noexcept { return _dataSize; } + //! Sets the data size. + inline void setDataSize(uint32_t dataSize) noexcept { _dataSize = dataSize; } + + //! \} +}; + +//! Label data node. +class EmbedLabelDeltaNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(EmbedLabelDeltaNode) + + //! \name Members + //! \{ + + uint32_t _labelId; + uint32_t _baseLabelId; + uint32_t _dataSize; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `EmbedLabelDeltaNode` instance. + inline EmbedLabelDeltaNode(BaseBuilder* cb, uint32_t labelId = 0, uint32_t baseLabelId = 0, uint32_t dataSize = 0) noexcept + : BaseNode(cb, NodeType::kEmbedLabelDelta, NodeFlags::kIsData), + _labelId(labelId), + _baseLabelId(baseLabelId), + _dataSize(dataSize) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the label as `Label` operand. + inline Label label() const noexcept { return Label(_labelId); } + //! Returns the id of the label. + inline uint32_t labelId() const noexcept { return _labelId; } + + //! Sets the label id from `label` operand. + inline void setLabel(const Label& label) noexcept { setLabelId(label.id()); } + //! Sets the label id. + inline void setLabelId(uint32_t labelId) noexcept { _labelId = labelId; } + + //! Returns the base label as `Label` operand. + inline Label baseLabel() const noexcept { return Label(_baseLabelId); } + //! Returns the id of the base label. + inline uint32_t baseLabelId() const noexcept { return _baseLabelId; } + + //! Sets the base label id from `label` operand. + inline void setBaseLabel(const Label& baseLabel) noexcept { setBaseLabelId(baseLabel.id()); } + //! Sets the base label id. + inline void setBaseLabelId(uint32_t baseLabelId) noexcept { _baseLabelId = baseLabelId; } + + //! Returns the size of the embedded label address. + inline uint32_t dataSize() const noexcept { return _dataSize; } + //! Sets the size of the embedded label address. + inline void setDataSize(uint32_t dataSize) noexcept { _dataSize = dataSize; } + + //! \} +}; + +//! A node that wraps `ConstPool`. +class ConstPoolNode : public LabelNode { +public: + ASMJIT_NONCOPYABLE(ConstPoolNode) + + //! \name Members + //! \{ + + ConstPool _constPool; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `ConstPoolNode` instance. + inline ConstPoolNode(BaseBuilder* cb, uint32_t id = 0) noexcept + : LabelNode(cb, id), + _constPool(&cb->_codeZone) { + + setType(NodeType::kConstPool); + addFlags(NodeFlags::kIsData); + clearFlags(NodeFlags::kIsCode | NodeFlags::kHasNoEffect); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the constant-pool is empty. + inline bool empty() const noexcept { return _constPool.empty(); } + //! Returns the size of the constant-pool in bytes. + inline size_t size() const noexcept { return _constPool.size(); } + //! Returns minimum alignment. + inline size_t alignment() const noexcept { return _constPool.alignment(); } + + //! Returns the wrapped `ConstPool` instance. + inline ConstPool& constPool() noexcept { return _constPool; } + //! Returns the wrapped `ConstPool` instance (const). + inline const ConstPool& constPool() const noexcept { return _constPool; } + + //! \} + + //! \name Utilities + //! \{ + + //! See `ConstPool::add()`. + inline Error add(const void* data, size_t size, size_t& dstOffset) noexcept { + return _constPool.add(data, size, dstOffset); + } + + //! \} +}; + +//! Comment node. +class CommentNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(CommentNode) + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `CommentNode` instance. + inline CommentNode(BaseBuilder* cb, const char* comment) noexcept + : BaseNode(cb, NodeType::kComment, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect | NodeFlags::kIsRemovable) { + _inlineComment = comment; + } + + //! \} +}; + +//! Sentinel node. +//! +//! Sentinel is a marker that is completely ignored by the code builder. It's used to remember a position in a code +//! as it never gets removed by any pass. +class SentinelNode : public BaseNode { +public: + ASMJIT_NONCOPYABLE(SentinelNode) + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `SentinelNode` instance. + inline SentinelNode(BaseBuilder* cb, SentinelType sentinelType = SentinelType::kUnknown) noexcept + : BaseNode(cb, NodeType::kSentinel, NodeFlags::kIsInformative | NodeFlags::kHasNoEffect) { + + _sentinel._sentinelType = sentinelType; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the type of the sentinel. + inline SentinelType sentinelType() const noexcept { + return _sentinel._sentinelType; + } + + //! Sets the type of the sentinel. + inline void setSentinelType(SentinelType type) noexcept { + _sentinel._sentinelType = type; + } + + //! \} +}; + +//! Pass can be used to implement code transformations, analysis, and lowering. +class ASMJIT_VIRTAPI Pass { +public: + ASMJIT_BASE_CLASS(Pass) + ASMJIT_NONCOPYABLE(Pass) + + //! \name Members + //! \{ + + //! BaseBuilder this pass is assigned to. + BaseBuilder* _cb = nullptr; + //! Name of the pass. + const char* _name = nullptr; + + //! \} + + //! \name Construction & Destruction + //! \{ + + ASMJIT_API Pass(const char* name) noexcept; + ASMJIT_API virtual ~Pass() noexcept; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns \ref BaseBuilder associated with the pass. + inline const BaseBuilder* cb() const noexcept { return _cb; } + //! Returns the name of the pass. + inline const char* name() const noexcept { return _name; } + + //! \} + + //! \name Pass Interface + //! \{ + + //! Processes the code stored in Builder or Compiler. + //! + //! This is the only function that is called by the `BaseBuilder` to process the code. It passes `zone`, + //! which will be reset after the `run()` finishes. + virtual Error run(Zone* zone, Logger* logger) = 0; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // !ASMJIT_NO_BUILDER +#endif // ASMJIT_CORE_BUILDER_H_INCLUDED diff --git a/src/asmjit/core/codebuffer.h b/src/asmjit/core/codebuffer.h new file mode 100644 index 0000000..2fe35a9 --- /dev/null +++ b/src/asmjit/core/codebuffer.h @@ -0,0 +1,113 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_CODEBUFFER_H_INCLUDED +#define ASMJIT_CORE_CODEBUFFER_H_INCLUDED + +#include "../core/globals.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! Flags used by \ref CodeBuffer. +enum class CodeBufferFlags : uint32_t { + //! No flags. + kNone = 0, + //! Buffer is external (not allocated by asmjit). + kIsExternal = 0x00000001u, + //! Buffer is fixed (cannot be reallocated). + kIsFixed = 0x00000002u +}; +ASMJIT_DEFINE_ENUM_FLAGS(CodeBufferFlags) + +//! Code or data buffer. +struct CodeBuffer { + //! \name Members + //! \{ + + //! The content of the buffer (data). + uint8_t* _data; + //! Number of bytes of `data` used. + size_t _size; + //! Buffer capacity (in bytes). + size_t _capacity; + //! Buffer flags. + CodeBufferFlags _flags; + + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Returns a reference to the byte at the given `index`. + inline uint8_t& operator[](size_t index) noexcept { + ASMJIT_ASSERT(index < _size); + return _data[index]; + } + //! \overload + inline const uint8_t& operator[](size_t index) const noexcept { + ASMJIT_ASSERT(index < _size); + return _data[index]; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns code buffer flags. + inline CodeBufferFlags flags() const noexcept { return _flags; } + //! Tests whether the code buffer has the given `flag` set. + inline bool hasFlag(CodeBufferFlags flag) const noexcept { return Support::test(_flags, flag); } + + //! Tests whether this code buffer has a fixed size. + //! + //! Fixed size means that the code buffer is fixed and cannot grow. + inline bool isFixed() const noexcept { return hasFlag(CodeBufferFlags::kIsFixed); } + + //! Tests whether the data in this code buffer is external. + //! + //! External data can only be provided by users, it's never used by AsmJit. + inline bool isExternal() const noexcept { return hasFlag(CodeBufferFlags::kIsExternal); } + + //! Tests whether the data in this code buffer is allocated (non-null). + inline bool isAllocated() const noexcept { return _data != nullptr; } + + //! Tests whether the code buffer is empty. + inline bool empty() const noexcept { return !_size; } + + //! Returns the size of the data. + inline size_t size() const noexcept { return _size; } + //! Returns the capacity of the data. + inline size_t capacity() const noexcept { return _capacity; } + + //! Returns the pointer to the data the buffer references. + inline uint8_t* data() noexcept { return _data; } + //! \overload + inline const uint8_t* data() const noexcept { return _data; } + + //! \} + + //! \name Iterators + //! \{ + + inline uint8_t* begin() noexcept { return _data; } + inline const uint8_t* begin() const noexcept { return _data; } + + inline uint8_t* end() noexcept { return _data + _size; } + inline const uint8_t* end() const noexcept { return _data + _size; } + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_CODEBUFFER_H_INCLUDED + diff --git a/src/asmjit/core/codeholder.cpp b/src/asmjit/core/codeholder.cpp new file mode 100644 index 0000000..cf763cf --- /dev/null +++ b/src/asmjit/core/codeholder.cpp @@ -0,0 +1,1149 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/assembler.h" +#include "../core/codewriter_p.h" +#include "../core/logger.h" +#include "../core/support.h" + +#include +#include + +ASMJIT_BEGIN_NAMESPACE + +// Globals +// ======= + +static const char CodeHolder_addrTabName[] = ".addrtab"; + +//! Encode MOD byte. +static inline uint32_t x86EncodeMod(uint32_t m, uint32_t o, uint32_t rm) noexcept { + return (m << 6) | (o << 3) | rm; +} + +// LabelLinkIterator +// ================= + +class LabelLinkIterator { +public: + inline LabelLinkIterator(LabelEntry* le) noexcept { reset(le); } + + inline explicit operator bool() const noexcept { return isValid(); } + inline bool isValid() const noexcept { return _link != nullptr; } + + inline LabelLink* link() const noexcept { return _link; } + inline LabelLink* operator->() const noexcept { return _link; } + + inline void reset(LabelEntry* le) noexcept { + _pPrev = &le->_links; + _link = *_pPrev; + } + + inline void next() noexcept { + _pPrev = &_link->next; + _link = *_pPrev; + } + + inline void resolveAndNext(CodeHolder* code) noexcept { + LabelLink* linkToDelete = _link; + + _link = _link->next; + *_pPrev = _link; + + code->_unresolvedLinkCount--; + code->_allocator.release(linkToDelete, sizeof(LabelLink)); + } + + LabelLink** _pPrev; + LabelLink* _link; +}; + +// CodeHolder - Utilities +// ====================== + +static void CodeHolder_resetInternal(CodeHolder* self, ResetPolicy resetPolicy) noexcept { + uint32_t i; + const ZoneVector& emitters = self->emitters(); + + i = emitters.size(); + while (i) + self->detach(emitters[--i]); + + // Reset everything into its construction state. + self->_environment.reset(); + self->_baseAddress = Globals::kNoBaseAddress; + self->_logger = nullptr; + self->_errorHandler = nullptr; + + // Reset all sections. + uint32_t numSections = self->_sections.size(); + for (i = 0; i < numSections; i++) { + Section* section = self->_sections[i]; + if (section->_buffer.data() && !section->_buffer.isExternal()) + ::free(section->_buffer._data); + section->_buffer._data = nullptr; + section->_buffer._capacity = 0; + } + + // Reset zone allocator and all containers using it. + ZoneAllocator* allocator = self->allocator(); + + self->_emitters.reset(); + self->_namedLabels.reset(); + self->_relocations.reset(); + self->_labelEntries.reset(); + self->_sections.reset(); + self->_sectionsByOrder.reset(); + + self->_unresolvedLinkCount = 0; + self->_addressTableSection = nullptr; + self->_addressTableEntries.reset(); + + allocator->reset(&self->_zone); + self->_zone.reset(resetPolicy); +} + +static void CodeHolder_onSettingsUpdated(CodeHolder* self) noexcept { + // Notify all attached emitters about a settings update. + for (BaseEmitter* emitter : self->emitters()) { + emitter->onSettingsUpdated(); + } +} + +// CodeHolder - Construction & Destruction +// ======================================= + +CodeHolder::CodeHolder(const Support::Temporary* temporary) noexcept + : _environment(), + _baseAddress(Globals::kNoBaseAddress), + _logger(nullptr), + _errorHandler(nullptr), + _zone(16384 - Zone::kBlockOverhead, 1, temporary), + _allocator(&_zone), + _unresolvedLinkCount(0), + _addressTableSection(nullptr) {} + +CodeHolder::~CodeHolder() noexcept { + CodeHolder_resetInternal(this, ResetPolicy::kHard); +} + +// CodeHolder - Init & Reset +// ========================= + +inline void CodeHolder_setSectionDefaultName( + Section* section, + char c0 = 0, char c1 = 0, char c2 = 0, char c3 = 0, + char c4 = 0, char c5 = 0, char c6 = 0, char c7 = 0) noexcept { + + section->_name.u32[0] = Support::bytepack32_4x8(uint8_t(c0), uint8_t(c1), uint8_t(c2), uint8_t(c3)); + section->_name.u32[1] = Support::bytepack32_4x8(uint8_t(c4), uint8_t(c5), uint8_t(c6), uint8_t(c7)); +} + +Error CodeHolder::init(const Environment& environment, uint64_t baseAddress) noexcept { + // Cannot reinitialize if it's locked or there is one or more emitter attached. + if (isInitialized()) + return DebugUtils::errored(kErrorAlreadyInitialized); + + // If we are just initializing there should be no emitters attached. + ASMJIT_ASSERT(_emitters.empty()); + + // Create a default section and insert it to the `_sections` array. + Error err = _sections.willGrow(&_allocator) | + _sectionsByOrder.willGrow(&_allocator); + if (err == kErrorOk) { + Section* section = _allocator.allocZeroedT
(); + if (ASMJIT_LIKELY(section)) { + section->_flags = SectionFlags::kExecutable | SectionFlags::kReadOnly; + CodeHolder_setSectionDefaultName(section, '.', 't', 'e', 'x', 't'); + _sections.appendUnsafe(section); + _sectionsByOrder.appendUnsafe(section); + } + else { + err = DebugUtils::errored(kErrorOutOfMemory); + } + } + + if (ASMJIT_UNLIKELY(err)) { + _zone.reset(); + return err; + } + else { + _environment = environment; + _baseAddress = baseAddress; + return kErrorOk; + } +} + +void CodeHolder::reset(ResetPolicy resetPolicy) noexcept { + CodeHolder_resetInternal(this, resetPolicy); +} + +// CodeHolder - Attach / Detach +// ============================ + +Error CodeHolder::attach(BaseEmitter* emitter) noexcept { + // Catch a possible misuse of the API. + if (ASMJIT_UNLIKELY(!emitter)) + return DebugUtils::errored(kErrorInvalidArgument); + + // Invalid emitter, this should not be possible. + EmitterType type = emitter->emitterType(); + if (ASMJIT_UNLIKELY(type == EmitterType::kNone || uint32_t(type) > uint32_t(EmitterType::kMaxValue))) + return DebugUtils::errored(kErrorInvalidState); + + uint64_t archMask = emitter->_archMask; + if (ASMJIT_UNLIKELY(!(archMask & (uint64_t(1) << uint32_t(arch()))))) + return DebugUtils::errored(kErrorInvalidArch); + + // This is suspicious, but don't fail if `emitter` is already attached + // to this code holder. This is not error, but it's not recommended. + if (emitter->_code != nullptr) { + if (emitter->_code == this) + return kErrorOk; + return DebugUtils::errored(kErrorInvalidState); + } + + // Reserve the space now as we cannot fail after `onAttach()` succeeded. + ASMJIT_PROPAGATE(_emitters.willGrow(&_allocator, 1)); + ASMJIT_PROPAGATE(emitter->onAttach(this)); + + // Connect CodeHolder <-> BaseEmitter. + ASMJIT_ASSERT(emitter->_code == this); + _emitters.appendUnsafe(emitter); + + return kErrorOk; +} + +Error CodeHolder::detach(BaseEmitter* emitter) noexcept { + if (ASMJIT_UNLIKELY(!emitter)) + return DebugUtils::errored(kErrorInvalidArgument); + + if (ASMJIT_UNLIKELY(emitter->_code != this)) + return DebugUtils::errored(kErrorInvalidState); + + // NOTE: We always detach if we were asked to, if error happens during + // `emitter->onDetach()` we just propagate it, but the BaseEmitter will + // be detached. + Error err = kErrorOk; + if (!emitter->isDestroyed()) + err = emitter->onDetach(this); + + // Disconnect CodeHolder <-> BaseEmitter. + uint32_t index = _emitters.indexOf(emitter); + ASMJIT_ASSERT(index != Globals::kNotFound); + + _emitters.removeAt(index); + emitter->_code = nullptr; + + return err; +} + +// CodeHolder - Logging +// ==================== + +void CodeHolder::setLogger(Logger* logger) noexcept { +#ifndef ASMJIT_NO_LOGGING + _logger = logger; + CodeHolder_onSettingsUpdated(this); +#else + DebugUtils::unused(logger); +#endif +} + +// CodeHolder - Error Handling +// =========================== + +void CodeHolder::setErrorHandler(ErrorHandler* errorHandler) noexcept { + _errorHandler = errorHandler; + CodeHolder_onSettingsUpdated(this); +} + +// CodeHolder - Code Buffer +// ======================== + +static Error CodeHolder_reserveInternal(CodeHolder* self, CodeBuffer* cb, size_t n) noexcept { + uint8_t* oldData = cb->_data; + uint8_t* newData; + + if (oldData && !cb->isExternal()) + newData = static_cast(::realloc(oldData, n)); + else + newData = static_cast(::malloc(n)); + + if (ASMJIT_UNLIKELY(!newData)) + return DebugUtils::errored(kErrorOutOfMemory); + + cb->_data = newData; + cb->_capacity = n; + + // Update pointers used by assemblers, if attached. + for (BaseEmitter* emitter : self->emitters()) { + if (emitter->isAssembler()) { + BaseAssembler* a = static_cast(emitter); + if (&a->_section->_buffer == cb) { + size_t offset = a->offset(); + + a->_bufferData = newData; + a->_bufferEnd = newData + n; + a->_bufferPtr = newData + offset; + } + } + } + + return kErrorOk; +} + +Error CodeHolder::growBuffer(CodeBuffer* cb, size_t n) noexcept { + // The size of the section must be valid. + size_t size = cb->size(); + if (ASMJIT_UNLIKELY(n > std::numeric_limits::max() - size)) + return DebugUtils::errored(kErrorOutOfMemory); + + // We can now check if growing the buffer is really necessary. It's unlikely + // that this function is called while there is still room for `n` bytes. + size_t capacity = cb->capacity(); + size_t required = cb->size() + n; + if (ASMJIT_UNLIKELY(required <= capacity)) + return kErrorOk; + + if (cb->isFixed()) + return DebugUtils::errored(kErrorTooLarge); + + size_t kInitialCapacity = 8096; + if (capacity < kInitialCapacity) + capacity = kInitialCapacity; + else + capacity += Globals::kAllocOverhead; + + do { + size_t old = capacity; + if (capacity < Globals::kGrowThreshold) + capacity *= 2; + else + capacity += Globals::kGrowThreshold; + + // Overflow. + if (ASMJIT_UNLIKELY(old > capacity)) + return DebugUtils::errored(kErrorOutOfMemory); + } while (capacity - Globals::kAllocOverhead < required); + + return CodeHolder_reserveInternal(this, cb, capacity - Globals::kAllocOverhead); +} + +Error CodeHolder::reserveBuffer(CodeBuffer* cb, size_t n) noexcept { + size_t capacity = cb->capacity(); + + if (n <= capacity) + return kErrorOk; + + if (cb->isFixed()) + return DebugUtils::errored(kErrorTooLarge); + + return CodeHolder_reserveInternal(this, cb, n); +} + +// CodeHolder - Sections +// ===================== + +Error CodeHolder::newSection(Section** sectionOut, const char* name, size_t nameSize, SectionFlags flags, uint32_t alignment, int32_t order) noexcept { + *sectionOut = nullptr; + + if (nameSize == SIZE_MAX) + nameSize = strlen(name); + + if (alignment == 0) + alignment = 1; + + if (ASMJIT_UNLIKELY(!Support::isPowerOf2(alignment))) + return DebugUtils::errored(kErrorInvalidArgument); + + if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxSectionNameSize)) + return DebugUtils::errored(kErrorInvalidSectionName); + + uint32_t sectionId = _sections.size(); + if (ASMJIT_UNLIKELY(sectionId == Globals::kInvalidId)) + return DebugUtils::errored(kErrorTooManySections); + + ASMJIT_PROPAGATE(_sections.willGrow(&_allocator)); + ASMJIT_PROPAGATE(_sectionsByOrder.willGrow(&_allocator)); + + Section* section = _allocator.allocZeroedT
(); + if (ASMJIT_UNLIKELY(!section)) + return DebugUtils::errored(kErrorOutOfMemory); + + section->_id = sectionId; + section->_flags = flags; + section->_alignment = alignment; + section->_order = order; + memcpy(section->_name.str, name, nameSize); + + Section** insertPosition = std::lower_bound(_sectionsByOrder.begin(), _sectionsByOrder.end(), section, [](const Section* a, const Section* b) { + return std::make_tuple(a->order(), a->id()) < std::make_tuple(b->order(), b->id()); + }); + + _sections.appendUnsafe(section); + _sectionsByOrder.insertUnsafe((size_t)(insertPosition - _sectionsByOrder.data()), section); + + *sectionOut = section; + return kErrorOk; +} + +Section* CodeHolder::sectionByName(const char* name, size_t nameSize) const noexcept { + if (nameSize == SIZE_MAX) + nameSize = strlen(name); + + // This could be also put in a hash-table similarly like we do with labels, + // however it's questionable as the number of sections should be pretty low + // in general. Create an issue if this becomes a problem. + if (nameSize <= Globals::kMaxSectionNameSize) { + for (Section* section : _sections) + if (memcmp(section->_name.str, name, nameSize) == 0 && section->_name.str[nameSize] == '\0') + return section; + } + + return nullptr; +} + +Section* CodeHolder::ensureAddressTableSection() noexcept { + if (_addressTableSection) + return _addressTableSection; + + newSection(&_addressTableSection, + CodeHolder_addrTabName, + sizeof(CodeHolder_addrTabName) - 1, + SectionFlags::kNone, + _environment.registerSize(), + std::numeric_limits::max()); + return _addressTableSection; +} + +Error CodeHolder::addAddressToAddressTable(uint64_t address) noexcept { + AddressTableEntry* entry = _addressTableEntries.get(address); + if (entry) + return kErrorOk; + + Section* section = ensureAddressTableSection(); + if (ASMJIT_UNLIKELY(!section)) + return DebugUtils::errored(kErrorOutOfMemory); + + entry = _zone.newT(address); + if (ASMJIT_UNLIKELY(!entry)) + return DebugUtils::errored(kErrorOutOfMemory); + + _addressTableEntries.insert(entry); + section->_virtualSize += _environment.registerSize(); + + return kErrorOk; +} + +// CodeHolder - Labels & Symbols +// ============================= + +//! Only used to lookup a label from `_namedLabels`. +class LabelByName { +public: + inline LabelByName(const char* key, size_t keySize, uint32_t hashCode, uint32_t parentId) noexcept + : _key(key), + _keySize(uint32_t(keySize)), + _hashCode(hashCode), + _parentId(parentId) {} + + inline uint32_t hashCode() const noexcept { return _hashCode; } + + inline bool matches(const LabelEntry* entry) const noexcept { + return entry->nameSize() == _keySize && + entry->parentId() == _parentId && + ::memcmp(entry->name(), _key, _keySize) == 0; + } + + const char* _key; + uint32_t _keySize; + uint32_t _hashCode; + uint32_t _parentId; +}; + +// Returns a hash of `name` and fixes `nameSize` if it's `SIZE_MAX`. +static uint32_t CodeHolder_hashNameAndGetSize(const char* name, size_t& nameSize) noexcept { + uint32_t hashCode = 0; + if (nameSize == SIZE_MAX) { + size_t i = 0; + for (;;) { + uint8_t c = uint8_t(name[i]); + if (!c) break; + hashCode = Support::hashRound(hashCode, c); + i++; + } + nameSize = i; + } + else { + for (size_t i = 0; i < nameSize; i++) { + uint8_t c = uint8_t(name[i]); + if (ASMJIT_UNLIKELY(!c)) return DebugUtils::errored(kErrorInvalidLabelName); + hashCode = Support::hashRound(hashCode, c); + } + } + return hashCode; +} + +LabelLink* CodeHolder::newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept { + LabelLink* link = _allocator.allocT(); + if (ASMJIT_UNLIKELY(!link)) return nullptr; + + link->next = le->_links; + le->_links = link; + + link->sectionId = sectionId; + link->relocId = Globals::kInvalidId; + link->offset = offset; + link->rel = rel; + link->format = format; + + _unresolvedLinkCount++; + return link; +} + +Error CodeHolder::newLabelEntry(LabelEntry** entryOut) noexcept { + *entryOut = nullptr; + + uint32_t labelId = _labelEntries.size(); + if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) + return DebugUtils::errored(kErrorTooManyLabels); + + ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); + LabelEntry* le = _allocator.allocZeroedT(); + + if (ASMJIT_UNLIKELY(!le)) + return DebugUtils::errored(kErrorOutOfMemory); + + le->_setId(labelId); + le->_parentId = Globals::kInvalidId; + le->_offset = 0; + _labelEntries.appendUnsafe(le); + + *entryOut = le; + return kErrorOk; +} + +Error CodeHolder::newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId) noexcept { + *entryOut = nullptr; + uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize); + + if (ASMJIT_UNLIKELY(nameSize == 0)) { + if (type == LabelType::kAnonymous) + return newLabelEntry(entryOut); + else + return DebugUtils::errored(kErrorInvalidLabelName); + } + + if (ASMJIT_UNLIKELY(nameSize > Globals::kMaxLabelNameSize)) + return DebugUtils::errored(kErrorLabelNameTooLong); + + switch (type) { + case LabelType::kAnonymous: { + // Anonymous labels cannot have a parent (or more specifically, parent is useless here). + if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) + return DebugUtils::errored(kErrorInvalidParentLabel); + + uint32_t labelId = _labelEntries.size(); + if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) + return DebugUtils::errored(kErrorTooManyLabels); + + ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); + LabelEntry* le = _allocator.allocZeroedT(); + + if (ASMJIT_UNLIKELY(!le)) + return DebugUtils::errored(kErrorOutOfMemory); + + // NOTE: This LabelEntry has a name, but we leave its hashCode as zero as it's anonymous. + le->_setId(labelId); + le->_parentId = Globals::kInvalidId; + le->_offset = 0; + ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + + _labelEntries.appendUnsafe(le); + + *entryOut = le; + return kErrorOk; + } + + case LabelType::kLocal: { + if (ASMJIT_UNLIKELY(parentId >= _labelEntries.size())) + return DebugUtils::errored(kErrorInvalidParentLabel); + + hashCode ^= parentId; + break; + } + + case LabelType::kGlobal: + case LabelType::kExternal: { + if (ASMJIT_UNLIKELY(parentId != Globals::kInvalidId)) + return DebugUtils::errored(kErrorInvalidParentLabel); + break; + } + + default: { + return DebugUtils::errored(kErrorInvalidArgument); + } + } + + // Don't allow to insert duplicates. Local labels allow duplicates that have + // different id, this is already accomplished by having a different hashes + // between the same label names having different parent labels. + LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + if (ASMJIT_UNLIKELY(le)) + return DebugUtils::errored(kErrorLabelAlreadyDefined); + + Error err = kErrorOk; + uint32_t labelId = _labelEntries.size(); + + if (ASMJIT_UNLIKELY(labelId == Globals::kInvalidId)) + return DebugUtils::errored(kErrorTooManyLabels); + + ASMJIT_PROPAGATE(_labelEntries.willGrow(&_allocator)); + le = _allocator.allocZeroedT(); + + if (ASMJIT_UNLIKELY(!le)) + return DebugUtils::errored(kErrorOutOfMemory); + + le->_hashCode = hashCode; + le->_setId(labelId); + le->_type = type; + le->_parentId = parentId; + le->_offset = 0; + ASMJIT_PROPAGATE(le->_name.setData(&_zone, name, nameSize)); + + _labelEntries.appendUnsafe(le); + _namedLabels.insert(allocator(), le); + + *entryOut = le; + return err; +} + +uint32_t CodeHolder::labelIdByName(const char* name, size_t nameSize, uint32_t parentId) noexcept { + uint32_t hashCode = CodeHolder_hashNameAndGetSize(name, nameSize); + if (ASMJIT_UNLIKELY(!nameSize)) + return 0; + + if (parentId != Globals::kInvalidId) + hashCode ^= parentId; + + LabelEntry* le = _namedLabels.get(LabelByName(name, nameSize, hashCode, parentId)); + return le ? le->id() : uint32_t(Globals::kInvalidId); +} + +ASMJIT_API Error CodeHolder::resolveUnresolvedLinks() noexcept { + if (!hasUnresolvedLinks()) + return kErrorOk; + + Error err = kErrorOk; + for (LabelEntry* le : labelEntries()) { + if (!le->isBound()) + continue; + + LabelLinkIterator link(le); + if (link) { + Support::FastUInt8 of = 0; + Section* toSection = le->section(); + uint64_t toOffset = Support::addOverflow(toSection->offset(), le->offset(), &of); + + do { + uint32_t linkSectionId = link->sectionId; + if (link->relocId == Globals::kInvalidId) { + Section* fromSection = sectionById(linkSectionId); + size_t linkOffset = link->offset; + + CodeBuffer& buf = _sections[linkSectionId]->buffer(); + ASMJIT_ASSERT(linkOffset < buf.size()); + + // Calculate the offset relative to the start of the virtual base. + Support::FastUInt8 localOF = of; + uint64_t fromOffset = Support::addOverflow(fromSection->offset(), linkOffset, &localOF); + int64_t displacement = int64_t(toOffset - fromOffset + uint64_t(int64_t(link->rel))); + + if (!localOF) { + ASMJIT_ASSERT(size_t(linkOffset) < buf.size()); + ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.valueSize()); + + // Overwrite a real displacement in the CodeBuffer. + if (CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { + link.resolveAndNext(this); + continue; + } + } + + err = DebugUtils::errored(kErrorInvalidDisplacement); + // Falls through to `link.next()`. + } + + link.next(); + } while (link); + } + } + + return err; +} + +ASMJIT_API Error CodeHolder::bindLabel(const Label& label, uint32_t toSectionId, uint64_t toOffset) noexcept { + LabelEntry* le = labelEntry(label); + if (ASMJIT_UNLIKELY(!le)) + return DebugUtils::errored(kErrorInvalidLabel); + + if (ASMJIT_UNLIKELY(toSectionId > _sections.size())) + return DebugUtils::errored(kErrorInvalidSection); + + // Label can be bound only once. + if (ASMJIT_UNLIKELY(le->isBound())) + return DebugUtils::errored(kErrorLabelAlreadyBound); + + // Bind the label. + Section* section = _sections[toSectionId]; + le->_section = section; + le->_offset = toOffset; + + Error err = kErrorOk; + CodeBuffer& buf = section->buffer(); + + // Fix all links to this label we have collected so far if they are within + // the same section. We ignore any inter-section links as these have to be + // fixed later. + LabelLinkIterator link(le); + while (link) { + uint32_t linkSectionId = link->sectionId; + size_t linkOffset = link->offset; + + uint32_t relocId = link->relocId; + if (relocId != Globals::kInvalidId) { + // Adjust relocation data only. + RelocEntry* re = _relocations[relocId]; + re->_payload += toOffset; + re->_targetSectionId = toSectionId; + } + else { + if (linkSectionId != toSectionId) { + link.next(); + continue; + } + + ASMJIT_ASSERT(linkOffset < buf.size()); + int64_t displacement = int64_t(toOffset - uint64_t(linkOffset) + uint64_t(int64_t(link->rel))); + + // Size of the value we are going to patch. Only BYTE/DWORD is allowed. + ASMJIT_ASSERT(buf.size() - size_t(linkOffset) >= link->format.regionSize()); + + // Overwrite a real displacement in the CodeBuffer. + if (!CodeWriterUtils::writeOffset(buf._data + linkOffset, displacement, link->format)) { + err = DebugUtils::errored(kErrorInvalidDisplacement); + link.next(); + continue; + } + } + + link.resolveAndNext(this); + } + + return err; +} + +// CodeHolder - Relocations +// ======================== + +Error CodeHolder::newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept { + ASMJIT_PROPAGATE(_relocations.willGrow(&_allocator)); + + uint32_t relocId = _relocations.size(); + if (ASMJIT_UNLIKELY(relocId == Globals::kInvalidId)) + return DebugUtils::errored(kErrorTooManyRelocations); + + RelocEntry* re = _allocator.allocZeroedT(); + if (ASMJIT_UNLIKELY(!re)) + return DebugUtils::errored(kErrorOutOfMemory); + + re->_id = relocId; + re->_relocType = relocType; + re->_sourceSectionId = Globals::kInvalidId; + re->_targetSectionId = Globals::kInvalidId; + _relocations.appendUnsafe(re); + + *dst = re; + return kErrorOk; +} + +// CodeHolder - Expression Evaluation +// ================================== + +static Error CodeHolder_evaluateExpression(CodeHolder* self, Expression* exp, uint64_t* out) noexcept { + uint64_t value[2]; + for (size_t i = 0; i < 2; i++) { + uint64_t v; + switch (exp->valueType[i]) { + case ExpressionValueType::kNone: { + v = 0; + break; + } + + case ExpressionValueType::kConstant: { + v = exp->value[i].constant; + break; + } + + case ExpressionValueType::kLabel: { + LabelEntry* le = exp->value[i].label; + if (!le->isBound()) + return DebugUtils::errored(kErrorExpressionLabelNotBound); + v = le->section()->offset() + le->offset(); + break; + } + + case ExpressionValueType::kExpression: { + Expression* nested = exp->value[i].expression; + ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(self, nested, &v)); + break; + } + + default: + return DebugUtils::errored(kErrorInvalidState); + } + + value[i] = v; + } + + uint64_t result; + uint64_t& a = value[0]; + uint64_t& b = value[1]; + + switch (exp->opType) { + case ExpressionOpType::kAdd: + result = a + b; + break; + + case ExpressionOpType::kSub: + result = a - b; + break; + + case ExpressionOpType::kMul: + result = a * b; + break; + + case ExpressionOpType::kSll: + result = (b > 63) ? uint64_t(0) : uint64_t(a << b); + break; + + case ExpressionOpType::kSrl: + result = (b > 63) ? uint64_t(0) : uint64_t(a >> b); + break; + + case ExpressionOpType::kSra: + result = Support::sar(a, Support::min(b, 63)); + break; + + default: + return DebugUtils::errored(kErrorInvalidState); + } + + *out = result; + return kErrorOk; +} + +// CodeHolder - Utilities +// ====================== + +Error CodeHolder::flatten() noexcept { + uint64_t offset = 0; + for (Section* section : _sectionsByOrder) { + uint64_t realSize = section->realSize(); + if (realSize) { + uint64_t alignedOffset = Support::alignUp(offset, section->alignment()); + if (ASMJIT_UNLIKELY(alignedOffset < offset)) + return DebugUtils::errored(kErrorTooLarge); + + Support::FastUInt8 of = 0; + offset = Support::addOverflow(alignedOffset, realSize, &of); + + if (ASMJIT_UNLIKELY(of)) + return DebugUtils::errored(kErrorTooLarge); + } + } + + // Now we know that we can assign offsets of all sections properly. + Section* prev = nullptr; + offset = 0; + for (Section* section : _sectionsByOrder) { + uint64_t realSize = section->realSize(); + if (realSize) + offset = Support::alignUp(offset, section->alignment()); + section->_offset = offset; + + // Make sure the previous section extends a bit to cover the alignment. + if (prev) + prev->_virtualSize = offset - prev->_offset; + + prev = section; + offset += realSize; + } + + return kErrorOk; +} + +size_t CodeHolder::codeSize() const noexcept { + Support::FastUInt8 of = 0; + uint64_t offset = 0; + + for (Section* section : _sectionsByOrder) { + uint64_t realSize = section->realSize(); + + if (realSize) { + uint64_t alignedOffset = Support::alignUp(offset, section->alignment()); + ASMJIT_ASSERT(alignedOffset >= offset); + offset = Support::addOverflow(alignedOffset, realSize, &of); + } + } + + if ((sizeof(uint64_t) > sizeof(size_t) && offset > SIZE_MAX) || of) + return SIZE_MAX; + + return size_t(offset); +} + +Error CodeHolder::relocateToBase(uint64_t baseAddress) noexcept { + // Base address must be provided. + if (ASMJIT_UNLIKELY(baseAddress == Globals::kNoBaseAddress)) + return DebugUtils::errored(kErrorInvalidArgument); + + _baseAddress = baseAddress; + uint32_t addressSize = _environment.registerSize(); + + Section* addressTableSection = _addressTableSection; + uint32_t addressTableEntryCount = 0; + uint8_t* addressTableEntryData = nullptr; + + if (addressTableSection) { + ASMJIT_PROPAGATE( + reserveBuffer(&addressTableSection->_buffer, size_t(addressTableSection->virtualSize()))); + addressTableEntryData = addressTableSection->_buffer.data(); + } + + // Relocate all recorded locations. + for (const RelocEntry* re : _relocations) { + // Possibly deleted or optimized-out entry. + if (re->relocType() == RelocType::kNone) + continue; + + Section* sourceSection = sectionById(re->sourceSectionId()); + Section* targetSection = nullptr; + + if (re->targetSectionId() != Globals::kInvalidId) + targetSection = sectionById(re->targetSectionId()); + + uint64_t value = re->payload(); + uint64_t sectionOffset = sourceSection->offset(); + uint64_t sourceOffset = re->sourceOffset(); + + // Make sure that the `RelocEntry` doesn't go out of bounds. + size_t regionSize = re->format().regionSize(); + if (ASMJIT_UNLIKELY(re->sourceOffset() >= sourceSection->bufferSize() || + sourceSection->bufferSize() - size_t(re->sourceOffset()) < regionSize)) + return DebugUtils::errored(kErrorInvalidRelocEntry); + + uint8_t* buffer = sourceSection->data(); + + switch (re->relocType()) { + case RelocType::kExpression: { + Expression* expression = (Expression*)(uintptr_t(value)); + ASMJIT_PROPAGATE(CodeHolder_evaluateExpression(this, expression, &value)); + break; + } + + case RelocType::kAbsToAbs: { + break; + } + + case RelocType::kRelToAbs: { + // Value is currently a relative offset from the start of its section. + // We have to convert it to an absolute offset (including base address). + if (ASMJIT_UNLIKELY(!targetSection)) + return DebugUtils::errored(kErrorInvalidRelocEntry); + + //value += baseAddress + sectionOffset + sourceOffset + regionSize; + value += baseAddress + targetSection->offset(); + break; + } + + case RelocType::kAbsToRel: { + value -= baseAddress + sectionOffset + sourceOffset + regionSize; + + // Sign extend as we are not interested in the high 32-bit word in a 32-bit address space. + if (addressSize <= 4) + value = uint64_t(int64_t(int32_t(value & 0xFFFFFFFFu))); + else if (!Support::isInt32(int64_t(value))) + return DebugUtils::errored(kErrorRelocOffsetOutOfRange); + + break; + } + + case RelocType::kX64AddressEntry: { + size_t valueOffset = size_t(re->sourceOffset()) + re->format().valueOffset(); + if (re->format().valueSize() != 4 || valueOffset < 2) + return DebugUtils::errored(kErrorInvalidRelocEntry); + + // First try whether a relative 32-bit displacement would work. + value -= baseAddress + sectionOffset + sourceOffset + regionSize; + if (!Support::isInt32(int64_t(value))) { + // Relative 32-bit displacement is not possible, use '.addrtab' section. + AddressTableEntry* atEntry = _addressTableEntries.get(re->payload()); + if (ASMJIT_UNLIKELY(!atEntry)) + return DebugUtils::errored(kErrorInvalidRelocEntry); + + // Cannot be null as we have just matched the `AddressTableEntry`. + ASMJIT_ASSERT(addressTableSection != nullptr); + + if (!atEntry->hasAssignedSlot()) + atEntry->_slot = addressTableEntryCount++; + + size_t atEntryIndex = size_t(atEntry->slot()) * addressSize; + uint64_t addrSrc = sectionOffset + sourceOffset + regionSize; + uint64_t addrDst = addressTableSection->offset() + uint64_t(atEntryIndex); + + value = addrDst - addrSrc; + if (!Support::isInt32(int64_t(value))) + return DebugUtils::errored(kErrorRelocOffsetOutOfRange); + + // Bytes that replace [REX, OPCODE] bytes. + uint32_t byte0 = 0xFF; + uint32_t byte1 = buffer[valueOffset - 1]; + + if (byte1 == 0xE8) { + // Patch CALL/MOD byte to FF /2 (-> 0x15). + byte1 = x86EncodeMod(0, 2, 5); + } + else if (byte1 == 0xE9) { + // Patch JMP/MOD byte to FF /4 (-> 0x25). + byte1 = x86EncodeMod(0, 4, 5); + } + else { + return DebugUtils::errored(kErrorInvalidRelocEntry); + } + + // Patch `jmp/call` instruction. + buffer[valueOffset - 2] = uint8_t(byte0); + buffer[valueOffset - 1] = uint8_t(byte1); + + Support::writeU64uLE(addressTableEntryData + atEntryIndex, re->payload()); + } + break; + } + + default: + return DebugUtils::errored(kErrorInvalidRelocEntry); + } + + if (!CodeWriterUtils::writeOffset(buffer + re->sourceOffset(), int64_t(value), re->format())) { + return DebugUtils::errored(kErrorInvalidRelocEntry); + } + } + + // Fixup the virtual size of the address table if it's the last section. + if (_sectionsByOrder.last() == addressTableSection) { + ASMJIT_ASSERT(addressTableSection != nullptr); + + size_t addressTableSize = addressTableEntryCount * addressSize; + addressTableSection->_buffer._size = addressTableSize; + addressTableSection->_virtualSize = addressTableSize; + } + + return kErrorOk; +} + +Error CodeHolder::copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags) noexcept { + if (ASMJIT_UNLIKELY(!isSectionValid(sectionId))) + return DebugUtils::errored(kErrorInvalidSection); + + Section* section = sectionById(sectionId); + size_t bufferSize = section->bufferSize(); + + if (ASMJIT_UNLIKELY(dstSize < bufferSize)) + return DebugUtils::errored(kErrorInvalidArgument); + + memcpy(dst, section->data(), bufferSize); + + if (bufferSize < dstSize && Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer)) { + size_t paddingSize = dstSize - bufferSize; + memset(static_cast(dst) + bufferSize, 0, paddingSize); + } + + return kErrorOk; +} + +Error CodeHolder::copyFlattenedData(void* dst, size_t dstSize, CopySectionFlags copyFlags) noexcept { + size_t end = 0; + for (Section* section : _sectionsByOrder) { + if (section->offset() > dstSize) + return DebugUtils::errored(kErrorInvalidArgument); + + size_t bufferSize = section->bufferSize(); + size_t offset = size_t(section->offset()); + + if (ASMJIT_UNLIKELY(dstSize - offset < bufferSize)) + return DebugUtils::errored(kErrorInvalidArgument); + + uint8_t* dstTarget = static_cast(dst) + offset; + size_t paddingSize = 0; + memcpy(dstTarget, section->data(), bufferSize); + + if (Support::test(copyFlags, CopySectionFlags::kPadSectionBuffer) && bufferSize < section->virtualSize()) { + paddingSize = Support::min(dstSize - offset, size_t(section->virtualSize())) - bufferSize; + memset(dstTarget + bufferSize, 0, paddingSize); + } + + end = Support::max(end, offset + bufferSize + paddingSize); + } + + if (end < dstSize && Support::test(copyFlags, CopySectionFlags::kPadTargetBuffer)) { + memset(static_cast(dst) + end, 0, dstSize - end); + } + + return kErrorOk; +} + +// CodeHolder - Tests +// ================== + +#if defined(ASMJIT_TEST) +UNIT(code_holder) { + CodeHolder code; + + INFO("Verifying CodeHolder::init()"); + Environment env; + env.init(Arch::kX86); + + code.init(env); + EXPECT(code.arch() == Arch::kX86); + + INFO("Verifying named labels"); + LabelEntry* le; + EXPECT(code.newNamedLabelEntry(&le, "NamedLabel", SIZE_MAX, LabelType::kGlobal) == kErrorOk); + EXPECT(strcmp(le->name(), "NamedLabel") == 0); + EXPECT(code.labelIdByName("NamedLabel") == le->id()); + + INFO("Verifying section ordering"); + Section* section1; + EXPECT(code.newSection(§ion1, "high-priority", SIZE_MAX, SectionFlags::kNone, 1, -1) == kErrorOk); + EXPECT(code.sections()[1] == section1); + EXPECT(code.sectionsByOrder()[0] == section1); + + Section* section0; + EXPECT(code.newSection(§ion0, "higher-priority", SIZE_MAX, SectionFlags::kNone, 1, -2) == kErrorOk); + EXPECT(code.sections()[2] == section0); + EXPECT(code.sectionsByOrder()[0] == section0); + EXPECT(code.sectionsByOrder()[1] == section1); + + Section* section3; + EXPECT(code.newSection(§ion3, "low-priority", SIZE_MAX, SectionFlags::kNone, 1, 2) == kErrorOk); + EXPECT(code.sections()[3] == section3); + EXPECT(code.sectionsByOrder()[3] == section3); +} +#endif + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/codeholder.h b/src/asmjit/core/codeholder.h new file mode 100644 index 0000000..6ed2ddf --- /dev/null +++ b/src/asmjit/core/codeholder.h @@ -0,0 +1,1035 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_CODEHOLDER_H_INCLUDED +#define ASMJIT_CORE_CODEHOLDER_H_INCLUDED + +#include "../core/archtraits.h" +#include "../core/codebuffer.h" +#include "../core/errorhandler.h" +#include "../core/operand.h" +#include "../core/string.h" +#include "../core/support.h" +#include "../core/target.h" +#include "../core/zone.h" +#include "../core/zonehash.h" +#include "../core/zonestring.h" +#include "../core/zonetree.h" +#include "../core/zonevector.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +class BaseEmitter; +class CodeHolder; +class LabelEntry; +class Logger; + +//! Operator type that can be used within an \ref Expression. +enum class ExpressionOpType : uint8_t { + //! Addition. + kAdd = 0, + //! Subtraction. + kSub = 1, + //! Multiplication + kMul = 2, + //! Logical left shift. + kSll = 3, + //! Logical right shift. + kSrl = 4, + //! Arithmetic right shift. + kSra = 5 +}; + +//! Value tyoe that can be used within an \ref Expression. +enum class ExpressionValueType : uint8_t { + //! No value or invalid. + kNone = 0, + //! Value is 64-bit unsigned integer (constant). + kConstant = 1, + //! Value is \ref LabelEntry, which references a \ref Label. + kLabel = 2, + //! Value is \ref Expression + kExpression = 3 +}; + +//! Expression node that can reference constants, labels, and another expressions. +struct Expression { + //! Expression value. + union Value { + //! Constant. + uint64_t constant; + //! Pointer to another expression. + Expression* expression; + //! Pointer to \ref LabelEntry. + LabelEntry* label; + }; + + //! \name Members + //! \{ + + //! Operation type. + ExpressionOpType opType; + //! Value types of \ref value. + ExpressionValueType valueType[2]; + //! Reserved for future use, should be initialized to zero. + uint8_t reserved[5]; + //! Expression left and right values. + Value value[2]; + + //! \} + + //! \name Accessors + //! \{ + + //! Resets the whole expression. + //! + //! Changes both values to \ref ExpressionValueType::kNone. + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! Sets the value type at `index` to \ref ExpressionValueType::kConstant and its content to `constant`. + inline void setValueAsConstant(size_t index, uint64_t constant) noexcept { + valueType[index] = ExpressionValueType::kConstant; + value[index].constant = constant; + } + + //! Sets the value type at `index` to \ref ExpressionValueType::kLabel and its content to `labelEntry`. + inline void setValueAsLabel(size_t index, LabelEntry* labelEntry) noexcept { + valueType[index] = ExpressionValueType::kLabel; + value[index].label = labelEntry; + } + + //! Sets the value type at `index` to \ref ExpressionValueType::kExpression and its content to `expression`. + inline void setValueAsExpression(size_t index, Expression* expression) noexcept { + valueType[index] = ExpressionValueType::kExpression; + value[index].expression = expression; + } + + //! \} +}; + +//! Section flags, used by \ref Section. +enum class SectionFlags : uint32_t { + //! No flags. + kNone = 0, + //! Executable (.text sections). + kExecutable = 0x00000001u, + //! Read-only (.text and .data sections). + kReadOnly = 0x00000002u, + //! Zero initialized by the loader (BSS). + kZeroInitialized = 0x00000004u, + //! Info / comment flag. + kComment = 0x00000008u, + //! Section created implicitly, can be deleted by \ref Target. + kImplicit = 0x80000000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(SectionFlags) + +//! Flags that can be used with \ref CodeHolder::copySectionData() and \ref CodeHolder::copyFlattenedData(). +enum class CopySectionFlags : uint32_t { + //! No flags. + kNone = 0, + + //! If virtual size of a section is greater than the size of its \ref CodeBuffer then all bytes between the buffer + //! size and virtual size will be zeroed. If this option is not set then those bytes would be left as is, which + //! means that if the user didn't initialize them they would have a previous content, which may be unwanted. + kPadSectionBuffer = 0x00000001u, + + //! Clears the target buffer if the flattened data is less than the destination size. This option works + //! only with \ref CodeHolder::copyFlattenedData() as it processes multiple sections. It is ignored by + //! \ref CodeHolder::copySectionData(). + kPadTargetBuffer = 0x00000002u +}; +ASMJIT_DEFINE_ENUM_FLAGS(CopySectionFlags) + +//! Section entry. +class Section { +public: + //! \name Members + //! \{ + + //! Section id. + uint32_t _id; + //! Section flags. + SectionFlags _flags; + //! Section alignment requirements (0 if no requirements). + uint32_t _alignment; + //! Order (lower value means higher priority). + int32_t _order; + //! Offset of this section from base-address. + uint64_t _offset; + //! Virtual size of the section (zero initialized sections). + uint64_t _virtualSize; + //! Section name (max 35 characters, PE allows max 8). + FixedString _name; + //! Code or data buffer. + CodeBuffer _buffer; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the section id. + inline uint32_t id() const noexcept { return _id; } + //! Returns the section name, as a null terminated string. + inline const char* name() const noexcept { return _name.str; } + + //! Returns the section data. + inline uint8_t* data() noexcept { return _buffer.data(); } + //! \overload + inline const uint8_t* data() const noexcept { return _buffer.data(); } + + //! Returns the section flags. + inline SectionFlags flags() const noexcept { return _flags; } + //! Tests whether the section has the given `flag`. + inline bool hasFlag(SectionFlags flag) const noexcept { return Support::test(_flags, flag); } + //! Adds `flags` to the section flags. + inline void addFlags(SectionFlags flags) noexcept { _flags |= flags; } + //! Removes `flags` from the section flags. + inline void clearFlags(SectionFlags flags) noexcept { _flags &= ~flags; } + + //! Returns the minimum section alignment + inline uint32_t alignment() const noexcept { return _alignment; } + //! Sets the minimum section alignment + inline void setAlignment(uint32_t alignment) noexcept { _alignment = alignment; } + + //! Returns the section order, which has a higher priority than section id. + inline int32_t order() const noexcept { return _order; } + + //! Returns the section offset, relative to base. + inline uint64_t offset() const noexcept { return _offset; } + //! Set the section offset. + inline void setOffset(uint64_t offset) noexcept { _offset = offset; } + + //! Returns the virtual size of the section. + //! + //! Virtual size is initially zero and is never changed by AsmJit. It's normal if virtual size is smaller than + //! size returned by `bufferSize()` as the buffer stores real data emitted by assemblers or appended by users. + //! + //! Use `realSize()` to get the real and final size of this section. + inline uint64_t virtualSize() const noexcept { return _virtualSize; } + //! Sets the virtual size of the section. + inline void setVirtualSize(uint64_t virtualSize) noexcept { _virtualSize = virtualSize; } + + //! Returns the buffer size of the section. + inline size_t bufferSize() const noexcept { return _buffer.size(); } + //! Returns the real size of the section calculated from virtual and buffer sizes. + inline uint64_t realSize() const noexcept { return Support::max(virtualSize(), bufferSize()); } + + //! Returns the `CodeBuffer` used by this section. + inline CodeBuffer& buffer() noexcept { return _buffer; } + //! Returns the `CodeBuffer` used by this section (const). + inline const CodeBuffer& buffer() const noexcept { return _buffer; } + + //! \} +}; + +//! Entry in an address table. +class AddressTableEntry : public ZoneTreeNodeT { +public: + ASMJIT_NONCOPYABLE(AddressTableEntry) + + //! \name Members + //! \{ + + //! Address. + uint64_t _address; + //! Slot. + uint32_t _slot; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline explicit AddressTableEntry(uint64_t address) noexcept + : _address(address), + _slot(0xFFFFFFFFu) {} + + //! \} + + //! \name Accessors + //! \{ + + inline uint64_t address() const noexcept { return _address; } + inline uint32_t slot() const noexcept { return _slot; } + + inline bool hasAssignedSlot() const noexcept { return _slot != 0xFFFFFFFFu; } + + inline bool operator<(const AddressTableEntry& other) const noexcept { return _address < other._address; } + inline bool operator>(const AddressTableEntry& other) const noexcept { return _address > other._address; } + + inline bool operator<(uint64_t queryAddress) const noexcept { return _address < queryAddress; } + inline bool operator>(uint64_t queryAddress) const noexcept { return _address > queryAddress; } + + //! \} +}; + +//! Offset format type, used by \ref OffsetFormat. +enum class OffsetType : uint8_t { + //! A value having `_immBitCount` bits and shifted by `_immBitShift`. + //! + //! This offset type is sufficient for many targets that store offset as a continuous set bits within an + //! instruction word / sequence of bytes. + kSignedOffset, + + //! An unsigned value having `_immBitCount` bits and shifted by `_immBitShift`. + kUnsignedOffset, + + // AArch64 Specific Offset Formats + // ------------------------------- + + //! AARCH64 ADR format of `[.|immlo:2|.....|immhi:19|.....]`. + kAArch64_ADR, + + //! AARCH64 ADRP format of `[.|immlo:2|.....|immhi:19|.....]` (4kB pages). + kAArch64_ADRP, + + //! Maximum value of `OffsetFormatType`. + kMaxValue = kAArch64_ADRP +}; + +//! Provides information about formatting offsets, absolute addresses, or their parts. Offset format is used by both +//! \ref RelocEntry and \ref LabelLink. The illustration below describes the relation of region size and offset size. +//! Region size is the size of the whole unit whereas offset size is the size of the unit that will be patched. +//! +//! ``` +//! +-> Code buffer | The subject of the relocation (region) | +//! | | (Word-Offset) (Word-Size) | +//! |xxxxxxxxxxxxxxx|................|*PATCHED*|................|xxxxxxxxxxxx-> +//! | | +//! [Word Offset points here]----+ +--- [WordOffset + WordSize] +//! ``` +//! +//! Once the offset word has been located it can be patched like this: +//! +//! ``` +//! |ImmDiscardLSB (discard LSB bits). +//! |.. +//! [0000000000000iiiiiiiiiiiiiiiiiDD] - Offset value (32-bit) +//! [000000000000000iiiiiiiiiiiiiiiii] - Offset value after discard LSB. +//! [00000000000iiiiiiiiiiiiiiiii0000] - Offset value shifted by ImmBitShift. +//! [xxxxxxxxxxxiiiiiiiiiiiiiiiiixxxx] - Patched word (32-bit) +//! |...............| +//! (ImmBitCount) +- ImmBitShift +//! ``` +struct OffsetFormat { + //! \name Members + //! \{ + + //! Type of the offset. + OffsetType _type; + //! Encoding flags. + uint8_t _flags; + //! Size of the region (in bytes) containing the offset value, if the offset value is part of an instruction, + //! otherwise it would be the same as `_valueSize`. + uint8_t _regionSize; + //! Size of the offset value, in bytes (1, 2, 4, or 8). + uint8_t _valueSize; + //! Offset of the offset value, in bytes, relative to the start of the region or data. Value offset would be + //! zero if both region size and value size are equal. + uint8_t _valueOffset; + //! Size of the offset immediate value in bits. + uint8_t _immBitCount; + //! Shift of the offset immediate value in bits in the target word. + uint8_t _immBitShift; + //! Number of least significant bits to discard before writing the immediate to the destination. All discarded + //! bits must be zero otherwise the value is invalid. + uint8_t _immDiscardLsb; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the type of the offset. + inline OffsetType type() const noexcept { return _type; } + + //! Returns flags. + inline uint32_t flags() const noexcept { return _flags; } + + //! Returns the size of the region/instruction where the offset is encoded. + inline uint32_t regionSize() const noexcept { return _regionSize; } + + //! Returns the offset of the word relative to the start of the region where the offset is. + inline uint32_t valueOffset() const noexcept { return _valueOffset; } + + //! Returns the size of the data-type (word) that contains the offset, in bytes. + inline uint32_t valueSize() const noexcept { return _valueSize; } + //! Returns the count of bits of the offset value in the data it's stored in. + inline uint32_t immBitCount() const noexcept { return _immBitCount; } + //! Returns the bit-shift of the offset value in the data it's stored in. + inline uint32_t immBitShift() const noexcept { return _immBitShift; } + //! Returns the number of least significant bits of the offset value, that must be zero and that are not part of + //! the encoded data. + inline uint32_t immDiscardLsb() const noexcept { return _immDiscardLsb; } + + //! Resets this offset format to a simple data value of `dataSize` bytes. + //! + //! The region will be the same size as data and immediate bits would correspond to `dataSize * 8`. There will be + //! no immediate bit shift or discarded bits. + inline void resetToSimpleValue(OffsetType type, size_t valueSize) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(valueSize * 8u); + _immBitShift = uint8_t(0); + _immDiscardLsb = uint8_t(0); + } + + inline void resetToImmValue(OffsetType type, size_t valueSize, uint32_t immBitShift, uint32_t immBitCount, uint32_t immDiscardLsb) noexcept { + ASMJIT_ASSERT(valueSize <= 8u); + ASMJIT_ASSERT(immBitShift < valueSize * 8u); + ASMJIT_ASSERT(immBitCount <= 64u); + ASMJIT_ASSERT(immDiscardLsb <= 64u); + + _type = type; + _flags = uint8_t(0); + _regionSize = uint8_t(valueSize); + _valueSize = uint8_t(valueSize); + _valueOffset = uint8_t(0); + _immBitCount = uint8_t(immBitCount); + _immBitShift = uint8_t(immBitShift); + _immDiscardLsb = uint8_t(immDiscardLsb); + } + + inline void setRegion(size_t regionSize, size_t valueOffset) noexcept { + _regionSize = uint8_t(regionSize); + _valueOffset = uint8_t(valueOffset); + } + + inline void setLeadingAndTrailingSize(size_t leadingSize, size_t trailingSize) noexcept { + _regionSize = uint8_t(leadingSize + trailingSize + _valueSize); + _valueOffset = uint8_t(leadingSize); + } + + //! \} +}; + +//! Relocation type. +enum class RelocType : uint32_t { + //! None/deleted (no relocation). + kNone = 0, + //! Expression evaluation, `_payload` is pointer to `Expression`. + kExpression = 1, + //! Relocate absolute to absolute. + kAbsToAbs = 2, + //! Relocate relative to absolute. + kRelToAbs = 3, + //! Relocate absolute to relative. + kAbsToRel = 4, + //! Relocate absolute to relative or use trampoline. + kX64AddressEntry = 5 +}; + +//! Relocation entry. +struct RelocEntry { + //! \name Members + //! \{ + + //! Relocation id. + uint32_t _id; + //! Type of the relocation. + RelocType _relocType; + //! Format of the relocated value. + OffsetFormat _format; + //! Source section id. + uint32_t _sourceSectionId; + //! Target section id. + uint32_t _targetSectionId; + //! Source offset (relative to start of the section). + uint64_t _sourceOffset; + //! Payload (target offset, target address, expression, etc). + uint64_t _payload; + + //! \} + + //! \name Accessors + //! \{ + + inline uint32_t id() const noexcept { return _id; } + + inline RelocType relocType() const noexcept { return _relocType; } + inline const OffsetFormat& format() const noexcept { return _format; } + + inline uint32_t sourceSectionId() const noexcept { return _sourceSectionId; } + inline uint32_t targetSectionId() const noexcept { return _targetSectionId; } + + inline uint64_t sourceOffset() const noexcept { return _sourceOffset; } + inline uint64_t payload() const noexcept { return _payload; } + + Expression* payloadAsExpression() const noexcept { + return reinterpret_cast(uintptr_t(_payload)); + } + + //! \} +}; + +//! Type of the \ref Label. +enum class LabelType : uint8_t { + //! Anonymous label that can optionally have a name, which is only used for debugging purposes. + kAnonymous = 0, + //! Local label (always has parentId). + kLocal = 1, + //! Global label (never has parentId). + kGlobal = 2, + //! External label (references an external symbol). + kExternal = 3, + + //! Maximum value of `LabelType`. + kMaxValue = kExternal +}; + +//! Data structure used to link either unbound labels or cross-section links. +struct LabelLink { + //! Next link (single-linked list). + LabelLink* next; + //! Section id where the label is bound. + uint32_t sectionId; + //! Relocation id or Globals::kInvalidId. + uint32_t relocId; + //! Label offset relative to the start of the section. + size_t offset; + //! Inlined rel8/rel32. + intptr_t rel; + //! Offset format information. + OffsetFormat format; +}; + +//! Label entry. +//! +//! Contains the following properties: +//! - Label id - This is the only thing that is set to the `Label` operand. +//! - Label name - Optional, used mostly to create executables and libraries. +//! - Label type - Type of the label, default `LabelType::kAnonymous`. +//! - Label parent id - Derived from many assemblers that allow to define a local label that falls under a global +//! label. This allows to define many labels of the same name that have different parent (global) label. +//! - Offset - offset of the label bound by `Assembler`. +//! - Links - single-linked list that contains locations of code that has to be patched when the label gets bound. +//! Every use of unbound label adds one link to `_links` list. +//! - HVal - Hash value of label's name and optionally parentId. +//! - HashNext - Hash-table implementation detail. +class LabelEntry : public ZoneHashNode { +public: + //! \name Constants + //! \{ + + enum : uint32_t { + //! SSO size of \ref _name. + //! + //! \cond INTERNAL + //! Let's round the size of `LabelEntry` to 64 bytes (as `ZoneAllocator` has granularity of 32 bytes anyway). This + //! gives `_name` the remaining space, which is should be 16 bytes on 64-bit and 28 bytes on 32-bit architectures. + //! \endcond + kStaticNameSize = 64 - (sizeof(ZoneHashNode) + 8 + sizeof(Section*) + sizeof(size_t) + sizeof(LabelLink*)) + }; + + //! \} + + //! \name Members + //! \{ + + //! Type of the label. + LabelType _type; + //! Must be zero. + uint8_t _reserved[3]; + //! Label parent id or zero. + uint32_t _parentId; + //! Label offset relative to the start of the `_section`. + uint64_t _offset; + //! Section where the label was bound. + Section* _section; + //! Label links. + LabelLink* _links; + //! Label name. + ZoneString _name; + + //! \} + + //! \name Accessors + //! \{ + + // NOTE: Label id is stored in `_customData`, which is provided by ZoneHashNode to fill a padding that a C++ + // compiler targeting 64-bit CPU will add to align the structure to 64-bits. + + //! Returns label id. + inline uint32_t id() const noexcept { return _customData; } + //! Sets label id (internal, used only by `CodeHolder`). + inline void _setId(uint32_t id) noexcept { _customData = id; } + + //! Returns label type. + inline LabelType type() const noexcept { return _type; } + + //! Tests whether the label has a parent label. + inline bool hasParent() const noexcept { return _parentId != Globals::kInvalidId; } + //! Returns label's parent id. + inline uint32_t parentId() const noexcept { return _parentId; } + + //! Returns the section where the label was bound. + //! + //! If the label was not yet bound the return value is `nullptr`. + inline Section* section() const noexcept { return _section; } + + //! Tests whether the label has name. + inline bool hasName() const noexcept { return !_name.empty(); } + + //! Returns the label's name. + //! + //! \note Local labels will return their local name without their parent part, for example ".L1". + inline const char* name() const noexcept { return _name.data(); } + + //! Returns size of label's name. + //! + //! \note Label name is always null terminated, so you can use `strlen()` to get it, however, it's also cached in + //! `LabelEntry` itself, so if you want to know the size the fastest way is to call `LabelEntry::nameSize()`. + inline uint32_t nameSize() const noexcept { return _name.size(); } + + //! Returns links associated with this label. + inline LabelLink* links() const noexcept { return _links; } + + //! Tests whether the label is bound. + inline bool isBound() const noexcept { return _section != nullptr; } + //! Tests whether the label is bound to a the given `sectionId`. + inline bool isBoundTo(Section* section) const noexcept { return _section == section; } + + //! Returns the label offset (only useful if the label is bound). + inline uint64_t offset() const noexcept { return _offset; } + + //! Returns the hash-value of label's name and its parent label (if any). + //! + //! Label hash is calculated as `HASH(Name) ^ ParentId`. The hash function is implemented in `Support::hashString()` + //! and `Support::hashRound()`. + inline uint32_t hashCode() const noexcept { return _hashCode; } + + //! \} +}; + +//! Holds assembled code and data (including sections, labels, and relocation information). +//! +//! CodeHolder connects emitters with their targets. It provides them interface that can be used to query information +//! about the target environment (architecture, etc...) and API to create labels, sections, relocations, and to write +//! data to a \ref CodeBuffer, which is always part of \ref Section. More than one emitter can be attached to a single +//! CodeHolder instance at a time, which is used in practice +//! +//! CodeHolder provides interface for all emitter types. Assemblers use CodeHolder to write into \ref CodeBuffer, and +//! higher level emitters like Builder and Compiler use CodeHolder to manage labels and sections so higher level code +//! can be serialized to Assembler by \ref BaseEmitter::finalize() and \ref BaseBuilder::serializeTo(). +//! +//! In order to use CodeHolder, it must be first initialized by \ref init(). After the CodeHolder has been successfully +//! initialized it can be used to hold assembled code, sections, labels, relocations, and to attach / detach code +//! emitters. After the end of code generation it can be used to query physical locations of labels and to relocate +//! the assembled code into the right address. +//! +//! \note \ref CodeHolder has an ability to attach an \ref ErrorHandler, however, the error handler is not triggered +//! by \ref CodeHolder itself, it's instead propagated to all emitters that attach to it. +class CodeHolder { +public: + ASMJIT_NONCOPYABLE(CodeHolder) + + //! \name Members + //! \{ + + //! Environment information. + Environment _environment; + //! Base address or \ref Globals::kNoBaseAddress. + uint64_t _baseAddress; + + //! Attached `Logger`, used by all consumers. + Logger* _logger; + //! Attached `ErrorHandler`. + ErrorHandler* _errorHandler; + + //! Code zone (used to allocate core structures). + Zone _zone; + //! Zone allocator, used to manage internal containers. + ZoneAllocator _allocator; + + //! Attached emitters. + ZoneVector _emitters; + //! Section entries. + ZoneVector _sections; + //! Section entries sorted by section order and then section id. + ZoneVector _sectionsByOrder; + //! Label entries. + ZoneVector _labelEntries; + //! Relocation entries. + ZoneVector _relocations; + //! Label name -> LabelEntry (only named labels). + ZoneHash _namedLabels; + + //! Count of label links, which are not resolved. + size_t _unresolvedLinkCount; + //! Pointer to an address table section (or null if this section doesn't exist). + Section* _addressTableSection; + //! Address table entries. + ZoneTree _addressTableEntries; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates an uninitialized CodeHolder (you must init() it before it can be used). + //! + //! An optional `temporary` argument can be used to initialize the first block of \ref Zone that the CodeHolder + //! uses into a temporary memory provided by the user. + ASMJIT_API explicit CodeHolder(const Support::Temporary* temporary = nullptr) noexcept; + + //! \overload + inline explicit CodeHolder(const Support::Temporary& temporary) noexcept + : CodeHolder(&temporary) {} + + //! Destroys the CodeHolder and frees all resources it has allocated. + ASMJIT_API ~CodeHolder() noexcept; + + //! Tests whether the `CodeHolder` has been initialized. + //! + //! Emitters can be only attached to initialized `CodeHolder` instances. + inline bool isInitialized() const noexcept { return _environment.isInitialized(); } + + //! Initializes CodeHolder to hold code described by the given `environment` and `baseAddress`. + ASMJIT_API Error init(const Environment& environment, uint64_t baseAddress = Globals::kNoBaseAddress) noexcept; + //! Detaches all code-generators attached and resets the `CodeHolder`. + ASMJIT_API void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept; + + //! \} + + //! \name Attach & Detach + //! \{ + + //! Attaches an emitter to this `CodeHolder`. + ASMJIT_API Error attach(BaseEmitter* emitter) noexcept; + //! Detaches an emitter from this `CodeHolder`. + ASMJIT_API Error detach(BaseEmitter* emitter) noexcept; + + //! \} + + //! \name Allocators + //! \{ + + //! Returns the allocator that the `CodeHolder` uses. + //! + //! \note This should be only used for AsmJit's purposes. Code holder uses arena allocator to allocate everything, + //! so anything allocated through this allocator will be invalidated by \ref CodeHolder::reset() or by CodeHolder's + //! destructor. + inline ZoneAllocator* allocator() const noexcept { return const_cast(&_allocator); } + + //! \} + + //! \name Code & Architecture + //! \{ + + //! Returns the target environment information. + inline const Environment& environment() const noexcept { return _environment; } + + //! Returns the target architecture. + inline Arch arch() const noexcept { return environment().arch(); } + //! Returns the target sub-architecture. + inline SubArch subArch() const noexcept { return environment().subArch(); } + + //! Tests whether a static base-address is set. + inline bool hasBaseAddress() const noexcept { return _baseAddress != Globals::kNoBaseAddress; } + //! Returns a static base-address or \ref Globals::kNoBaseAddress, if not set. + inline uint64_t baseAddress() const noexcept { return _baseAddress; } + + //! \} + + //! \name Emitters + //! \{ + + //! Returns a vector of attached emitters. + inline const ZoneVector& emitters() const noexcept { return _emitters; } + + //! \} + + //! \name Logging + //! \{ + + //! Returns the attached logger. + inline Logger* logger() const noexcept { return _logger; } + //! Attaches a `logger` to CodeHolder and propagates it to all attached emitters. + ASMJIT_API void setLogger(Logger* logger) noexcept; + //! Resets the logger to none. + inline void resetLogger() noexcept { setLogger(nullptr); } + + //! \name Error Handling + //! \{ + + //! Tests whether the CodeHolder has an attached error handler, see \ref ErrorHandler. + inline bool hasErrorHandler() const noexcept { return _errorHandler != nullptr; } + //! Returns the attached error handler. + inline ErrorHandler* errorHandler() const noexcept { return _errorHandler; } + //! Attach an error handler to this `CodeHolder`. + ASMJIT_API void setErrorHandler(ErrorHandler* errorHandler) noexcept; + //! Resets the error handler to none. + inline void resetErrorHandler() noexcept { setErrorHandler(nullptr); } + + //! \} + + //! \name Code Buffer + //! \{ + + //! Makes sure that at least `n` bytes can be added to CodeHolder's buffer `cb`. + //! + //! \note The buffer `cb` must be managed by `CodeHolder` - otherwise the behavior of the function is undefined. + ASMJIT_API Error growBuffer(CodeBuffer* cb, size_t n) noexcept; + + //! Reserves the size of `cb` to at least `n` bytes. + //! + //! \note The buffer `cb` must be managed by `CodeHolder` - otherwise the behavior of the function is undefined. + ASMJIT_API Error reserveBuffer(CodeBuffer* cb, size_t n) noexcept; + + //! \} + + //! \name Sections + //! \{ + + //! Returns an array of `Section*` records. + inline const ZoneVector& sections() const noexcept { return _sections; } + //! Returns an array of `Section*` records sorted according to section order first, then section id. + inline const ZoneVector& sectionsByOrder() const noexcept { return _sectionsByOrder; } + //! Returns the number of sections. + inline uint32_t sectionCount() const noexcept { return _sections.size(); } + + //! Tests whether the given `sectionId` is valid. + inline bool isSectionValid(uint32_t sectionId) const noexcept { return sectionId < _sections.size(); } + + //! Creates a new section and return its pointer in `sectionOut`. + //! + //! Returns `Error`, does not report a possible error to `ErrorHandler`. + ASMJIT_API Error newSection(Section** sectionOut, const char* name, size_t nameSize = SIZE_MAX, SectionFlags flags = SectionFlags::kNone, uint32_t alignment = 1, int32_t order = 0) noexcept; + + //! Returns a section entry of the given index. + inline Section* sectionById(uint32_t sectionId) const noexcept { return _sections[sectionId]; } + + //! Returns section-id that matches the given `name`. + //! + //! If there is no such section `Section::kInvalidId` is returned. + ASMJIT_API Section* sectionByName(const char* name, size_t nameSize = SIZE_MAX) const noexcept; + + //! Returns '.text' section (section that commonly represents code). + //! + //! \note Text section is always the first section in \ref CodeHolder::sections() array. + inline Section* textSection() const noexcept { return _sections[0]; } + + //! Tests whether '.addrtab' section exists. + inline bool hasAddressTable() const noexcept { return _addressTableSection != nullptr; } + + //! Returns '.addrtab' section. + //! + //! This section is used exclusively by AsmJit to store absolute 64-bit + //! addresses that cannot be encoded in instructions like 'jmp' or 'call'. + //! + //! \note This section is created on demand, the returned pointer can be null. + inline Section* addressTableSection() const noexcept { return _addressTableSection; } + + //! Ensures that '.addrtab' section exists (creates it if it doesn't) and + //! returns it. Can return `nullptr` on out of memory condition. + ASMJIT_API Section* ensureAddressTableSection() noexcept; + + //! Used to add an address to an address table. + //! + //! This implicitly calls `ensureAddressTableSection()` and then creates `AddressTableEntry` that is inserted + //! to `_addressTableEntries`. If the address already exists this operation does nothing as the same addresses + //! use the same slot. + //! + //! This function should be considered internal as it's used by assemblers to insert an absolute address into the + //! address table. Inserting address into address table without creating a particula relocation entry makes no sense. + ASMJIT_API Error addAddressToAddressTable(uint64_t address) noexcept; + + //! \} + + //! \name Labels & Symbols + //! \{ + + //! Returns array of `LabelEntry*` records. + inline const ZoneVector& labelEntries() const noexcept { return _labelEntries; } + + //! Returns number of labels created. + inline uint32_t labelCount() const noexcept { return _labelEntries.size(); } + + //! Tests whether the label having `id` is valid (i.e. created by `newLabelEntry()`). + inline bool isLabelValid(uint32_t labelId) const noexcept { + return labelId < _labelEntries.size(); + } + + //! Tests whether the `label` is valid (i.e. created by `newLabelEntry()`). + inline bool isLabelValid(const Label& label) const noexcept { + return label.id() < _labelEntries.size(); + } + + //! \overload + inline bool isLabelBound(uint32_t labelId) const noexcept { + return isLabelValid(labelId) && _labelEntries[labelId]->isBound(); + } + + //! Tests whether the `label` is already bound. + //! + //! Returns `false` if the `label` is not valid. + inline bool isLabelBound(const Label& label) const noexcept { + return isLabelBound(label.id()); + } + + //! Returns LabelEntry of the given label `id`. + inline LabelEntry* labelEntry(uint32_t labelId) const noexcept { + return isLabelValid(labelId) ? _labelEntries[labelId] : static_cast(nullptr); + } + + //! Returns LabelEntry of the given `label`. + inline LabelEntry* labelEntry(const Label& label) const noexcept { + return labelEntry(label.id()); + } + + //! Returns offset of a `Label` by its `labelId`. + //! + //! The offset returned is relative to the start of the section. Zero offset is returned for unbound labels, + //! which is their initial offset value. + inline uint64_t labelOffset(uint32_t labelId) const noexcept { + ASMJIT_ASSERT(isLabelValid(labelId)); + return _labelEntries[labelId]->offset(); + } + + //! \overload + inline uint64_t labelOffset(const Label& label) const noexcept { + return labelOffset(label.id()); + } + + //! Returns offset of a label by it's `labelId` relative to the base offset. + //! + //! \remarks The offset of the section where the label is bound must be valid in order to use this function, + //! otherwise the value returned will not be reliable. + inline uint64_t labelOffsetFromBase(uint32_t labelId) const noexcept { + ASMJIT_ASSERT(isLabelValid(labelId)); + const LabelEntry* le = _labelEntries[labelId]; + return (le->isBound() ? le->section()->offset() : uint64_t(0)) + le->offset(); + } + + //! \overload + inline uint64_t labelOffsetFromBase(const Label& label) const noexcept { + return labelOffsetFromBase(label.id()); + } + + //! Creates a new anonymous label and return its id in `idOut`. + //! + //! Returns `Error`, does not report error to `ErrorHandler`. + ASMJIT_API Error newLabelEntry(LabelEntry** entryOut) noexcept; + + //! Creates a new named \ref LabelEntry of the given label `type`. + //! + //! \param entryOut Where to store the created \ref LabelEntry. + //! \param name The name of the label. + //! \param nameSize The length of `name` argument, or `SIZE_MAX` if `name` is a null terminated string, which + //! means that the `CodeHolder` will use `strlen()` to determine the length. + //! \param type The type of the label to create, see \ref LabelType. + //! \param parentId Parent id of a local label, otherwise it must be \ref Globals::kInvalidId. + //! \retval Always returns \ref Error, does not report a possible error to the attached \ref ErrorHandler. + //! + //! AsmJit has a support for local labels (\ref LabelType::kLocal) which require a parent label id (parentId). + //! The names of local labels can conflict with names of other local labels that have a different parent. In + //! addition, AsmJit supports named anonymous labels, which are useful only for debugging purposes as the + //! anonymous name will have a name, which will be formatted, but the label itself cannot be queried by such + //! name. + ASMJIT_API Error newNamedLabelEntry(LabelEntry** entryOut, const char* name, size_t nameSize, LabelType type, uint32_t parentId = Globals::kInvalidId) noexcept; + + //! Returns a label by name. + //! + //! If the named label doesn't a default constructed \ref Label is returned, + //! which has its id set to \ref Globals::kInvalidId. + inline Label labelByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept { + return Label(labelIdByName(name, nameSize, parentId)); + } + + //! Returns a label id by name. + //! + //! If the named label doesn't exist \ref Globals::kInvalidId is returned. + ASMJIT_API uint32_t labelIdByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept; + + //! Tests whether there are any unresolved label links. + inline bool hasUnresolvedLinks() const noexcept { return _unresolvedLinkCount != 0; } + //! Returns the number of label links, which are unresolved. + inline size_t unresolvedLinkCount() const noexcept { return _unresolvedLinkCount; } + + //! Creates a new label-link used to store information about yet unbound labels. + //! + //! Returns `null` if the allocation failed. + ASMJIT_API LabelLink* newLabelLink(LabelEntry* le, uint32_t sectionId, size_t offset, intptr_t rel, const OffsetFormat& format) noexcept; + + //! Resolves cross-section links (`LabelLink`) associated with each label that was used as a destination in code + //! of a different section. It's only useful to people that use multiple sections as it will do nothing if the code + //! only contains a single section in which cross-section links are not possible. + ASMJIT_API Error resolveUnresolvedLinks() noexcept; + + //! Binds a label to a given `sectionId` and `offset` (relative to start of the section). + //! + //! This function is generally used by `BaseAssembler::bind()` to do the heavy lifting. + ASMJIT_API Error bindLabel(const Label& label, uint32_t sectionId, uint64_t offset) noexcept; + + //! \} + + //! \name Relocations + //! \{ + + //! Tests whether the code contains relocation entries. + inline bool hasRelocEntries() const noexcept { return !_relocations.empty(); } + //! Returns array of `RelocEntry*` records. + inline const ZoneVector& relocEntries() const noexcept { return _relocations; } + + //! Returns a RelocEntry of the given `id`. + inline RelocEntry* relocEntry(uint32_t id) const noexcept { return _relocations[id]; } + + //! Creates a new relocation entry of type `relocType`. + //! + //! Additional fields can be set after the relocation entry was created. + ASMJIT_API Error newRelocEntry(RelocEntry** dst, RelocType relocType) noexcept; + + //! \} + + //! \name Utilities + //! \{ + + //! Flattens all sections by recalculating their offsets, starting at 0. + //! + //! \note This should never be called more than once. + ASMJIT_API Error flatten() noexcept; + + //! Returns computed the size of code & data of all sections. + //! + //! \note All sections will be iterated over and the code size returned would represent the minimum code size of + //! all combined sections after applying minimum alignment. Code size may decrease after calling `flatten()` and + //! `relocateToBase()`. + ASMJIT_API size_t codeSize() const noexcept; + + //! Relocates the code to the given `baseAddress`. + //! + //! \param baseAddress Absolute base address where the code will be relocated to. Please note that nothing is + //! copied to such base address, it's just an absolute value used by the relocator to resolve all stored relocations. + //! + //! \note This should never be called more than once. + ASMJIT_API Error relocateToBase(uint64_t baseAddress) noexcept; + + //! Copies a single section into `dst`. + ASMJIT_API Error copySectionData(void* dst, size_t dstSize, uint32_t sectionId, CopySectionFlags copyFlags = CopySectionFlags::kNone) noexcept; + + //! Copies all sections into `dst`. + //! + //! This should only be used if the data was flattened and there are no gaps between the sections. The `dstSize` + //! is always checked and the copy will never write anything outside the provided buffer. + ASMJIT_API Error copyFlattenedData(void* dst, size_t dstSize, CopySectionFlags copyFlags = CopySectionFlags::kNone) noexcept; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_CODEHOLDER_H_INCLUDED diff --git a/src/asmjit/core/codewriter.cpp b/src/asmjit/core/codewriter.cpp new file mode 100644 index 0000000..1babc5f --- /dev/null +++ b/src/asmjit/core/codewriter.cpp @@ -0,0 +1,175 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/codeholder.h" +#include "../core/codewriter_p.h" + +ASMJIT_BEGIN_NAMESPACE + +bool CodeWriterUtils::encodeOffset32(uint32_t* dst, int64_t offset64, const OffsetFormat& format) noexcept { + uint32_t bitCount = format.immBitCount(); + uint32_t bitShift = format.immBitShift(); + uint32_t discardLsb = format.immDiscardLsb(); + + // Invalid offset (should not happen). + if (!bitCount || bitCount > format.valueSize() * 8u) + return false; + + uint32_t value; + + // First handle all unsigned offset types. + if (format.type() == OffsetType::kUnsignedOffset) { + if (discardLsb) { + ASMJIT_ASSERT(discardLsb <= 32); + if ((offset64 & Support::lsbMask(discardLsb)) != 0) + return false; + offset64 = int64_t(uint64_t(offset64) >> discardLsb); + } + + value = uint32_t(offset64 & Support::lsbMask(bitCount)); + if (value != offset64) + return false; + } + else { + // The rest of OffsetType options are all signed. + if (discardLsb) { + ASMJIT_ASSERT(discardLsb <= 32); + if ((offset64 & Support::lsbMask(discardLsb)) != 0) + return false; + offset64 >>= discardLsb; + } + + if (!Support::isInt32(offset64)) + return false; + + value = uint32_t(int32_t(offset64)); + if (!Support::isEncodableOffset32(int32_t(value), bitCount)) + return false; + } + + switch (format.type()) { + case OffsetType::kSignedOffset: + case OffsetType::kUnsignedOffset: { + *dst = (value & Support::lsbMask(bitCount)) << bitShift; + return true; + } + + case OffsetType::kAArch64_ADR: + case OffsetType::kAArch64_ADRP: { + // Sanity checks. + if (format.valueSize() != 4 || bitCount != 21 || bitShift != 5) + return false; + + uint32_t immLo = value & 0x3u; + uint32_t immHi = (value >> 2) & Support::lsbMask(19); + + *dst = (immLo << 29) | (immHi << 5); + return true; + } + + default: + return false; + } +} + +bool CodeWriterUtils::encodeOffset64(uint64_t* dst, int64_t offset64, const OffsetFormat& format) noexcept { + uint32_t bitCount = format.immBitCount(); + uint32_t discardLsb = format.immDiscardLsb(); + + if (!bitCount || bitCount > format.valueSize() * 8u) + return false; + + uint64_t value; + + // First handle all unsigned offset types. + if (format.type() == OffsetType::kUnsignedOffset) { + if (discardLsb) { + ASMJIT_ASSERT(discardLsb <= 32); + if ((offset64 & Support::lsbMask(discardLsb)) != 0) + return false; + offset64 = int64_t(uint64_t(offset64) >> discardLsb); + } + + value = uint64_t(offset64) & Support::lsbMask(bitCount); + if (value != uint64_t(offset64)) + return false; + } + else { + // The rest of OffsetType options are all signed. + if (discardLsb) { + ASMJIT_ASSERT(discardLsb <= 32); + if ((offset64 & Support::lsbMask(discardLsb)) != 0) + return false; + offset64 >>= discardLsb; + } + + if (!Support::isEncodableOffset64(offset64, bitCount)) + return false; + + value = uint64_t(offset64); + } + + switch (format.type()) { + case OffsetType::kSignedOffset: + case OffsetType::kUnsignedOffset: { + *dst = (value & Support::lsbMask(bitCount)) << format.immBitShift(); + return true; + } + + default: + return false; + } +} + +bool CodeWriterUtils::writeOffset(void* dst, int64_t offset64, const OffsetFormat& format) noexcept { + // Offset the destination by ValueOffset so the `dst` points to the + // patched word instead of the beginning of the patched region. + dst = static_cast(dst) + format.valueOffset(); + + switch (format.valueSize()) { + case 1: { + uint32_t mask; + if (!encodeOffset32(&mask, offset64, format)) + return false; + + Support::writeU8(dst, uint8_t(Support::readU8(dst) | mask)); + return true; + } + + case 2: { + uint32_t mask; + if (!encodeOffset32(&mask, offset64, format)) + return false; + + Support::writeU16uLE(dst, uint16_t(Support::readU16uLE(dst) | mask)); + return true; + } + + case 4: { + uint32_t mask; + if (!encodeOffset32(&mask, offset64, format)) { + return false; + } + + Support::writeU32uLE(dst, Support::readU32uLE(dst) | mask); + return true; + } + + case 8: { + uint64_t mask; + if (!encodeOffset64(&mask, offset64, format)) + return false; + + Support::writeU64uLE(dst, Support::readU64uLE(dst) | mask); + return true; + } + + default: + return false; + } +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/codewriter_p.h b/src/asmjit/core/codewriter_p.h new file mode 100644 index 0000000..c799241 --- /dev/null +++ b/src/asmjit/core/codewriter_p.h @@ -0,0 +1,179 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_CODEBUFFERWRITER_P_H_INCLUDED +#define ASMJIT_CORE_CODEBUFFERWRITER_P_H_INCLUDED + +#include "../core/assembler.h" +#include "../core/codebuffer.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_assembler +//! \{ + +struct OffsetFormat; + +//! Helper that is used to write into a \ref CodeBuffer held by \ref BaseAssembler. +class CodeWriter { +public: + uint8_t* _cursor; + + ASMJIT_FORCE_INLINE explicit CodeWriter(BaseAssembler* a) noexcept + : _cursor(a->_bufferPtr) {} + + ASMJIT_FORCE_INLINE Error ensureSpace(BaseAssembler* a, size_t n) noexcept { + size_t remainingSpace = (size_t)(a->_bufferEnd - _cursor); + if (ASMJIT_UNLIKELY(remainingSpace < n)) { + CodeBuffer& buffer = a->_section->_buffer; + Error err = a->_code->growBuffer(&buffer, n); + if (ASMJIT_UNLIKELY(err)) + return a->reportError(err); + _cursor = a->_bufferPtr; + } + return kErrorOk; + } + + ASMJIT_FORCE_INLINE uint8_t* cursor() const noexcept { return _cursor; } + ASMJIT_FORCE_INLINE void setCursor(uint8_t* cursor) noexcept { _cursor = cursor; } + ASMJIT_FORCE_INLINE void advance(size_t n) noexcept { _cursor += n; } + + ASMJIT_FORCE_INLINE size_t offsetFrom(uint8_t* from) const noexcept { + ASMJIT_ASSERT(_cursor >= from); + return (size_t)(_cursor - from); + } + + template + ASMJIT_FORCE_INLINE void emit8(T val) noexcept { + typedef typename std::make_unsigned::type U; + _cursor[0] = uint8_t(U(val) & U(0xFF)); + _cursor++; + } + + template + ASMJIT_FORCE_INLINE void emit8If(T val, Y cond) noexcept { + typedef typename std::make_unsigned::type U; + ASMJIT_ASSERT(size_t(cond) <= 1u); + + _cursor[0] = uint8_t(U(val) & U(0xFF)); + _cursor += size_t(cond); + } + + template + ASMJIT_FORCE_INLINE void emit16uLE(T val) noexcept { + typedef typename std::make_unsigned::type U; + Support::writeU16uLE(_cursor, uint16_t(U(val) & 0xFFFFu)); + _cursor += 2; + } + + template + ASMJIT_FORCE_INLINE void emit16uBE(T val) noexcept { + typedef typename std::make_unsigned::type U; + Support::writeU16uBE(_cursor, uint16_t(U(val) & 0xFFFFu)); + _cursor += 2; + } + + template + ASMJIT_FORCE_INLINE void emit32uLE(T val) noexcept { + typedef typename std::make_unsigned::type U; + Support::writeU32uLE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + _cursor += 4; + } + + template + ASMJIT_FORCE_INLINE void emit32uBE(T val) noexcept { + typedef typename std::make_unsigned::type U; + Support::writeU32uBE(_cursor, uint32_t(U(val) & 0xFFFFFFFFu)); + _cursor += 4; + } + + ASMJIT_FORCE_INLINE void emitData(const void* data, size_t size) noexcept { + ASMJIT_ASSERT(size != 0); + memcpy(_cursor, data, size); + _cursor += size; + } + + template + ASMJIT_FORCE_INLINE void emitValueLE(const T& value, size_t size) noexcept { + typedef typename std::make_unsigned::type U; + ASMJIT_ASSERT(size <= sizeof(T)); + + U v = U(value); + for (uint32_t i = 0; i < size; i++) { + _cursor[i] = uint8_t(v & 0xFFu); + v >>= 8; + } + _cursor += size; + } + + template + ASMJIT_FORCE_INLINE void emitValueBE(const T& value, size_t size) noexcept { + typedef typename std::make_unsigned::type U; + ASMJIT_ASSERT(size <= sizeof(T)); + + U v = U(value); + for (uint32_t i = 0; i < size; i++) { + _cursor[i] = uint8_t(v >> (sizeof(T) - 8)); + v <<= 8; + } + _cursor += size; + } + + ASMJIT_FORCE_INLINE void emitZeros(size_t size) noexcept { + ASMJIT_ASSERT(size != 0); + memset(_cursor, 0, size); + _cursor += size; + } + + ASMJIT_FORCE_INLINE void remove8(uint8_t* where) noexcept { + ASMJIT_ASSERT(where < _cursor); + + uint8_t* p = where; + while (++p != _cursor) + p[-1] = p[0]; + _cursor--; + } + + template + ASMJIT_FORCE_INLINE void insert8(uint8_t* where, T val) noexcept { + uint8_t* p = _cursor; + + while (p != where) { + p[0] = p[-1]; + p--; + } + + *p = uint8_t(val & 0xFF); + _cursor++; + } + + ASMJIT_FORCE_INLINE void done(BaseAssembler* a) noexcept { + CodeBuffer& buffer = a->_section->_buffer; + size_t newSize = (size_t)(_cursor - a->_bufferData); + ASMJIT_ASSERT(newSize <= buffer.capacity()); + + a->_bufferPtr = _cursor; + buffer._size = Support::max(buffer._size, newSize); + } +}; + +//! Code writer utilities. +namespace CodeWriterUtils { + +bool encodeOffset32(uint32_t* dst, int64_t offset64, const OffsetFormat& format) noexcept; +bool encodeOffset64(uint64_t* dst, int64_t offset64, const OffsetFormat& format) noexcept; + +bool writeOffset(void* dst, int64_t offset64, const OffsetFormat& format) noexcept; + +} // {CodeWriterUtils} + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_CODEBUFFERWRITER_P_H_INCLUDED diff --git a/src/asmjit/core/compiler.cpp b/src/asmjit/core/compiler.cpp new file mode 100644 index 0000000..b1c6b80 --- /dev/null +++ b/src/asmjit/core/compiler.cpp @@ -0,0 +1,582 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_COMPILER + +#include "../core/assembler.h" +#include "../core/compiler.h" +#include "../core/cpuinfo.h" +#include "../core/logger.h" +#include "../core/rapass_p.h" +#include "../core/rastack_p.h" +#include "../core/support.h" +#include "../core/type.h" + +ASMJIT_BEGIN_NAMESPACE + +// GlobalConstPoolPass +// =================== + +class GlobalConstPoolPass : public Pass { +public: + typedef Pass Base; +public: + ASMJIT_NONCOPYABLE(GlobalConstPoolPass) + + GlobalConstPoolPass() noexcept : Pass("GlobalConstPoolPass") {} + + Error run(Zone* zone, Logger* logger) override { + DebugUtils::unused(zone, logger); + + // Flush the global constant pool. + BaseCompiler* compiler = static_cast(_cb); + ConstPoolNode* globalConstPool = compiler->_constPools[uint32_t(ConstPoolScope::kGlobal)]; + + if (globalConstPool) { + compiler->addAfter(globalConstPool, compiler->lastNode()); + compiler->_constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; + } + + return kErrorOk; + } +}; + +// BaseCompiler - Construction & Destruction +// ========================================= + +BaseCompiler::BaseCompiler() noexcept + : BaseBuilder(), + _func(nullptr), + _vRegZone(4096 - Zone::kBlockOverhead), + _vRegArray(), + _constPools { nullptr, nullptr } { + _emitterType = EmitterType::kCompiler; + _validationFlags = ValidationFlags::kEnableVirtRegs; +} +BaseCompiler::~BaseCompiler() noexcept {} + +// BaseCompiler - Function Management +// ================================== + +Error BaseCompiler::newFuncNode(FuncNode** out, const FuncSignature& signature) { + *out = nullptr; + + // Create FuncNode together with all the required surrounding nodes. + FuncNode* funcNode; + ASMJIT_PROPAGATE(_newNodeT(&funcNode)); + ASMJIT_PROPAGATE(newLabelNode(&funcNode->_exitNode)); + ASMJIT_PROPAGATE(_newNodeT(&funcNode->_end, SentinelType::kFuncEnd)); + + // Initialize the function's detail info. + Error err = funcNode->detail().init(signature, environment()); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + // If the Target guarantees greater stack alignment than required by the calling convention + // then override it as we can prevent having to perform dynamic stack alignment + uint32_t environmentStackAlignment = _environment.stackAlignment(); + + if (funcNode->_funcDetail._callConv.naturalStackAlignment() < environmentStackAlignment) + funcNode->_funcDetail._callConv.setNaturalStackAlignment(environmentStackAlignment); + + // Initialize the function frame. + err = funcNode->_frame.init(funcNode->_funcDetail); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + // Allocate space for function arguments. + funcNode->_args = nullptr; + if (funcNode->argCount() != 0) { + funcNode->_args = _allocator.allocT(funcNode->argCount() * sizeof(FuncNode::ArgPack)); + if (ASMJIT_UNLIKELY(!funcNode->_args)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + memset(funcNode->_args, 0, funcNode->argCount() * sizeof(FuncNode::ArgPack)); + } + + ASMJIT_PROPAGATE(registerLabelNode(funcNode)); + + *out = funcNode; + return kErrorOk; +} + +Error BaseCompiler::addFuncNode(FuncNode** out, const FuncSignature& signature) { + ASMJIT_PROPAGATE(newFuncNode(out, signature)); + ASMJIT_ASSUME(*out != nullptr); + + addFunc(*out); + return kErrorOk; +} + +Error BaseCompiler::newFuncRetNode(FuncRetNode** out, const Operand_& o0, const Operand_& o1) { + uint32_t opCount = !o1.isNone() ? 2u : !o0.isNone() ? 1u : 0u; + FuncRetNode* node; + + ASMJIT_PROPAGATE(_newNodeT(&node)); + ASMJIT_ASSUME(node != nullptr); + + node->setOpCount(opCount); + node->setOp(0, o0); + node->setOp(1, o1); + node->resetOpRange(2, node->opCapacity()); + + *out = node; + return kErrorOk; +} + +Error BaseCompiler::addFuncRetNode(FuncRetNode** out, const Operand_& o0, const Operand_& o1) { + ASMJIT_PROPAGATE(newFuncRetNode(out, o0, o1)); + addNode(*out); + return kErrorOk; +} + +FuncNode* BaseCompiler::addFunc(FuncNode* func) { + _func = func; + + addNode(func); // Function node. + BaseNode* prev = cursor(); // {CURSOR}. + addNode(func->exitNode()); // Function exit label. + addNode(func->endNode()); // Function end sentinel. + + _setCursor(prev); + return func; +} + +Error BaseCompiler::endFunc() { + FuncNode* func = _func; + + if (ASMJIT_UNLIKELY(!func)) + return reportError(DebugUtils::errored(kErrorInvalidState)); + + // Add the local constant pool at the end of the function (if exists). + ConstPoolNode* localConstPool = _constPools[uint32_t(ConstPoolScope::kLocal)]; + if (localConstPool) { + setCursor(func->endNode()->prev()); + addNode(localConstPool); + _constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; + } + + // Mark as finished. + _func = nullptr; + + SentinelNode* end = func->endNode(); + setCursor(end); + + return kErrorOk; +} + +// BaseCompiler - Function Invocation +// ================================== + +Error BaseCompiler::newInvokeNode(InvokeNode** out, InstId instId, const Operand_& o0, const FuncSignature& signature) { + InvokeNode* node; + ASMJIT_PROPAGATE(_newNodeT(&node, instId, InstOptions::kNone)); + + node->setOpCount(1); + node->setOp(0, o0); + node->resetOpRange(1, node->opCapacity()); + + Error err = node->detail().init(signature, environment()); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + // Skip the allocation if there are no arguments. + uint32_t argCount = signature.argCount(); + if (argCount) { + node->_args = static_cast(_allocator.alloc(argCount * sizeof(InvokeNode::OperandPack))); + if (!node->_args) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + memset(node->_args, 0, argCount * sizeof(InvokeNode::OperandPack)); + } + + *out = node; + return kErrorOk; +} + +Error BaseCompiler::addInvokeNode(InvokeNode** out, InstId instId, const Operand_& o0, const FuncSignature& signature) { + ASMJIT_PROPAGATE(newInvokeNode(out, instId, o0, signature)); + addNode(*out); + return kErrorOk; +} + +// BaseCompiler - Virtual Registers +// ================================ + +static void BaseCompiler_assignGenericName(BaseCompiler* self, VirtReg* vReg) { + uint32_t index = unsigned(Operand::virtIdToIndex(vReg->_id)); + + char buf[64]; + int size = snprintf(buf, ASMJIT_ARRAY_SIZE(buf), "%%%u", unsigned(index)); + + ASMJIT_ASSERT(size > 0 && size < int(ASMJIT_ARRAY_SIZE(buf))); + vReg->_name.setData(&self->_dataZone, buf, unsigned(size)); +} + +Error BaseCompiler::newVirtReg(VirtReg** out, TypeId typeId, OperandSignature signature, const char* name) { + *out = nullptr; + uint32_t index = _vRegArray.size(); + + if (ASMJIT_UNLIKELY(index >= uint32_t(Operand::kVirtIdCount))) + return reportError(DebugUtils::errored(kErrorTooManyVirtRegs)); + + if (ASMJIT_UNLIKELY(_vRegArray.willGrow(&_allocator) != kErrorOk)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + VirtReg* vReg = _vRegZone.allocZeroedT(); + if (ASMJIT_UNLIKELY(!vReg)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + uint32_t size = TypeUtils::sizeOf(typeId); + uint32_t alignment = Support::min(size, 64); + + vReg = new(vReg) VirtReg(signature, Operand::indexToVirtId(index), size, alignment, typeId); + +#ifndef ASMJIT_NO_LOGGING + if (name && name[0] != '\0') + vReg->_name.setData(&_dataZone, name, SIZE_MAX); + else + BaseCompiler_assignGenericName(this, vReg); +#else + DebugUtils::unused(name); +#endif + + _vRegArray.appendUnsafe(vReg); + *out = vReg; + + return kErrorOk; +} + +Error BaseCompiler::_newReg(BaseReg* out, TypeId typeId, const char* name) { + OperandSignature regSignature; + out->reset(); + + Error err = ArchUtils::typeIdToRegSignature(arch(), typeId, &typeId, ®Signature); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + VirtReg* vReg; + ASMJIT_PROPAGATE(newVirtReg(&vReg, typeId, regSignature, name)); + ASMJIT_ASSUME(vReg != nullptr); + + out->_initReg(regSignature, vReg->id()); + return kErrorOk; +} + +Error BaseCompiler::_newRegFmt(BaseReg* out, TypeId typeId, const char* fmt, ...) { + va_list ap; + StringTmp<256> sb; + + va_start(ap, fmt); + sb.appendVFormat(fmt, ap); + va_end(ap); + + return _newReg(out, typeId, sb.data()); +} + +Error BaseCompiler::_newReg(BaseReg* out, const BaseReg& ref, const char* name) { + out->reset(); + + OperandSignature regSignature; + TypeId typeId; + + if (isVirtRegValid(ref)) { + VirtReg* vRef = virtRegByReg(ref); + typeId = vRef->typeId(); + + // NOTE: It's possible to cast one register type to another if it's the same register group. However, VirtReg + // always contains the TypeId that was used to create the register. This means that in some cases we may end + // up having different size of `ref` and `vRef`. In such case we adjust the TypeId to match the `ref` register + // type instead of the original register type, which should be the expected behavior. + uint32_t typeSize = TypeUtils::sizeOf(typeId); + uint32_t refSize = ref.size(); + + if (typeSize != refSize) { + if (TypeUtils::isInt(typeId)) { + // GP register - change TypeId to match `ref`, but keep sign of `vRef`. + switch (refSize) { + case 1: typeId = TypeId(uint32_t(TypeId::kInt8 ) | (uint32_t(typeId) & 1)); break; + case 2: typeId = TypeId(uint32_t(TypeId::kInt16) | (uint32_t(typeId) & 1)); break; + case 4: typeId = TypeId(uint32_t(TypeId::kInt32) | (uint32_t(typeId) & 1)); break; + case 8: typeId = TypeId(uint32_t(TypeId::kInt64) | (uint32_t(typeId) & 1)); break; + default: typeId = TypeId::kVoid; break; + } + } + else if (TypeUtils::isMmx(typeId)) { + // MMX register - always use 64-bit. + typeId = TypeId::kMmx64; + } + else if (TypeUtils::isMask(typeId)) { + // Mask register - change TypeId to match `ref` size. + switch (refSize) { + case 1: typeId = TypeId::kMask8; break; + case 2: typeId = TypeId::kMask16; break; + case 4: typeId = TypeId::kMask32; break; + case 8: typeId = TypeId::kMask64; break; + default: typeId = TypeId::kVoid; break; + } + } + else { + // Vector register - change TypeId to match `ref` size, keep vector metadata. + TypeId scalarTypeId = TypeUtils::scalarOf(typeId); + switch (refSize) { + case 16: typeId = TypeUtils::scalarToVector(scalarTypeId, TypeId::_kVec128Start); break; + case 32: typeId = TypeUtils::scalarToVector(scalarTypeId, TypeId::_kVec256Start); break; + case 64: typeId = TypeUtils::scalarToVector(scalarTypeId, TypeId::_kVec512Start); break; + default: typeId = TypeId::kVoid; break; + } + } + + if (typeId == TypeId::kVoid) + return reportError(DebugUtils::errored(kErrorInvalidState)); + } + } + else { + typeId = ArchTraits::byArch(arch()).regTypeToTypeId(ref.type()); + } + + Error err = ArchUtils::typeIdToRegSignature(arch(), typeId, &typeId, ®Signature); + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + VirtReg* vReg; + ASMJIT_PROPAGATE(newVirtReg(&vReg, typeId, regSignature, name)); + ASMJIT_ASSUME(vReg != nullptr); + + out->_initReg(regSignature, vReg->id()); + return kErrorOk; +} + +Error BaseCompiler::_newRegFmt(BaseReg* out, const BaseReg& ref, const char* fmt, ...) { + va_list ap; + StringTmp<256> sb; + + va_start(ap, fmt); + sb.appendVFormat(fmt, ap); + va_end(ap); + + return _newReg(out, ref, sb.data()); +} + +Error BaseCompiler::_newStack(BaseMem* out, uint32_t size, uint32_t alignment, const char* name) { + out->reset(); + + if (size == 0) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + if (alignment == 0) + alignment = 1; + + if (!Support::isPowerOf2(alignment)) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + if (alignment > 64) + alignment = 64; + + VirtReg* vReg; + ASMJIT_PROPAGATE(newVirtReg(&vReg, TypeId::kVoid, OperandSignature{0}, name)); + ASMJIT_ASSUME(vReg != nullptr); + + vReg->_virtSize = size; + vReg->_isStack = true; + vReg->_alignment = uint8_t(alignment); + + // Set the memory operand to GPD/GPQ and its id to VirtReg. + *out = BaseMem(OperandSignature::fromOpType(OperandType::kMem) | + OperandSignature::fromMemBaseType(_gpSignature.regType()) | + OperandSignature::fromBits(OperandSignature::kMemRegHomeFlag), + vReg->id(), 0, 0); + return kErrorOk; +} + +Error BaseCompiler::setStackSize(uint32_t virtId, uint32_t newSize, uint32_t newAlignment) { + if (!isVirtIdValid(virtId)) + return DebugUtils::errored(kErrorInvalidVirtId); + + if (newAlignment && !Support::isPowerOf2(newAlignment)) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + if (newAlignment > 64) + newAlignment = 64; + + VirtReg* vReg = virtRegById(virtId); + if (newSize) + vReg->_virtSize = newSize; + + if (newAlignment) + vReg->_alignment = uint8_t(newAlignment); + + // This is required if the RAPass is already running. There is a chance that a stack-slot has been already + // allocated and in that case it has to be updated as well, otherwise we would allocate wrong amount of memory. + RAWorkReg* workReg = vReg->_workReg; + if (workReg && workReg->_stackSlot) { + workReg->_stackSlot->_size = vReg->_virtSize; + workReg->_stackSlot->_alignment = vReg->_alignment; + } + + return kErrorOk; +} + +Error BaseCompiler::_newConst(BaseMem* out, ConstPoolScope scope, const void* data, size_t size) { + out->reset(); + + if (uint32_t(scope) > 1) + return reportError(DebugUtils::errored(kErrorInvalidArgument)); + + if (!_constPools[uint32_t(scope)]) + ASMJIT_PROPAGATE(newConstPoolNode(&_constPools[uint32_t(scope)])); + + ConstPoolNode* pool = _constPools[uint32_t(scope)]; + size_t off; + Error err = pool->add(data, size, off); + + if (ASMJIT_UNLIKELY(err)) + return reportError(err); + + *out = BaseMem(OperandSignature::fromOpType(OperandType::kMem) | + OperandSignature::fromMemBaseType(RegType::kLabelTag) | + OperandSignature::fromSize(uint32_t(size)), + pool->labelId(), 0, int32_t(off)); + return kErrorOk; +} + +void BaseCompiler::rename(const BaseReg& reg, const char* fmt, ...) { + if (!reg.isVirtReg()) return; + + VirtReg* vReg = virtRegById(reg.id()); + if (!vReg) return; + + if (fmt && fmt[0] != '\0') { + char buf[128]; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, ASMJIT_ARRAY_SIZE(buf), fmt, ap); + va_end(ap); + + vReg->_name.setData(&_dataZone, buf, SIZE_MAX); + } + else { + BaseCompiler_assignGenericName(this, vReg); + } +} + +// BaseCompiler - Jump Annotations +// =============================== + +Error BaseCompiler::newJumpNode(JumpNode** out, InstId instId, InstOptions instOptions, const Operand_& o0, JumpAnnotation* annotation) { + JumpNode* node = _allocator.allocT(); + uint32_t opCount = 1; + + *out = node; + if (ASMJIT_UNLIKELY(!node)) + return reportError(DebugUtils::errored(kErrorOutOfMemory)); + + node = new(node) JumpNode(this, instId, instOptions, opCount, annotation); + node->setOp(0, o0); + node->resetOpRange(opCount, JumpNode::kBaseOpCapacity); + + return kErrorOk; +} + +Error BaseCompiler::emitAnnotatedJump(InstId instId, const Operand_& o0, JumpAnnotation* annotation) { + InstOptions options = instOptions() | forcedInstOptions(); + RegOnly extra = extraReg(); + const char* comment = inlineComment(); + + resetInstOptions(); + resetInlineComment(); + resetExtraReg(); + + JumpNode* node; + ASMJIT_PROPAGATE(newJumpNode(&node, instId, options, o0, annotation)); + + node->setExtraReg(extra); + if (comment) + node->setInlineComment(static_cast(_dataZone.dup(comment, strlen(comment), true))); + + addNode(node); + return kErrorOk; +} + +JumpAnnotation* BaseCompiler::newJumpAnnotation() { + if (_jumpAnnotations.grow(&_allocator, 1) != kErrorOk) { + reportError(DebugUtils::errored(kErrorOutOfMemory)); + return nullptr; + } + + uint32_t id = _jumpAnnotations.size(); + JumpAnnotation* jumpAnnotation = _allocator.newT(this, id); + + if (!jumpAnnotation) { + reportError(DebugUtils::errored(kErrorOutOfMemory)); + return nullptr; + } + + _jumpAnnotations.appendUnsafe(jumpAnnotation); + return jumpAnnotation; +} + +// BaseCompiler - Events +// ===================== + +Error BaseCompiler::onAttach(CodeHolder* code) noexcept { + ASMJIT_PROPAGATE(Base::onAttach(code)); + + const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); + RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; + _gpSignature = archTraits.regTypeToSignature(nativeRegType); + + Error err = addPassT(); + if (ASMJIT_UNLIKELY(err)) { + onDetach(code); + return err; + } + + return kErrorOk; +} + +Error BaseCompiler::onDetach(CodeHolder* code) noexcept { + _func = nullptr; + _constPools[uint32_t(ConstPoolScope::kLocal)] = nullptr; + _constPools[uint32_t(ConstPoolScope::kGlobal)] = nullptr; + + _vRegArray.reset(); + _vRegZone.reset(); + + return Base::onDetach(code); +} + +// FuncPass - Construction & Destruction +// ===================================== + +FuncPass::FuncPass(const char* name) noexcept + : Pass(name) {} + +// FuncPass - Run +// ============== + +Error FuncPass::run(Zone* zone, Logger* logger) { + BaseNode* node = cb()->firstNode(); + if (!node) return kErrorOk; + + do { + if (node->type() == NodeType::kFunc) { + FuncNode* func = node->as(); + node = func->endNode(); + ASMJIT_PROPAGATE(runOnFunction(zone, logger, func)); + } + + // Find a function by skipping all nodes that are not `NodeType::kFunc`. + do { + node = node->next(); + } while (node && node->type() != NodeType::kFunc); + } while (node); + + return kErrorOk; +} + +ASMJIT_END_NAMESPACE + +#endif // !ASMJIT_NO_COMPILER diff --git a/src/asmjit/core/compiler.h b/src/asmjit/core/compiler.h new file mode 100644 index 0000000..709fd95 --- /dev/null +++ b/src/asmjit/core/compiler.h @@ -0,0 +1,737 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_COMPILER_H_INCLUDED +#define ASMJIT_CORE_COMPILER_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_COMPILER + +#include "../core/assembler.h" +#include "../core/builder.h" +#include "../core/constpool.h" +#include "../core/compilerdefs.h" +#include "../core/func.h" +#include "../core/inst.h" +#include "../core/operand.h" +#include "../core/support.h" +#include "../core/zone.h" +#include "../core/zonevector.h" + +ASMJIT_BEGIN_NAMESPACE + +class JumpAnnotation; +class JumpNode; +class FuncNode; +class FuncRetNode; +class InvokeNode; + +//! \addtogroup asmjit_compiler +//! \{ + +//! Code emitter that uses virtual registers and performs register allocation. +//! +//! Compiler is a high-level code-generation tool that provides register allocation and automatic handling of function +//! calling conventions. It was primarily designed for merging multiple parts of code into a function without worrying +//! about registers and function calling conventions. +//! +//! BaseCompiler can be used, with a minimum effort, to handle 32-bit and 64-bit code generation within a single code +//! base. +//! +//! BaseCompiler is based on BaseBuilder and contains all the features it provides. It means that the code it stores +//! can be modified (removed, added, injected) and analyzed. When the code is finalized the compiler can emit the code +//! into an Assembler to translate the abstract representation into a machine code. +//! +//! Check out architecture specific compilers for more details and examples: +//! +//! - \ref x86::Compiler - X86/X64 compiler implementation. +class ASMJIT_VIRTAPI BaseCompiler : public BaseBuilder { +public: + ASMJIT_NONCOPYABLE(BaseCompiler) + typedef BaseBuilder Base; + + //! \name Members + //! \{ + + //! Current function. + FuncNode* _func; + //! Allocates `VirtReg` objects. + Zone _vRegZone; + //! Stores array of `VirtReg` pointers. + ZoneVector _vRegArray; + //! Stores jump annotations. + ZoneVector _jumpAnnotations; + + //! Local and global constant pools. + //! + //! Local constant pool is flushed with each function, global constant pool is flushed only by \ref finalize(). + ConstPoolNode* _constPools[2]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `BaseCompiler` instance. + ASMJIT_API BaseCompiler() noexcept; + //! Destroys the `BaseCompiler` instance. + ASMJIT_API virtual ~BaseCompiler() noexcept; + + //! \} + + //! \name Function Management + //! \{ + + //! Creates a new \ref FuncNode. + ASMJIT_API Error newFuncNode(FuncNode** ASMJIT_NONNULL(out), const FuncSignature& signature); + //! Creates a new \ref FuncNode adds it to the instruction stream. + ASMJIT_API Error addFuncNode(FuncNode** ASMJIT_NONNULL(out), const FuncSignature& signature); + + //! Creates a new \ref FuncRetNode. + ASMJIT_API Error newFuncRetNode(FuncRetNode** ASMJIT_NONNULL(out), const Operand_& o0, const Operand_& o1); + //! Creates a new \ref FuncRetNode and adds it to the instruction stream. + ASMJIT_API Error addFuncRetNode(FuncRetNode** ASMJIT_NONNULL(out), const Operand_& o0, const Operand_& o1); + + //! Returns the current function. + inline FuncNode* func() const noexcept { return _func; } + + //! Creates a new \ref FuncNode with the given `signature` and returns it. + inline FuncNode* newFunc(const FuncSignature& signature) { + FuncNode* node; + newFuncNode(&node, signature); + return node; + } + + //! Creates a new \ref FuncNode with the given `signature`, adds it to the instruction stream by using + //! the \ref addFunc(FuncNode*) overload, and returns it. + inline FuncNode* addFunc(const FuncSignature& signature) { + FuncNode* node; + addFuncNode(&node, signature); + return node; + } + + //! Adds a function `node` to the instruction stream. + ASMJIT_API FuncNode* addFunc(FuncNode* ASMJIT_NONNULL(func)); + //! Emits a sentinel that marks the end of the current function. + ASMJIT_API Error endFunc(); + +#if !defined(ASMJIT_NO_DEPRECATED) + inline Error _setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg); + + //! Sets a function argument at `argIndex` to `reg`. + ASMJIT_DEPRECATED("Setting arguments through Compiler is deprecated, use FuncNode->setArg() instead") + inline Error setArg(size_t argIndex, const BaseReg& reg) { return _setArg(argIndex, 0, reg); } + + //! Sets a function argument at `argIndex` at `valueIndex` to `reg`. + ASMJIT_DEPRECATED("Setting arguments through Compiler is deprecated, use FuncNode->setArg() instead") + inline Error setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) { return _setArg(argIndex, valueIndex, reg); } +#endif + + inline Error addRet(const Operand_& o0, const Operand_& o1) { + FuncRetNode* node; + return addFuncRetNode(&node, o0, o1); + } + + //! \} + + //! \name Function Invocation + //! \{ + + //! Creates a new \ref InvokeNode. + ASMJIT_API Error newInvokeNode(InvokeNode** ASMJIT_NONNULL(out), InstId instId, const Operand_& o0, const FuncSignature& signature); + //! Creates a new \ref InvokeNode and adds it to the instruction stream. + ASMJIT_API Error addInvokeNode(InvokeNode** ASMJIT_NONNULL(out), InstId instId, const Operand_& o0, const FuncSignature& signature); + + //! \} + + //! \name Virtual Registers + //! \{ + + //! Creates a new virtual register representing the given `typeId` and `signature`. + //! + //! \note This function is public, but it's not generally recommended to be used by AsmJit users, use architecture + //! specific `newReg()` functionality instead or functions like \ref _newReg() and \ref _newRegFmt(). + ASMJIT_API Error newVirtReg(VirtReg** ASMJIT_NONNULL(out), TypeId typeId, OperandSignature signature, const char* name); + + //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. + ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* name = nullptr); + + //! Creates a new virtual register of the given `typeId` and stores it to `out` operand. + //! + //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. + ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), TypeId typeId, const char* fmt, ...); + + //! Creates a new virtual register compatible with the provided reference register `ref`. + ASMJIT_API Error _newReg(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* name = nullptr); + + //! Creates a new virtual register compatible with the provided reference register `ref`. + //! + //! \note This version accepts a snprintf() format `fmt` followed by a variadic arguments. + ASMJIT_API Error _newRegFmt(BaseReg* ASMJIT_NONNULL(out), const BaseReg& ref, const char* fmt, ...); + + //! Tests whether the given `id` is a valid virtual register id. + inline bool isVirtIdValid(uint32_t id) const noexcept { + uint32_t index = Operand::virtIdToIndex(id); + return index < _vRegArray.size(); + } + //! Tests whether the given `reg` is a virtual register having a valid id. + inline bool isVirtRegValid(const BaseReg& reg) const noexcept { + return isVirtIdValid(reg.id()); + } + + //! Returns \ref VirtReg associated with the given `id`. + inline VirtReg* virtRegById(uint32_t id) const noexcept { + ASMJIT_ASSERT(isVirtIdValid(id)); + return _vRegArray[Operand::virtIdToIndex(id)]; + } + + //! Returns \ref VirtReg associated with the given `reg`. + inline VirtReg* virtRegByReg(const BaseReg& reg) const noexcept { return virtRegById(reg.id()); } + + //! Returns \ref VirtReg associated with the given virtual register `index`. + //! + //! \note This is not the same as virtual register id. The conversion between id and its index is implemented + //! by \ref Operand_::virtIdToIndex() and \ref Operand_::indexToVirtId() functions. + inline VirtReg* virtRegByIndex(uint32_t index) const noexcept { return _vRegArray[index]; } + + //! Returns an array of all virtual registers managed by the Compiler. + inline const ZoneVector& virtRegs() const noexcept { return _vRegArray; } + + //! \name Stack + //! \{ + + //! Creates a new stack of the given `size` and `alignment` and stores it to `out`. + //! + //! \note `name` can be used to give the stack a name, for debugging purposes. + ASMJIT_API Error _newStack(BaseMem* ASMJIT_NONNULL(out), uint32_t size, uint32_t alignment, const char* name = nullptr); + + //! Updates the stack size of a stack created by `_newStack()` by its `virtId`. + ASMJIT_API Error setStackSize(uint32_t virtId, uint32_t newSize, uint32_t newAlignment = 0); + + //! Updates the stack size of a stack created by `_newStack()`. + inline Error setStackSize(const BaseMem& mem, uint32_t newSize, uint32_t newAlignment = 0) { + return setStackSize(mem.id(), newSize, newAlignment); + } + + //! \} + + //! \name Constants + //! \{ + + //! Creates a new constant of the given `scope` (see \ref ConstPoolScope). + //! + //! This function adds a constant of the given `size` to the built-in \ref ConstPool and stores the reference to that + //! constant to the `out` operand. + ASMJIT_API Error _newConst(BaseMem* ASMJIT_NONNULL(out), ConstPoolScope scope, const void* data, size_t size); + + //! \} + + //! \name Miscellaneous + //! \{ + + //! Rename the given virtual register `reg` to a formatted string `fmt`. + ASMJIT_API void rename(const BaseReg& reg, const char* fmt, ...); + + //! \} + + //! \name Jump Annotations + //! \{ + + inline const ZoneVector& jumpAnnotations() const noexcept { + return _jumpAnnotations; + } + + ASMJIT_API Error newJumpNode(JumpNode** ASMJIT_NONNULL(out), InstId instId, InstOptions instOptions, const Operand_& o0, JumpAnnotation* annotation); + ASMJIT_API Error emitAnnotatedJump(InstId instId, const Operand_& o0, JumpAnnotation* annotation); + + //! Returns a new `JumpAnnotation` instance, which can be used to aggregate possible targets of a jump where the + //! target is not a label, for example to implement jump tables. + ASMJIT_API JumpAnnotation* newJumpAnnotation(); + + //! \} + + //! \name Events + //! \{ + + ASMJIT_API Error onAttach(CodeHolder* code) noexcept override; + ASMJIT_API Error onDetach(CodeHolder* code) noexcept override; + + //! \} +}; + +//! Jump annotation used to annotate jumps. +//! +//! \ref BaseCompiler allows to emit jumps where the target is either register or memory operand. Such jumps cannot be +//! trivially inspected, so instead of doing heuristics AsmJit allows to annotate such jumps with possible targets. +//! Register allocator then uses the annotation to construct control-flow, which is then used by liveness analysis and +//! other tools to prepare ground for register allocation. +class JumpAnnotation { +public: + ASMJIT_NONCOPYABLE(JumpAnnotation) + + //! \name Members + //! \{ + + //! Compiler that owns this JumpAnnotation. + BaseCompiler* _compiler; + //! Annotation identifier. + uint32_t _annotationId; + //! Vector of label identifiers, see \ref labelIds(). + ZoneVector _labelIds; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline JumpAnnotation(BaseCompiler* ASMJIT_NONNULL(compiler), uint32_t annotationId) noexcept + : _compiler(compiler), + _annotationId(annotationId) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the compiler that owns this JumpAnnotation. + inline BaseCompiler* compiler() const noexcept { return _compiler; } + //! Returns the annotation id. + inline uint32_t annotationId() const noexcept { return _annotationId; } + //! Returns a vector of label identifiers that lists all targets of the jump. + const ZoneVector& labelIds() const noexcept { return _labelIds; } + + //! Tests whether the given `label` is a target of this JumpAnnotation. + inline bool hasLabel(const Label& label) const noexcept { return hasLabelId(label.id()); } + //! Tests whether the given `labelId` is a target of this JumpAnnotation. + inline bool hasLabelId(uint32_t labelId) const noexcept { return _labelIds.contains(labelId); } + + //! \} + + //! \name Annotation Building API + //! \{ + + //! Adds the `label` to the list of targets of this JumpAnnotation. + inline Error addLabel(const Label& label) noexcept { return addLabelId(label.id()); } + //! Adds the `labelId` to the list of targets of this JumpAnnotation. + inline Error addLabelId(uint32_t labelId) noexcept { return _labelIds.append(&_compiler->_allocator, labelId); } + + //! \} +}; + +//! Jump instruction with \ref JumpAnnotation. +//! +//! \note This node should be only used to represent jump where the jump target cannot be deduced by examining +//! instruction operands. For example if the jump target is register or memory location. This pattern is often +//! used to perform indirect jumps that use jump table, e.g. to implement `switch{}` statement. +class JumpNode : public InstNode { +public: + ASMJIT_NONCOPYABLE(JumpNode) + + //! \name Members + //! \{ + + JumpAnnotation* _annotation; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline JumpNode(BaseCompiler* ASMJIT_NONNULL(cc), InstId instId, InstOptions options, uint32_t opCount, JumpAnnotation* annotation) noexcept + : InstNode(cc, instId, options, opCount, kBaseOpCapacity), + _annotation(annotation) { + setType(NodeType::kJump); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether this JumpNode has associated a \ref JumpAnnotation. + inline bool hasAnnotation() const noexcept { return _annotation != nullptr; } + //! Returns the \ref JumpAnnotation associated with this jump, or `nullptr`. + inline JumpAnnotation* annotation() const noexcept { return _annotation; } + //! Sets the \ref JumpAnnotation associated with this jump to `annotation`. + inline void setAnnotation(JumpAnnotation* annotation) noexcept { _annotation = annotation; } + + //! \} +}; + +//! Function node represents a function used by \ref BaseCompiler. +//! +//! A function is composed of the following: +//! +//! - Function entry, \ref FuncNode acts as a label, so the entry is implicit. To get the entry, simply use +//! \ref FuncNode::label(), which is the same as \ref LabelNode::label(). +//! +//! - Function exit, which is represented by \ref FuncNode::exitNode(). A helper function +//! \ref FuncNode::exitLabel() exists and returns an exit label instead of node. +//! +//! - Function \ref FuncNode::endNode() sentinel. This node marks the end of a function - there should be no +//! code that belongs to the function after this node, but the Compiler doesn't enforce that at the moment. +//! +//! - Function detail, see \ref FuncNode::detail(). +//! +//! - Function frame, see \ref FuncNode::frame(). +//! +//! - Function arguments mapped to virtual registers, see \ref FuncNode::argPacks(). +//! +//! In a node list, the function and its body looks like the following: +//! +//! \code{.unparsed} +//! [...] - Anything before the function. +//! +//! [FuncNode] - Entry point of the function, acts as a label as well. +//! - Prolog inserted by the register allocator. +//! {...} - Function body - user code basically. +//! [ExitLabel] - Exit label +//! - Epilog inserted by the register allocator. +//! - Return inserted by the register allocator. +//! {...} - Can contain data or user code (error handling, special cases, ...). +//! [FuncEnd] - End sentinel +//! +//! [...] - Anything after the function. +//! \endcode +//! +//! When a function is added to the instruction stream by \ref BaseCompiler::addFunc() it actually inserts 3 nodes +//! (FuncNode, ExitLabel, and FuncEnd) and sets the current cursor to be FuncNode. When \ref BaseCompiler::endFunc() +//! is called the cursor is set to FuncEnd. This guarantees that user can use ExitLabel as a marker after additional +//! code or data can be placed, which is a common practice. +class FuncNode : public LabelNode { +public: + ASMJIT_NONCOPYABLE(FuncNode) + + //! Arguments pack. + struct ArgPack { + RegOnly _data[Globals::kMaxValuePack]; + + inline void reset() noexcept { + for (size_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) + _data[valueIndex].reset(); + } + + inline RegOnly& operator[](size_t valueIndex) noexcept { return _data[valueIndex]; } + inline const RegOnly& operator[](size_t valueIndex) const noexcept { return _data[valueIndex]; } + }; + + //! \name Members + //! \{ + + //! Function detail. + FuncDetail _funcDetail; + //! Function frame. + FuncFrame _frame; + //! Function exit label. + LabelNode* _exitNode; + //! Function end (sentinel). + SentinelNode* _end; + //! Argument packs. + ArgPack* _args; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `FuncNode` instance. + //! + //! Always use `BaseCompiler::addFunc()` to create a new `FuncNode`. + inline FuncNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept + : LabelNode(cb), + _funcDetail(), + _frame(), + _exitNode(nullptr), + _end(nullptr), + _args(nullptr) { + setType(NodeType::kFunc); + } + + //! \} + + //! \{ + //! \name Accessors + + //! Returns function exit `LabelNode`. + inline LabelNode* exitNode() const noexcept { return _exitNode; } + //! Returns function exit label. + inline Label exitLabel() const noexcept { return _exitNode->label(); } + + //! Returns "End of Func" sentinel node. + inline SentinelNode* endNode() const noexcept { return _end; } + + //! Returns function detail. + inline FuncDetail& detail() noexcept { return _funcDetail; } + //! Returns function detail. + inline const FuncDetail& detail() const noexcept { return _funcDetail; } + + //! Returns function frame. + inline FuncFrame& frame() noexcept { return _frame; } + //! Returns function frame. + inline const FuncFrame& frame() const noexcept { return _frame; } + + //! Returns function attributes. + inline FuncAttributes attributes() const noexcept { return _frame.attributes(); } + //! Adds `attrs` to the function attributes. + inline void addAttributes(FuncAttributes attrs) noexcept { _frame.addAttributes(attrs); } + + //! Returns arguments count. + inline uint32_t argCount() const noexcept { return _funcDetail.argCount(); } + //! Returns argument packs. + inline ArgPack* argPacks() const noexcept { return _args; } + + //! Tests whether the function has a return value. + inline bool hasRet() const noexcept { return _funcDetail.hasRet(); } + + //! Returns argument pack at `argIndex`. + inline ArgPack& argPack(size_t argIndex) const noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + return _args[argIndex]; + } + + //! Sets argument at `argIndex`. + inline void setArg(size_t argIndex, const BaseReg& vReg) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][0].init(vReg); + } + + //! \overload + inline void setArg(size_t argIndex, const RegOnly& vReg) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][0].init(vReg); + } + + //! Sets argument at `argIndex` and `valueIndex`. + inline void setArg(size_t argIndex, size_t valueIndex, const BaseReg& vReg) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][valueIndex].init(vReg); + } + + //! \overload + inline void setArg(size_t argIndex, size_t valueIndex, const RegOnly& vReg) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][valueIndex].init(vReg); + } + + //! Resets argument pack at `argIndex`. + inline void resetArg(size_t argIndex) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex].reset(); + } + + //! Resets argument pack at `argIndex`. + inline void resetArg(size_t argIndex, size_t valueIndex) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][valueIndex].reset(); + } + + //! \} +}; + +//! Function return, used by \ref BaseCompiler. +class FuncRetNode : public InstNode { +public: + ASMJIT_NONCOPYABLE(FuncRetNode) + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `FuncRetNode` instance. + inline FuncRetNode(BaseBuilder* ASMJIT_NONNULL(cb)) noexcept : InstNode(cb, BaseInst::kIdAbstract, InstOptions::kNone, 0) { + _any._nodeType = NodeType::kFuncRet; + } + + //! \} +}; + +//! Function invocation, used by \ref BaseCompiler. +class InvokeNode : public InstNode { +public: + ASMJIT_NONCOPYABLE(InvokeNode) + + //! Operand pack provides multiple operands that can be associated with a single return value of function + //! argument. Sometims this is necessary to express an argument or return value that requires multiple + //! registers, for example 64-bit value in 32-bit mode or passing / returning homogeneous data structures. + struct OperandPack { + //! Operands. + Operand_ _data[Globals::kMaxValuePack]; + + //! Reset the pack by resetting all operands in the pack. + inline void reset() noexcept { + for (size_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) + _data[valueIndex].reset(); + } + + //! Returns an operand at the given `valueIndex`. + inline Operand& operator[](size_t valueIndex) noexcept { + ASMJIT_ASSERT(valueIndex < Globals::kMaxValuePack); + return _data[valueIndex].as(); + } + + //! Returns an operand at the given `valueIndex` (const). + const inline Operand& operator[](size_t valueIndex) const noexcept { + ASMJIT_ASSERT(valueIndex < Globals::kMaxValuePack); + return _data[valueIndex].as(); + } + }; + + //! \name Members + //! \{ + + //! Function detail. + FuncDetail _funcDetail; + //! Function return value(s). + OperandPack _rets; + //! Function arguments. + OperandPack* _args; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `InvokeNode` instance. + inline InvokeNode(BaseBuilder* ASMJIT_NONNULL(cb), InstId instId, InstOptions options) noexcept + : InstNode(cb, instId, options, kBaseOpCapacity), + _funcDetail(), + _args(nullptr) { + setType(NodeType::kInvoke); + _resetOps(); + _rets.reset(); + addFlags(NodeFlags::kIsRemovable); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Sets the function signature. + inline Error init(const FuncSignature& signature, const Environment& environment) noexcept { + return _funcDetail.init(signature, environment); + } + + //! Returns the function detail. + inline FuncDetail& detail() noexcept { return _funcDetail; } + //! Returns the function detail. + inline const FuncDetail& detail() const noexcept { return _funcDetail; } + + //! Returns the target operand. + inline Operand& target() noexcept { return _opArray[0].as(); } + //! \overload + inline const Operand& target() const noexcept { return _opArray[0].as(); } + + //! Returns the number of function return values. + inline bool hasRet() const noexcept { return _funcDetail.hasRet(); } + //! Returns the number of function arguments. + inline uint32_t argCount() const noexcept { return _funcDetail.argCount(); } + + //! Returns operand pack representing function return value(s). + inline OperandPack& retPack() noexcept { return _rets; } + //! Returns operand pack representing function return value(s). + inline const OperandPack& retPack() const noexcept { return _rets; } + + //! Returns the return value at the given `valueIndex`. + inline Operand& ret(size_t valueIndex = 0) noexcept { return _rets[valueIndex]; } + //! \overload + inline const Operand& ret(size_t valueIndex = 0) const noexcept { return _rets[valueIndex]; } + + //! Returns operand pack representing function return value(s). + inline OperandPack& argPack(size_t argIndex) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + return _args[argIndex]; + } + //! \overload + inline const OperandPack& argPack(size_t argIndex) const noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + return _args[argIndex]; + } + + //! Returns a function argument at the given `argIndex`. + inline Operand& arg(size_t argIndex, size_t valueIndex) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + return _args[argIndex][valueIndex]; + } + //! \overload + inline const Operand& arg(size_t argIndex, size_t valueIndex) const noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + return _args[argIndex][valueIndex]; + } + + //! Sets the function return value at `i` to `op`. + inline void _setRet(size_t valueIndex, const Operand_& op) noexcept { _rets[valueIndex] = op; } + //! Sets the function argument at `i` to `op`. + inline void _setArg(size_t argIndex, size_t valueIndex, const Operand_& op) noexcept { + ASMJIT_ASSERT(argIndex < argCount()); + _args[argIndex][valueIndex] = op; + } + + //! Sets the function return value at `valueIndex` to `reg`. + inline void setRet(size_t valueIndex, const BaseReg& reg) noexcept { _setRet(valueIndex, reg); } + + //! Sets the first function argument in a value-pack at `argIndex` to `reg`. + inline void setArg(size_t argIndex, const BaseReg& reg) noexcept { _setArg(argIndex, 0, reg); } + //! Sets the first function argument in a value-pack at `argIndex` to `imm`. + inline void setArg(size_t argIndex, const Imm& imm) noexcept { _setArg(argIndex, 0, imm); } + + //! Sets the function argument at `argIndex` and `valueIndex` to `reg`. + inline void setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) noexcept { _setArg(argIndex, valueIndex, reg); } + //! Sets the function argument at `argIndex` and `valueIndex` to `imm`. + inline void setArg(size_t argIndex, size_t valueIndex, const Imm& imm) noexcept { _setArg(argIndex, valueIndex, imm); } + + //! \} +}; + +//! Function pass extends \ref Pass with \ref FuncPass::runOnFunction(). +class ASMJIT_VIRTAPI FuncPass : public Pass { +public: + ASMJIT_NONCOPYABLE(FuncPass) + typedef Pass Base; + + //! \name Construction & Destruction + //! \{ + + ASMJIT_API FuncPass(const char* name) noexcept; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the associated `BaseCompiler`. + inline BaseCompiler* cc() const noexcept { return static_cast(_cb); } + + //! \} + + //! \name Pass Interface + //! \{ + + //! Calls `runOnFunction()` on each `FuncNode` node found. + ASMJIT_API Error run(Zone* zone, Logger* logger) override; + + //! Called once per `FuncNode`. + virtual Error runOnFunction(Zone* zone, Logger* logger, FuncNode* func) = 0; + + //! \} +}; + +#if !defined(ASMJIT_NO_DEPRECATED) +inline Error BaseCompiler::_setArg(size_t argIndex, size_t valueIndex, const BaseReg& reg) { + FuncNode* func = _func; + + if (ASMJIT_UNLIKELY(!func)) + return reportError(DebugUtils::errored(kErrorInvalidState)); + + func->setArg(argIndex, valueIndex, reg); + return kErrorOk; +} +#endif + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // !ASMJIT_NO_COMPILER +#endif // ASMJIT_CORE_COMPILER_H_INCLUDED diff --git a/src/asmjit/core/compilerdefs.h b/src/asmjit/core/compilerdefs.h new file mode 100644 index 0000000..1870e68 --- /dev/null +++ b/src/asmjit/core/compilerdefs.h @@ -0,0 +1,173 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_COMPILERDEFS_H_INCLUDED +#define ASMJIT_CORE_COMPILERDEFS_H_INCLUDED + +#include "../core/api-config.h" +#include "../core/operand.h" +#include "../core/type.h" +#include "../core/zonestring.h" + +ASMJIT_BEGIN_NAMESPACE + +class RAWorkReg; + +//! \addtogroup asmjit_compiler +//! \{ + +//! Virtual register data, managed by \ref BaseCompiler. +class VirtReg { +public: + ASMJIT_NONCOPYABLE(VirtReg) + + //! \name Members + //! \{ + + //! Virtual register signature. + OperandSignature _signature {}; + //! Virtual register id. + uint32_t _id = 0; + //! Virtual register size (can be smaller than `_signature._size`). + uint32_t _virtSize = 0; + //! Virtual register alignment (for spilling). + uint8_t _alignment = 0; + //! Type-id. + TypeId _typeId = TypeId::kVoid; + //! Virtual register weight for alloc/spill decisions. + uint8_t _weight = 1; + //! True if this is a fixed register, never reallocated. + uint8_t _isFixed : 1; + //! True if the virtual register is only used as a stack (never accessed as register). + uint8_t _isStack : 1; + //! True if this virtual register has assigned stack offset (can be only valid after register allocation pass). + uint8_t _hasStackSlot : 1; + uint8_t _reservedBits : 5; + + //! Stack offset assigned by the register allocator relative to stack pointer (can be negative as well). + int32_t _stackOffset = 0; + + //! Reserved for future use (padding). + uint32_t _reservedU32 = 0; + + //! Virtual register name (user provided or automatically generated). + ZoneString<16> _name {}; + + // The following members are used exclusively by RAPass. They are initialized when the VirtReg is created to + // null pointers and then changed during RAPass execution. RAPass sets them back to NULL before it returns. + + //! Reference to `RAWorkReg`, used during register allocation. + RAWorkReg* _workReg = nullptr; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline VirtReg(OperandSignature signature, uint32_t id, uint32_t virtSize, uint32_t alignment, TypeId typeId) noexcept + : _signature(signature), + _id(id), + _virtSize(virtSize), + _alignment(uint8_t(alignment)), + _typeId(typeId), + _isFixed(false), + _isStack(false), + _hasStackSlot(false), + _reservedBits(0), + _stackOffset(0), + _reservedU32(0) {} + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the virtual register id. + inline uint32_t id() const noexcept { return _id; } + + //! Returns the virtual register name. + inline const char* name() const noexcept { return _name.data(); } + //! Returns the size of the virtual register name. + inline uint32_t nameSize() const noexcept { return _name.size(); } + + //! Returns a register signature of this virtual register. + inline OperandSignature signature() const noexcept { return _signature; } + //! Returns a virtual register type (maps to the physical register type as well). + inline RegType type() const noexcept { return _signature.regType(); } + //! Returns a virtual register group (maps to the physical register group as well). + inline RegGroup group() const noexcept { return _signature.regGroup(); } + + //! Returns a real size of the register this virtual register maps to. + //! + //! For example if this is a 128-bit SIMD register used for a scalar single precision floating point value then + //! its virtSize would be 4, however, the `regSize` would still say 16 (128-bits), because it's the smallest size + //! of that register type. + inline uint32_t regSize() const noexcept { return _signature.size(); } + + //! Returns the virtual register size. + //! + //! The virtual register size describes how many bytes the virtual register needs to store its content. It can be + //! smaller than the physical register size, see `regSize()`. + inline uint32_t virtSize() const noexcept { return _virtSize; } + + //! Returns the virtual register alignment. + inline uint32_t alignment() const noexcept { return _alignment; } + + //! Returns the virtual register type id. + inline TypeId typeId() const noexcept { return _typeId; } + + //! Returns the virtual register weight - the register allocator can use it as explicit hint for alloc/spill + //! decisions. + inline uint32_t weight() const noexcept { return _weight; } + //! Sets the virtual register weight (0 to 255) - the register allocator can use it as explicit hint for + //! alloc/spill decisions and initial bin-packing. + inline void setWeight(uint32_t weight) noexcept { _weight = uint8_t(weight); } + + //! Returns whether the virtual register is always allocated to a fixed physical register (and never reallocated). + //! + //! \note This is only used for special purposes and it's mostly internal. + inline bool isFixed() const noexcept { return bool(_isFixed); } + + //! Tests whether the virtual register is in fact a stack that only uses the virtual register id. + //! + //! \note It's an error if a stack is accessed as a register. + inline bool isStack() const noexcept { return bool(_isStack); } + + //! Tests whether this virtual register (or stack) has assigned a stack offset. + //! + //! If this is a virtual register that was never allocated on stack, it would return false, otherwise if + //! it's a virtual register that was spilled or explicitly allocated stack, the return value would be true. + inline bool hasStackSlot() const noexcept { return bool(_hasStackSlot); } + + //! Assigns a stack offset of this virtual register to `stackOffset` and sets `_hasStackSlot` to true. + inline void assignStackSlot(int32_t stackOffset) noexcept { + _hasStackSlot = 1; + _stackOffset = stackOffset; + } + + //! Returns a stack offset associated with a virtual register or explicit stack allocation. + //! + //! \note Always verify that the stack offset has been assigned by calling \ref hasStackSlot(). The return + //! value will be zero when the stack offset was not assigned. + inline int32_t stackOffset() const noexcept { return _stackOffset; } + + //! Tests whether the virtual register has an associated `RAWorkReg` at the moment. + inline bool hasWorkReg() const noexcept { return _workReg != nullptr; } + //! Returns an associated RAWorkReg with this virtual register (only valid during register allocation). + inline RAWorkReg* workReg() const noexcept { return _workReg; } + //! Associates a RAWorkReg with this virtual register (used by register allocator). + inline void setWorkReg(RAWorkReg* workReg) noexcept { _workReg = workReg; } + //! Reset the RAWorkReg association (used by register allocator). + inline void resetWorkReg() noexcept { _workReg = nullptr; } + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_COMPILERDEFS_H_INCLUDED + diff --git a/src/asmjit/core/constpool.cpp b/src/asmjit/core/constpool.cpp new file mode 100644 index 0000000..ad5fe4f --- /dev/null +++ b/src/asmjit/core/constpool.cpp @@ -0,0 +1,363 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/constpool.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// ConstPool - Construction & Destruction +// ====================================== + +ConstPool::ConstPool(Zone* zone) noexcept { reset(zone); } +ConstPool::~ConstPool() noexcept {} + +// ConstPool - Reset +// ================= + +void ConstPool::reset(Zone* zone) noexcept { + _zone = zone; + + size_t dataSize = 1; + for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_tree); i++) { + _tree[i].reset(); + _tree[i].setDataSize(dataSize); + _gaps[i] = nullptr; + dataSize <<= 1; + } + + _gapPool = nullptr; + _size = 0; + _alignment = 0; + _minItemSize = 0; +} + +// ConstPool - Operations +// ====================== + +static inline ConstPool::Gap* ConstPool_allocGap(ConstPool* self) noexcept { + ConstPool::Gap* gap = self->_gapPool; + if (!gap) + return self->_zone->allocT(); + + self->_gapPool = gap->_next; + return gap; +} + +static inline void ConstPool_freeGap(ConstPool* self, ConstPool::Gap* gap) noexcept { + gap->_next = self->_gapPool; + self->_gapPool = gap; +} + +static void ConstPool_addGap(ConstPool* self, size_t offset, size_t size) noexcept { + ASMJIT_ASSERT(size > 0); + + while (size > 0) { + size_t gapIndex; + size_t gapSize; + + if (size >= 32 && Support::isAligned(offset, 32)) { + gapIndex = ConstPool::kIndex32; + gapSize = 32; + } + else if (size >= 16 && Support::isAligned(offset, 16)) { + gapIndex = ConstPool::kIndex16; + gapSize = 16; + } + else if (size >= 8 && Support::isAligned(offset, 8)) { + gapIndex = ConstPool::kIndex8; + gapSize = 8; + } + else if (size >= 4 && Support::isAligned(offset, 4)) { + gapIndex = ConstPool::kIndex4; + gapSize = 4; + } + else if (size >= 2 && Support::isAligned(offset, 2)) { + gapIndex = ConstPool::kIndex2; + gapSize = 2; + } + else { + gapIndex = ConstPool::kIndex1; + gapSize = 1; + } + + // We don't have to check for errors here, if this failed nothing really happened (just the gap won't be + // visible) and it will fail again at place where the same check would generate `kErrorOutOfMemory` error. + ConstPool::Gap* gap = ConstPool_allocGap(self); + if (!gap) + return; + + gap->_next = self->_gaps[gapIndex]; + self->_gaps[gapIndex] = gap; + + gap->_offset = offset; + gap->_size = gapSize; + + offset += gapSize; + size -= gapSize; + } +} + +Error ConstPool::add(const void* data, size_t size, size_t& dstOffset) noexcept { + size_t treeIndex; + + if (size == 64) + treeIndex = kIndex64; + else if (size == 32) + treeIndex = kIndex32; + else if (size == 16) + treeIndex = kIndex16; + else if (size == 8) + treeIndex = kIndex8; + else if (size == 4) + treeIndex = kIndex4; + else if (size == 2) + treeIndex = kIndex2; + else if (size == 1) + treeIndex = kIndex1; + else + return DebugUtils::errored(kErrorInvalidArgument); + + ConstPool::Node* node = _tree[treeIndex].get(data); + if (node) { + dstOffset = node->_offset; + return kErrorOk; + } + + // Before incrementing the current offset try if there is a gap that can be used for the requested data. + size_t offset = ~size_t(0); + size_t gapIndex = treeIndex; + + while (gapIndex != kIndexCount - 1) { + ConstPool::Gap* gap = _gaps[treeIndex]; + + // Check if there is a gap. + if (gap) { + size_t gapOffset = gap->_offset; + size_t gapSize = gap->_size; + + // Destroy the gap for now. + _gaps[treeIndex] = gap->_next; + ConstPool_freeGap(this, gap); + + offset = gapOffset; + ASMJIT_ASSERT(Support::isAligned(offset, size)); + + gapSize -= size; + if (gapSize > 0) + ConstPool_addGap(this, gapOffset, gapSize); + } + + gapIndex++; + } + + if (offset == ~size_t(0)) { + // Get how many bytes have to be skipped so the address is aligned accordingly to the 'size'. + size_t diff = Support::alignUpDiff(_size, size); + + if (diff != 0) { + ConstPool_addGap(this, _size, diff); + _size += diff; + } + + offset = _size; + _size += size; + } + + // Add the initial node to the right index. + node = ConstPool::Tree::_newNode(_zone, data, size, offset, false); + if (ASMJIT_UNLIKELY(!node)) + return DebugUtils::errored(kErrorOutOfMemory); + + _tree[treeIndex].insert(node); + _alignment = Support::max(_alignment, size); + + dstOffset = offset; + + // Now create a bunch of shared constants that are based on the data pattern. We stop at size 4, + // it probably doesn't make sense to split constants down to 1 byte. + size_t pCount = 1; + size_t smallerSize = size; + + while (smallerSize > 4) { + pCount <<= 1; + smallerSize >>= 1; + + ASMJIT_ASSERT(treeIndex != 0); + treeIndex--; + + const uint8_t* pData = static_cast(data); + for (size_t i = 0; i < pCount; i++, pData += smallerSize) { + node = _tree[treeIndex].get(pData); + if (node) continue; + + node = ConstPool::Tree::_newNode(_zone, pData, smallerSize, offset + (i * smallerSize), true); + _tree[treeIndex].insert(node); + } + } + + if (_minItemSize == 0) + _minItemSize = size; + else + _minItemSize = Support::min(_minItemSize, size); + + return kErrorOk; +} + +// ConstPool - Reset +// ================= + +struct ConstPoolFill { + inline ConstPoolFill(uint8_t* dst, size_t dataSize) noexcept : + _dst(dst), + _dataSize(dataSize) {} + + inline void operator()(const ConstPool::Node* node) noexcept { + if (!node->_shared) + memcpy(_dst + node->_offset, node->data(), _dataSize); + } + + uint8_t* _dst; + size_t _dataSize; +}; + +void ConstPool::fill(void* dst) const noexcept { + // Clears possible gaps, asmjit should never emit garbage to the output. + memset(dst, 0, _size); + + ConstPoolFill filler(static_cast(dst), 1); + for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_tree); i++) { + _tree[i].forEach(filler); + filler._dataSize <<= 1; + } +} + +// ConstPool - Tests +// ================= + +#if defined(ASMJIT_TEST) +UNIT(const_pool) { + Zone zone(32384 - Zone::kBlockOverhead); + ConstPool pool(&zone); + + uint32_t i; + uint32_t kCount = BrokenAPI::hasArg("--quick") ? 1000 : 1000000; + + INFO("Adding %u constants to the pool", kCount); + { + size_t prevOffset; + size_t curOffset; + uint64_t c = 0x0101010101010101u; + + EXPECT(pool.add(&c, 8, prevOffset) == kErrorOk); + EXPECT(prevOffset == 0); + + for (i = 1; i < kCount; i++) { + c++; + EXPECT(pool.add(&c, 8, curOffset) == kErrorOk); + EXPECT(prevOffset + 8 == curOffset); + EXPECT(pool.size() == (i + 1) * 8); + prevOffset = curOffset; + } + + EXPECT(pool.alignment() == 8); + } + + INFO("Retrieving %u constants from the pool", kCount); + { + uint64_t c = 0x0101010101010101u; + + for (i = 0; i < kCount; i++) { + size_t offset; + EXPECT(pool.add(&c, 8, offset) == kErrorOk); + EXPECT(offset == i * 8); + c++; + } + } + + INFO("Checking if the constants were split into 4-byte patterns"); + { + uint32_t c = 0x01010101; + for (i = 0; i < kCount; i++) { + size_t offset; + EXPECT(pool.add(&c, 4, offset) == kErrorOk); + EXPECT(offset == i * 8); + c++; + } + } + + INFO("Adding 2 byte constant to misalign the current offset"); + { + uint16_t c = 0xFFFF; + size_t offset; + + EXPECT(pool.add(&c, 2, offset) == kErrorOk); + EXPECT(offset == kCount * 8); + EXPECT(pool.alignment() == 8); + } + + INFO("Adding 8 byte constant to check if pool gets aligned again"); + { + uint64_t c = 0xFFFFFFFFFFFFFFFFu; + size_t offset; + + EXPECT(pool.add(&c, 8, offset) == kErrorOk); + EXPECT(offset == kCount * 8 + 8); + } + + INFO("Adding 2 byte constant to verify the gap is filled"); + { + uint16_t c = 0xFFFE; + size_t offset; + + EXPECT(pool.add(&c, 2, offset) == kErrorOk); + EXPECT(offset == kCount * 8 + 2); + EXPECT(pool.alignment() == 8); + } + + INFO("Checking reset functionality"); + { + pool.reset(&zone); + zone.reset(); + + EXPECT(pool.size() == 0); + EXPECT(pool.alignment() == 0); + } + + INFO("Checking pool alignment when combined constants are added"); + { + uint8_t bytes[32] = { 0 }; + size_t offset; + + pool.add(bytes, 1, offset); + EXPECT(pool.size() == 1); + EXPECT(pool.alignment() == 1); + EXPECT(offset == 0); + + pool.add(bytes, 2, offset); + EXPECT(pool.size() == 4); + EXPECT(pool.alignment() == 2); + EXPECT(offset == 2); + + pool.add(bytes, 4, offset); + EXPECT(pool.size() == 8); + EXPECT(pool.alignment() == 4); + EXPECT(offset == 4); + + pool.add(bytes, 4, offset); + EXPECT(pool.size() == 8); + EXPECT(pool.alignment() == 4); + EXPECT(offset == 4); + + pool.add(bytes, 32, offset); + EXPECT(pool.size() == 64); + EXPECT(pool.alignment() == 32); + EXPECT(offset == 32); + } +} +#endif + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/constpool.h b/src/asmjit/core/constpool.h new file mode 100644 index 0000000..32b84b1 --- /dev/null +++ b/src/asmjit/core/constpool.h @@ -0,0 +1,250 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_CONSTPOOL_H_INCLUDED +#define ASMJIT_CORE_CONSTPOOL_H_INCLUDED + +#include "../core/support.h" +#include "../core/zone.h" +#include "../core/zonetree.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_utilities +//! \{ + +//! Constant pool scope. +enum class ConstPoolScope : uint32_t { + //! Local constant, always embedded right after the current function. + kLocal = 0, + //! Global constant, embedded at the end of the currently compiled code. + kGlobal = 1, + + //! Maximum value of `ConstPoolScope`. + kMaxValue = kGlobal +}; + +//! Constant pool. +class ConstPool { +public: + ASMJIT_NONCOPYABLE(ConstPool) + + //! \cond INTERNAL + + //! Index of a given size in const-pool table. + enum Index : uint32_t { + kIndex1 = 0, + kIndex2 = 1, + kIndex4 = 2, + kIndex8 = 3, + kIndex16 = 4, + kIndex32 = 5, + kIndex64 = 6, + kIndexCount = 7 + }; + + //! Zone-allocated const-pool gap created by two differently aligned constants. + struct Gap { + //! Pointer to the next gap + Gap* _next; + //! Offset of the gap. + size_t _offset; + //! Remaining bytes of the gap (basically a gap size). + size_t _size; + }; + + //! Zone-allocated const-pool node. + class Node : public ZoneTreeNodeT { + public: + ASMJIT_NONCOPYABLE(Node) + + //! If this constant is shared with another. + uint32_t _shared : 1; + //! Data offset from the beginning of the pool. + uint32_t _offset; + + inline Node(size_t offset, bool shared) noexcept + : ZoneTreeNodeT(), + _shared(shared), + _offset(uint32_t(offset)) {} + + inline void* data() const noexcept { + return static_cast(const_cast(this) + 1); + } + }; + + //! Data comparer used internally. + class Compare { + public: + size_t _dataSize; + + inline Compare(size_t dataSize) noexcept + : _dataSize(dataSize) {} + + inline int operator()(const Node& a, const Node& b) const noexcept { + return ::memcmp(a.data(), b.data(), _dataSize); + } + + inline int operator()(const Node& a, const void* data) const noexcept { + return ::memcmp(a.data(), data, _dataSize); + } + }; + + //! Zone-allocated const-pool tree. + struct Tree { + //! RB tree. + ZoneTree _tree; + //! Size of the tree (number of nodes). + size_t _size; + //! Size of the data. + size_t _dataSize; + + inline explicit Tree(size_t dataSize = 0) noexcept + : _tree(), + _size(0), + _dataSize(dataSize) {} + + inline void reset() noexcept { + _tree.reset(); + _size = 0; + } + + inline bool empty() const noexcept { return _size == 0; } + inline size_t size() const noexcept { return _size; } + + inline void setDataSize(size_t dataSize) noexcept { + ASMJIT_ASSERT(empty()); + _dataSize = dataSize; + } + + inline Node* get(const void* data) noexcept { + Compare cmp(_dataSize); + return _tree.get(data, cmp); + } + + inline void insert(Node* node) noexcept { + Compare cmp(_dataSize); + _tree.insert(node, cmp); + _size++; + } + + template + inline void forEach(Visitor& visitor) const noexcept { + Node* node = _tree.root(); + if (!node) return; + + Node* stack[Globals::kMaxTreeHeight]; + size_t top = 0; + + for (;;) { + Node* left = node->left(); + if (left != nullptr) { + ASMJIT_ASSERT(top != Globals::kMaxTreeHeight); + stack[top++] = node; + + node = left; + continue; + } + + for (;;) { + visitor(node); + node = node->right(); + + if (node != nullptr) + break; + + if (top == 0) + return; + + node = stack[--top]; + } + } + } + + static inline Node* _newNode(Zone* zone, const void* data, size_t size, size_t offset, bool shared) noexcept { + Node* node = zone->allocT(sizeof(Node) + size); + if (ASMJIT_UNLIKELY(!node)) return nullptr; + + node = new(node) Node(offset, shared); + memcpy(node->data(), data, size); + return node; + } + }; + + //! \endcond + + //! \name Members + //! \{ + + //! Zone allocator. + Zone* _zone; + //! Tree per size. + Tree _tree[kIndexCount]; + //! Gaps per size. + Gap* _gaps[kIndexCount]; + //! Gaps pool + Gap* _gapPool; + + //! Size of the pool (in bytes). + size_t _size; + //! Required pool alignment. + size_t _alignment; + //! Minimum item size in the pool. + size_t _minItemSize; + + //! \} + + //! \name Construction & Destruction + //! \{ + + ASMJIT_API ConstPool(Zone* zone) noexcept; + ASMJIT_API ~ConstPool() noexcept; + + ASMJIT_API void reset(Zone* zone) noexcept; + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the constant-pool is empty. + inline bool empty() const noexcept { return _size == 0; } + //! Returns the size of the constant-pool in bytes. + inline size_t size() const noexcept { return _size; } + //! Returns minimum alignment. + inline size_t alignment() const noexcept { return _alignment; } + //! Returns the minimum size of all items added to the constant pool. + inline size_t minItemSize() const noexcept { return _minItemSize; } + + //! \} + + //! \name Utilities + //! \{ + + //! Adds a constant to the constant pool. + //! + //! The constant must have known size, which is 1, 2, 4, 8, 16 or 32 bytes. The constant is added to the pool only + //! if it doesn't not exist, otherwise cached value is returned. + //! + //! AsmJit is able to subdivide added constants, so for example if you add 8-byte constant 0x1122334455667788 it + //! will create the following slots: + //! + //! 8-byte: 0x1122334455667788 + //! 4-byte: 0x11223344, 0x55667788 + //! + //! The reason is that when combining MMX/SSE/AVX code some patterns are used frequently. However, AsmJit is not + //! able to reallocate a constant that has been already added. For example if you try to add 4-byte constant and + //! then 8-byte constant having the same 4-byte pattern as the previous one, two independent slots will be used. + ASMJIT_API Error add(const void* data, size_t size, size_t& dstOffset) noexcept; + + //! Fills the destination with the content of this constant pool. + ASMJIT_API void fill(void* dst) const noexcept; +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_CONSTPOOL_H_INCLUDED diff --git a/src/asmjit/core/cpuinfo.cpp b/src/asmjit/core/cpuinfo.cpp new file mode 100644 index 0000000..8dbe257 --- /dev/null +++ b/src/asmjit/core/cpuinfo.cpp @@ -0,0 +1,1162 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/cpuinfo.h" +#include "../core/support.h" + +#if !defined(_WIN32) + #include + #include + #include +#endif + +// Required by `getauxval()` on Linux. +#if defined(__linux__) + //#include +#endif + +//! Required to detect CPU and features on Apple platforms. +#if defined(__APPLE__) + #include + #include + #include +#endif + +// Required by `__cpuidex()` and `_xgetbv()`. +#if defined(_MSC_VER) + #include +#endif + +ASMJIT_BEGIN_NAMESPACE + +// CpuInfo - Detect - HW-Thread Count +// ================================== + +#if defined(_WIN32) +static inline uint32_t detectHWThreadCount() noexcept { + SYSTEM_INFO info; + ::GetSystemInfo(&info); + return info.dwNumberOfProcessors; +} +#elif defined(_SC_NPROCESSORS_ONLN) +static inline uint32_t detectHWThreadCount() noexcept { + long res = ::sysconf(_SC_NPROCESSORS_ONLN); + return res <= 0 ? uint32_t(1) : uint32_t(res); +} +#else +static inline uint32_t detectHWThreadCount() noexcept { + return 1; +} +#endif + +// CpuInfo - Detect - X86 +// ====================== + +#if ASMJIT_ARCH_X86 + +struct cpuid_t { uint32_t eax, ebx, ecx, edx; }; +struct xgetbv_t { uint32_t eax, edx; }; + +// Executes `cpuid` instruction. +static inline void cpuidQuery(cpuid_t* out, uint32_t inEax, uint32_t inEcx = 0) noexcept { +#if defined(_MSC_VER) + __cpuidex(reinterpret_cast(out), inEax, inEcx); +#elif defined(__GNUC__) && ASMJIT_ARCH_X86 == 32 + __asm__ __volatile__( + "mov %%ebx, %%edi\n" + "cpuid\n" + "xchg %%edi, %%ebx\n" : "=a"(out->eax), "=D"(out->ebx), "=c"(out->ecx), "=d"(out->edx) : "a"(inEax), "c"(inEcx)); +#elif defined(__GNUC__) && ASMJIT_ARCH_X86 == 64 + __asm__ __volatile__( + "mov %%rbx, %%rdi\n" + "cpuid\n" + "xchg %%rdi, %%rbx\n" : "=a"(out->eax), "=D"(out->ebx), "=c"(out->ecx), "=d"(out->edx) : "a"(inEax), "c"(inEcx)); +#else + #error "[asmjit] x86::cpuidQuery() - Unsupported compiler." +#endif +} + +// Executes 'xgetbv' instruction. +static inline void xgetbvQuery(xgetbv_t* out, uint32_t inEcx) noexcept { +#if defined(_MSC_VER) + uint64_t value = _xgetbv(inEcx); + out->eax = uint32_t(value & 0xFFFFFFFFu); + out->edx = uint32_t(value >> 32); +#elif defined(__GNUC__) + uint32_t outEax; + uint32_t outEdx; + + // Replaced, because the world is not perfect: + // __asm__ __volatile__("xgetbv" : "=a"(outEax), "=d"(outEdx) : "c"(inEcx)); + __asm__ __volatile__(".byte 0x0F, 0x01, 0xD0" : "=a"(outEax), "=d"(outEdx) : "c"(inEcx)); + + out->eax = outEax; + out->edx = outEdx; +#else + out->eax = 0; + out->edx = 0; +#endif +} + +// Map a 12-byte vendor string returned by `cpuid` into a `CpuInfo::Vendor` ID. +static inline void simplifyCpuVendor(CpuInfo& cpu, uint32_t d0, uint32_t d1, uint32_t d2) noexcept { + struct Vendor { + char normalized[8]; + union { char text[12]; uint32_t d[3]; }; + }; + + static const Vendor table[] = { + { { 'A', 'M', 'D' }, {{ 'A', 'u', 't', 'h', 'e', 'n', 't', 'i', 'c', 'A', 'M', 'D' }} }, + { { 'I', 'N', 'T', 'E', 'L' }, {{ 'G', 'e', 'n', 'u', 'i', 'n', 'e', 'I', 'n', 't', 'e', 'l' }} }, + { { 'V', 'I', 'A' }, {{ 'C', 'e', 'n', 't', 'a', 'u', 'r', 'H', 'a', 'u', 'l', 's' }} }, + { { 'V', 'I', 'A' }, {{ 'V', 'I', 'A', 0 , 'V', 'I', 'A', 0 , 'V', 'I', 'A', 0 }} }, + { { 'U', 'N', 'K', 'N', 'O', 'W', 'N' }, {{ 0 }} } + }; + + uint32_t i; + for (i = 0; i < ASMJIT_ARRAY_SIZE(table) - 1; i++) + if (table[i].d[0] == d0 && table[i].d[1] == d1 && table[i].d[2] == d2) + break; + memcpy(cpu._vendor.str, table[i].normalized, 8); +} + +static ASMJIT_FAVOR_SIZE void simplifyCpuBrand(char* s) noexcept { + char* d = s; + + char c = s[0]; + char prev = 0; + + // Used to always clear the current character to ensure that the result + // doesn't contain garbage after a new null terminator is placed at the end. + s[0] = '\0'; + + for (;;) { + if (!c) + break; + + if (!(c == ' ' && (prev == '@' || s[1] == ' ' || s[1] == '@'))) { + *d++ = c; + prev = c; + } + + c = *++s; + s[0] = '\0'; + } + + d[0] = '\0'; +} + +static ASMJIT_FAVOR_SIZE void detectX86Cpu(CpuInfo& cpu) noexcept { + using Support::bitTest; + + cpuid_t regs; + xgetbv_t xcr0 { 0, 0 }; + CpuFeatures::X86& features = cpu.features().x86(); + + cpu._wasDetected = true; + cpu._maxLogicalProcessors = 1; + + // We are gonna execute CPUID, which was introduced by I486, so it's the requirement. + features.add(CpuFeatures::X86::kI486); + + // CPUID EAX=0 + // ----------- + + // Get vendor string/id. + cpuidQuery(®s, 0x0); + + uint32_t maxId = regs.eax; + uint32_t maxSubLeafId_0x7 = 0; + + simplifyCpuVendor(cpu, regs.ebx, regs.edx, regs.ecx); + + // CPUID EAX=1 + // ----------- + + if (maxId >= 0x1) { + // Get feature flags in ECX/EDX and family/model in EAX. + cpuidQuery(®s, 0x1); + + // Fill family and model fields. + uint32_t modelId = (regs.eax >> 4) & 0x0F; + uint32_t familyId = (regs.eax >> 8) & 0x0F; + + // Use extended family and model fields. + if (familyId == 0x06u || familyId == 0x0Fu) + modelId += (((regs.eax >> 16) & 0x0Fu) << 4); + + if (familyId == 0x0Fu) + familyId += ((regs.eax >> 20) & 0xFFu); + + cpu._modelId = modelId; + cpu._familyId = familyId; + cpu._brandId = ((regs.ebx ) & 0xFF); + cpu._processorType = ((regs.eax >> 12) & 0x03); + cpu._maxLogicalProcessors = ((regs.ebx >> 16) & 0xFF); + cpu._stepping = ((regs.eax ) & 0x0F); + cpu._cacheLineSize = ((regs.ebx >> 8) & 0xFF) * 8; + + features.addIf(bitTest(regs.ecx, 0), CpuFeatures::X86::kSSE3); + features.addIf(bitTest(regs.ecx, 1), CpuFeatures::X86::kPCLMULQDQ); + features.addIf(bitTest(regs.ecx, 3), CpuFeatures::X86::kMONITOR); + features.addIf(bitTest(regs.ecx, 5), CpuFeatures::X86::kVMX); + features.addIf(bitTest(regs.ecx, 6), CpuFeatures::X86::kSMX); + features.addIf(bitTest(regs.ecx, 9), CpuFeatures::X86::kSSSE3); + features.addIf(bitTest(regs.ecx, 13), CpuFeatures::X86::kCMPXCHG16B); + features.addIf(bitTest(regs.ecx, 19), CpuFeatures::X86::kSSE4_1); + features.addIf(bitTest(regs.ecx, 20), CpuFeatures::X86::kSSE4_2); + features.addIf(bitTest(regs.ecx, 22), CpuFeatures::X86::kMOVBE); + features.addIf(bitTest(regs.ecx, 23), CpuFeatures::X86::kPOPCNT); + features.addIf(bitTest(regs.ecx, 25), CpuFeatures::X86::kAESNI); + features.addIf(bitTest(regs.ecx, 26), CpuFeatures::X86::kXSAVE); + features.addIf(bitTest(regs.ecx, 27), CpuFeatures::X86::kOSXSAVE); + features.addIf(bitTest(regs.ecx, 30), CpuFeatures::X86::kRDRAND); + features.addIf(bitTest(regs.edx, 0), CpuFeatures::X86::kFPU); + features.addIf(bitTest(regs.edx, 4), CpuFeatures::X86::kRDTSC); + features.addIf(bitTest(regs.edx, 5), CpuFeatures::X86::kMSR); + features.addIf(bitTest(regs.edx, 8), CpuFeatures::X86::kCMPXCHG8B); + features.addIf(bitTest(regs.edx, 15), CpuFeatures::X86::kCMOV); + features.addIf(bitTest(regs.edx, 19), CpuFeatures::X86::kCLFLUSH); + features.addIf(bitTest(regs.edx, 23), CpuFeatures::X86::kMMX); + features.addIf(bitTest(regs.edx, 24), CpuFeatures::X86::kFXSR); + features.addIf(bitTest(regs.edx, 25), CpuFeatures::X86::kSSE); + features.addIf(bitTest(regs.edx, 25), CpuFeatures::X86::kSSE, CpuFeatures::X86::kSSE2); + features.addIf(bitTest(regs.edx, 28), CpuFeatures::X86::kMT); + + // Get the content of XCR0 if supported by the CPU and enabled by the OS. + if (features.hasXSAVE() && features.hasOSXSAVE()) { + xgetbvQuery(&xcr0, 0); + } + + // Detect AVX+. + if (bitTest(regs.ecx, 28)) { + // - XCR0[2:1] == 11b + // XMM & YMM states need to be enabled by OS. + if ((xcr0.eax & 0x00000006u) == 0x00000006u) { + features.add(CpuFeatures::X86::kAVX); + features.addIf(bitTest(regs.ecx, 12), CpuFeatures::X86::kFMA); + features.addIf(bitTest(regs.ecx, 29), CpuFeatures::X86::kF16C); + } + } + } + + constexpr uint32_t kXCR0_AMX_Bits = 0x3u << 17; + bool amxEnabledByOS = (xcr0.eax & kXCR0_AMX_Bits) == kXCR0_AMX_Bits; + +#if defined(__APPLE__) + // Apple platform provides on-demand AVX512 support. When an AVX512 instruction is used the first time it results + // in #UD, which would cause the thread being promoted to use AVX512 support by the OS in addition to enabling the + // necessary bits in XCR0 register. + bool avx512EnabledByOS = true; +#else + // - XCR0[2:1] == 11b - XMM/YMM states need to be enabled by OS. + // - XCR0[7:5] == 111b - Upper 256-bit of ZMM0-XMM15 and ZMM16-ZMM31 need to be enabled by OS. + constexpr uint32_t kXCR0_AVX512_Bits = (0x3u << 1) | (0x7u << 5); + bool avx512EnabledByOS = (xcr0.eax & kXCR0_AVX512_Bits) == kXCR0_AVX512_Bits; +#endif + + // CPUID EAX=7 ECX=0 + // ----------------- + + // Detect new features if the processor supports CPUID-07. + bool maybeMPX = false; + + if (maxId >= 0x7) { + cpuidQuery(®s, 0x7); + + maybeMPX = bitTest(regs.ebx, 14); + maxSubLeafId_0x7 = regs.eax; + + features.addIf(bitTest(regs.ebx, 0), CpuFeatures::X86::kFSGSBASE); + features.addIf(bitTest(regs.ebx, 3), CpuFeatures::X86::kBMI); + features.addIf(bitTest(regs.ebx, 4), CpuFeatures::X86::kHLE); + features.addIf(bitTest(regs.ebx, 7), CpuFeatures::X86::kSMEP); + features.addIf(bitTest(regs.ebx, 8), CpuFeatures::X86::kBMI2); + features.addIf(bitTest(regs.ebx, 9), CpuFeatures::X86::kERMS); + features.addIf(bitTest(regs.ebx, 11), CpuFeatures::X86::kRTM); + features.addIf(bitTest(regs.ebx, 18), CpuFeatures::X86::kRDSEED); + features.addIf(bitTest(regs.ebx, 19), CpuFeatures::X86::kADX); + features.addIf(bitTest(regs.ebx, 20), CpuFeatures::X86::kSMAP); + features.addIf(bitTest(regs.ebx, 23), CpuFeatures::X86::kCLFLUSHOPT); + features.addIf(bitTest(regs.ebx, 24), CpuFeatures::X86::kCLWB); + features.addIf(bitTest(regs.ebx, 29), CpuFeatures::X86::kSHA); + features.addIf(bitTest(regs.ecx, 0), CpuFeatures::X86::kPREFETCHWT1); + features.addIf(bitTest(regs.ecx, 4), CpuFeatures::X86::kOSPKE); + features.addIf(bitTest(regs.ecx, 5), CpuFeatures::X86::kWAITPKG); + features.addIf(bitTest(regs.ecx, 7), CpuFeatures::X86::kCET_SS); + features.addIf(bitTest(regs.ecx, 8), CpuFeatures::X86::kGFNI); + features.addIf(bitTest(regs.ecx, 9), CpuFeatures::X86::kVAES); + features.addIf(bitTest(regs.ecx, 10), CpuFeatures::X86::kVPCLMULQDQ); + features.addIf(bitTest(regs.ecx, 22), CpuFeatures::X86::kRDPID); + features.addIf(bitTest(regs.ecx, 25), CpuFeatures::X86::kCLDEMOTE); + features.addIf(bitTest(regs.ecx, 27), CpuFeatures::X86::kMOVDIRI); + features.addIf(bitTest(regs.ecx, 28), CpuFeatures::X86::kMOVDIR64B); + features.addIf(bitTest(regs.ecx, 29), CpuFeatures::X86::kENQCMD); + features.addIf(bitTest(regs.edx, 5), CpuFeatures::X86::kUINTR); + features.addIf(bitTest(regs.edx, 14), CpuFeatures::X86::kSERIALIZE); + features.addIf(bitTest(regs.edx, 16), CpuFeatures::X86::kTSXLDTRK); + features.addIf(bitTest(regs.edx, 18), CpuFeatures::X86::kPCONFIG); + features.addIf(bitTest(regs.edx, 20), CpuFeatures::X86::kCET_IBT); + + // Detect 'TSX' - Requires at least one of `HLE` and `RTM` features. + if (features.hasHLE() || features.hasRTM()) + features.add(CpuFeatures::X86::kTSX); + + // Detect 'AVX2' - Requires AVX as well. + if (bitTest(regs.ebx, 5) && features.hasAVX()) + features.add(CpuFeatures::X86::kAVX2); + + // Detect 'AVX512'. + if (avx512EnabledByOS && bitTest(regs.ebx, 16)) { + features.add(CpuFeatures::X86::kAVX512_F); + + features.addIf(bitTest(regs.ebx, 17), CpuFeatures::X86::kAVX512_DQ); + features.addIf(bitTest(regs.ebx, 21), CpuFeatures::X86::kAVX512_IFMA); + features.addIf(bitTest(regs.ebx, 26), CpuFeatures::X86::kAVX512_PFI); + features.addIf(bitTest(regs.ebx, 27), CpuFeatures::X86::kAVX512_ERI); + features.addIf(bitTest(regs.ebx, 28), CpuFeatures::X86::kAVX512_CDI); + features.addIf(bitTest(regs.ebx, 30), CpuFeatures::X86::kAVX512_BW); + features.addIf(bitTest(regs.ebx, 31), CpuFeatures::X86::kAVX512_VL); + features.addIf(bitTest(regs.ecx, 1), CpuFeatures::X86::kAVX512_VBMI); + features.addIf(bitTest(regs.ecx, 6), CpuFeatures::X86::kAVX512_VBMI2); + features.addIf(bitTest(regs.ecx, 11), CpuFeatures::X86::kAVX512_VNNI); + features.addIf(bitTest(regs.ecx, 12), CpuFeatures::X86::kAVX512_BITALG); + features.addIf(bitTest(regs.ecx, 14), CpuFeatures::X86::kAVX512_VPOPCNTDQ); + features.addIf(bitTest(regs.edx, 2), CpuFeatures::X86::kAVX512_4VNNIW); + features.addIf(bitTest(regs.edx, 3), CpuFeatures::X86::kAVX512_4FMAPS); + features.addIf(bitTest(regs.edx, 8), CpuFeatures::X86::kAVX512_VP2INTERSECT); + features.addIf(bitTest(regs.edx, 23), CpuFeatures::X86::kAVX512_FP16); + } + + // Detect 'AMX'. + if (amxEnabledByOS) { + features.addIf(bitTest(regs.edx, 22), CpuFeatures::X86::kAMX_BF16); + features.addIf(bitTest(regs.edx, 24), CpuFeatures::X86::kAMX_TILE); + features.addIf(bitTest(regs.edx, 25), CpuFeatures::X86::kAMX_INT8); + } + } + + // CPUID EAX=7 ECX=1 + // ----------------- + + if (features.hasAVX512_F() && maxSubLeafId_0x7 >= 1) { + cpuidQuery(®s, 0x7, 1); + + features.addIf(bitTest(regs.eax, 3), CpuFeatures::X86::kAVX_VNNI); + features.addIf(bitTest(regs.eax, 5), CpuFeatures::X86::kAVX512_BF16); + features.addIf(bitTest(regs.eax, 22), CpuFeatures::X86::kHRESET); + } + + // CPUID EAX=13 ECX=0 + // ------------------ + + if (maxId >= 0xD) { + cpuidQuery(®s, 0xD, 0); + + // Both CPUID result and XCR0 has to be enabled to have support for MPX. + if (((regs.eax & xcr0.eax) & 0x00000018u) == 0x00000018u && maybeMPX) + features.add(CpuFeatures::X86::kMPX); + + cpuidQuery(®s, 0xD, 1); + + features.addIf(bitTest(regs.eax, 0), CpuFeatures::X86::kXSAVEOPT); + features.addIf(bitTest(regs.eax, 1), CpuFeatures::X86::kXSAVEC); + features.addIf(bitTest(regs.eax, 3), CpuFeatures::X86::kXSAVES); + } + + // CPUID EAX=14 ECX=0 + // ------------------ + + if (maxId >= 0xE) { + cpuidQuery(®s, 0xE, 0); + + features.addIf(bitTest(regs.ebx, 4), CpuFeatures::X86::kPTWRITE); + } + + // CPUID EAX=0x80000000...maxId + // ---------------------------- + + maxId = 0x80000000u; + uint32_t i = maxId; + + // The highest EAX that we understand. + constexpr uint32_t kHighestProcessedEAX = 0x8000001Fu; + + // Several CPUID calls are required to get the whole branc string. It's easier + // to copy one DWORD at a time instead of copying the string a byte by byte. + uint32_t* brand = cpu._brand.u32; + do { + cpuidQuery(®s, i); + switch (i) { + case 0x80000000u: + maxId = Support::min(regs.eax, kHighestProcessedEAX); + break; + + case 0x80000001u: + features.addIf(bitTest(regs.ecx, 0), CpuFeatures::X86::kLAHFSAHF); + features.addIf(bitTest(regs.ecx, 2), CpuFeatures::X86::kSVM); + features.addIf(bitTest(regs.ecx, 5), CpuFeatures::X86::kLZCNT); + features.addIf(bitTest(regs.ecx, 6), CpuFeatures::X86::kSSE4A); + features.addIf(bitTest(regs.ecx, 7), CpuFeatures::X86::kMSSE); + features.addIf(bitTest(regs.ecx, 8), CpuFeatures::X86::kPREFETCHW); + features.addIf(bitTest(regs.ecx, 12), CpuFeatures::X86::kSKINIT); + features.addIf(bitTest(regs.ecx, 15), CpuFeatures::X86::kLWP); + features.addIf(bitTest(regs.ecx, 21), CpuFeatures::X86::kTBM); + features.addIf(bitTest(regs.ecx, 29), CpuFeatures::X86::kMONITORX); + features.addIf(bitTest(regs.edx, 20), CpuFeatures::X86::kNX); + features.addIf(bitTest(regs.edx, 21), CpuFeatures::X86::kFXSROPT); + features.addIf(bitTest(regs.edx, 22), CpuFeatures::X86::kMMX2); + features.addIf(bitTest(regs.edx, 27), CpuFeatures::X86::kRDTSCP); + features.addIf(bitTest(regs.edx, 29), CpuFeatures::X86::kPREFETCHW); + features.addIf(bitTest(regs.edx, 30), CpuFeatures::X86::k3DNOW2, CpuFeatures::X86::kMMX2); + features.addIf(bitTest(regs.edx, 31), CpuFeatures::X86::kPREFETCHW); + + if (features.hasAVX()) { + features.addIf(bitTest(regs.ecx, 11), CpuFeatures::X86::kXOP); + features.addIf(bitTest(regs.ecx, 16), CpuFeatures::X86::kFMA4); + } + + // This feature seems to be only supported by AMD. + if (cpu.isVendor("AMD")) { + features.addIf(bitTest(regs.ecx, 4), CpuFeatures::X86::kALTMOVCR8); + } + break; + + case 0x80000002u: + case 0x80000003u: + case 0x80000004u: + *brand++ = regs.eax; + *brand++ = regs.ebx; + *brand++ = regs.ecx; + *brand++ = regs.edx; + + // Go directly to the next one we are interested in. + if (i == 0x80000004u) + i = 0x80000008u - 1; + break; + + case 0x80000008u: + features.addIf(bitTest(regs.ebx, 0), CpuFeatures::X86::kCLZERO); + features.addIf(bitTest(regs.ebx, 0), CpuFeatures::X86::kRDPRU); + features.addIf(bitTest(regs.ebx, 8), CpuFeatures::X86::kMCOMMIT); + features.addIf(bitTest(regs.ebx, 9), CpuFeatures::X86::kWBNOINVD); + + // Go directly to the next one we are interested in. + i = 0x8000001Fu - 1; + break; + + case 0x8000001Fu: + features.addIf(bitTest(regs.eax, 4), CpuFeatures::X86::kSNP); + break; + } + } while (++i <= maxId); + + // Simplify CPU brand string a bit by removing some unnecessary spaces. + simplifyCpuBrand(cpu._brand.str); +} + +#endif // ASMJIT_ARCH_X86 + +// CpuInfo - Detect - ARM +// ====================== + +// The most relevant and accurate information can be found here: +// https://github.com/llvm-project/llvm/blob/master/lib/Target/AArch64/AArch64.td +// https://github.com/apple/llvm-project/blob/apple/main/llvm/lib/Target/AArch64/AArch64.td (Apple fork) +// +// Other resources: +// https://en.wikipedia.org/wiki/AArch64 +// https://en.wikipedia.org/wiki/Apple_silicon#List_of_Apple_processors +// https://developer.arm.com/architectures/learn-the-architecture/understanding-the-armv8-x-extensions/single-page + +#if ASMJIT_ARCH_ARM + +static inline void populateBaseARMFeatures(CpuInfo& cpu) noexcept { +#if ASMJIT_ARCH_ARM == 32 + // No baseline flags at the moment. + DebugUtils::unused(cpu); +#else + // AArch64 is based on ARMv8-A and later. + cpu.addFeature(CpuFeatures::ARM::kARMv6); + cpu.addFeature(CpuFeatures::ARM::kARMv7); + cpu.addFeature(CpuFeatures::ARM::kARMv8a); + + // AArch64 comes with these features by default. + cpu.addFeature(CpuFeatures::ARM::kVFPv2); + cpu.addFeature(CpuFeatures::ARM::kVFPv3); + cpu.addFeature(CpuFeatures::ARM::kVFPv4); + cpu.addFeature(CpuFeatures::ARM::kASIMD); + cpu.addFeature(CpuFeatures::ARM::kIDIVA); +#endif +} + +// Detects ARM version by macros defined at compile time. This means that AsmJit will report features forced at +// compile time that should always be provided by the target CPU. This also means that if we don't provide any +// means to detect CPU features the features reported by AsmJit will at least not report less features than the +// target it was compiled to. +ASMJIT_MAYBE_UNUSED +static ASMJIT_FAVOR_SIZE void detectARMFeaturesViaCompilerFlags(CpuInfo& cpu) noexcept { + DebugUtils::unused(cpu); + +#if ASMJIT_ARCH_ARM == 32 + + // ARM targets have no baseline at the moment. +# if defined(__ARM_ARCH_7A__) + cpu.addFeature(CpuFeatures::ARM::kARMv7); +# endif +# if defined(__ARM_ARCH_8A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8a); +# endif + +# if defined(__TARGET_ARCH_THUMB) + cpu.addFeature(CpuFeatures::ARM::kTHUMB); +# if __TARGET_ARCH_THUMB >= 4 + cpu.addFeature(CpuFeatures::ARM::kTHUMBv2); +# endif +# endif + +# if defined(__ARM_FEATURE_FMA) + cpu.addFeature(CpuFeatures::ARM::kVFPv3); + cpu.addFeature(CpuFeatures::ARM::kVFPv4); +# endif + +# if defined(__ARM_NEON) + cpu.addFeature(CpuFeatures::ARM::kASIMD); +# endif + +# if defined(__ARM_FEATURE_IDIV) && defined(__TARGET_ARCH_THUMB) + cpu.addFeature(CpuFeatures::ARM::kIDIVT); +#endif +# if defined(__ARM_FEATURE_IDIV) && !defined(__TARGET_ARCH_THUMB) + cpu.addFeature(CpuFeatures::ARM::kIDIVA); +# endif + +#endif + +#if defined(__ARM_ARCH_8_1A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_1a); +#endif +#if defined(__ARM_ARCH_8_2A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_2a); +#endif +#if defined(__ARM_ARCH_8_3A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_3a); +#endif +#if defined(__ARM_ARCH_8_4A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_4a); +#endif +#if defined(__ARM_ARCH_8_5A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_5a); +#endif +#if defined(__ARM_ARCH_8_6A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_6a); +#endif +#if defined(__ARM_ARCH_8_7A__) + cpu.addFeature(CpuFeatures::ARM::kARMv8_7a); +#endif + +#if defined(__ARM_FEATURE_AES) + cpu.addFeature(CpuFeatures::ARM::kAES); +#endif + +#if defined(__ARM_FEATURE_BF16_SCALAR_ARITHMETIC) && defined(__ARM_FEATURE_BF16_VECTOR_ARITHMETIC) + cpu.addFeature(CpuFeatures::ARM::kBF16); +#endif + +#if defined(__ARM_FEATURE_CRC32) + cpu.addFeature(CpuFeatures::ARM::kCRC32); +#endif + +#if defined(__ARM_FEATURE_CRYPTO) + cpu.addFeature(CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); +#endif + +#if defined(__ARM_FEATURE_DOTPROD) + cpu.addFeature(CpuFeatures::ARM::kDOTPROD); +#endif + +#if defined(__ARM_FEATURE_FP16FML) || defined(__ARM_FEATURE_FP16_FML) + cpu.addFeature(CpuFeatures::ARM::kFP16FML); +#endif + +#if defined(__ARM_FEATURE_FP16_SCALAR_ARITHMETIC) + cpu.addFeature(CpuFeatures::ARM::kFP16FULL); +#endif + +#if defined(__ARM_FEATURE_FRINT) + cpu.addFeature(CpuFeatures::ARM::kFRINT); +#endif + +#if defined(__ARM_FEATURE_JCVT) + cpu.addFeature(CpuFeatures::ARM::kFJCVTZS); +#endif + +#if defined(__ARM_FEATURE_MATMUL_INT8) + cpu.addFeature(CpuFeatures::ARM::kI8MM); +#endif + +#if defined(__ARM_FEATURE_ATOMICS) + cpu.addFeature(CpuFeatures::ARM::kLSE); +#endif + +#if defined(__ARM_FEATURE_MEMORY_TAGGING) + cpu.addFeature(CpuFeatures::ARM::kMTE); +#endif + +#if defined(__ARM_FEATURE_QRDMX) + cpu.addFeature(CpuFeatures::ARM::kRDM); +#endif + +#if defined(__ARM_FEATURE_RNG) + cpu.addFeature(CpuFeatures::ARM::kRNG); +#endif + +#if defined(__ARM_FEATURE_SHA2) + cpu.addFeature(CpuFeatures::ARM::kSHA2); +#endif + +#if defined(__ARM_FEATURE_SHA3) + cpu.addFeature(CpuFeatures::ARM::kSHA3); +#endif + +#if defined(__ARM_FEATURE_SHA512) + cpu.addFeature(CpuFeatures::ARM::kSHA512); +#endif + +#if defined(__ARM_FEATURE_SM3) + cpu.addFeature(CpuFeatures::ARM::kSM3); +#endif + +#if defined(__ARM_FEATURE_SM4) + cpu.addFeature(CpuFeatures::ARM::kSM4); +#endif + +#if defined(__ARM_FEATURE_SVE) || defined(__ARM_FEATURE_SVE_VECTOR_OPERATORS) + cpu.addFeature(CpuFeatures::ARM::kSVE); +#endif + +#if defined(__ARM_FEATURE_SVE_MATMUL_INT8) + cpu.addFeature(CpuFeatures::ARM::kSVE_I8MM); +#endif + +#if defined(__ARM_FEATURE_SVE_MATMUL_FP32) + cpu.addFeature(CpuFeatures::ARM::kSVE_F32MM); +#endif + +#if defined(__ARM_FEATURE_SVE_MATMUL_FP64) + cpu.addFeature(CpuFeatures::ARM::kSVE_F64MM); +#endif + +#if defined(__ARM_FEATURE_SVE2) + cpu.addFeature(CpuFeatures::ARM::kSVE2); +#endif + +#if defined(__ARM_FEATURE_SVE2_AES) + cpu.addFeature(CpuFeatures::ARM::kSVE2_AES); +#endif + +#if defined(__ARM_FEATURE_SVE2_BITPERM) + cpu.addFeature(CpuFeatures::ARM::kSVE2_BITPERM); +#endif + +#if defined(__ARM_FEATURE_SVE2_SHA3) + cpu.addFeature(CpuFeatures::ARM::kSVE2_SHA3); +#endif + +#if defined(__ARM_FEATURE_SVE2_SM4) + cpu.addFeature(CpuFeatures::ARM::kSVE2_SM4); +#endif + +#if defined(__ARM_FEATURE_TME) + cpu.addFeature(CpuFeatures::ARM::kTME); +#endif +} + +ASMJIT_MAYBE_UNUSED +static ASMJIT_FAVOR_SIZE void expandARMFeaturesByVersion(CpuInfo& cpu) noexcept { + CpuFeatures::ARM& features = cpu.features().arm(); + + if (features.hasARMv8_7a()) { + features.add(CpuFeatures::ARM::kARMv8_6a); + } + + if (features.hasARMv8_6a()) { + features.add(CpuFeatures::ARM::kARMv8_5a, + CpuFeatures::ARM::kBF16); + + if (features.hasSVE()) + features.add(CpuFeatures::ARM::kSVE_I8MM); + } + + if (features.hasARMv8_5a()) { + features.add(CpuFeatures::ARM::kARMv8_4a, + CpuFeatures::ARM::kALTNZCV, + CpuFeatures::ARM::kBTI, + CpuFeatures::ARM::kFRINT, + CpuFeatures::ARM::kSB, + CpuFeatures::ARM::kSSBS); + } + + if (features.hasARMv8_4a()) { + features.add(CpuFeatures::ARM::kARMv8_3a, + CpuFeatures::ARM::kDIT, + CpuFeatures::ARM::kDOTPROD, + CpuFeatures::ARM::kFLAGM, + CpuFeatures::ARM::kPMU, + CpuFeatures::ARM::kRCPC_IMMO); + } + + if (features.hasARMv8_3a()) { + features.add(CpuFeatures::ARM::kARMv8_2a, + CpuFeatures::ARM::kFCMA, + CpuFeatures::ARM::kFJCVTZS); + } + + if (features.hasARMv8_2a()) { + features.add(CpuFeatures::ARM::kARMv8_1a); + } + + if (features.hasARMv8_1a()) { + features.add(CpuFeatures::ARM::kARMv8a, + CpuFeatures::ARM::kCRC32, + CpuFeatures::ARM::kLSE, + CpuFeatures::ARM::kRDM); + } + + if (features.hasARMv8a()) { + features.add(CpuFeatures::ARM::kARMv7, + CpuFeatures::ARM::kVFPv2, + CpuFeatures::ARM::kVFPv3, + CpuFeatures::ARM::kVFPv4, + CpuFeatures::ARM::kVFP_D32, + CpuFeatures::ARM::kASIMD, + CpuFeatures::ARM::kIDIVA); + } +} + +// CpuInfo - Detect - ARM [Windows] +// ================================ + +#if defined(_WIN32) +struct WinPFPMapping { + uint8_t featureId; + uint8_t pfpFeatureId; +}; + +static ASMJIT_FAVOR_SIZE void detectPFPFeatures(CpuInfo& cpu, const WinPFPMapping* mapping, size_t size) noexcept { + for (size_t i = 0; i < size; i++) + if (::IsProcessorFeaturePresent(mapping[i].pfpFeatureId)) + cpu.addFeature(mapping[i].featureId); +} + +//! Detect ARM CPU features on Windows. +//! +//! The detection is based on `IsProcessorFeaturePresent()` API call. +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + cpu._wasDetected = true; + populateBaseARMFeatures(cpu); + + CpuFeatures::ARM& features = cpu.features().arm(); + + // Win32 for ARM requires ARMv7 with DSP extensions, VFPv3, and uses THUMBv2 by default. +#if ASMJIT_ARCH_ARM == 32 + features.add(CpuFeatures::ARM::kTHUMB); + features.add(CpuFeatures::ARM::kTHUMBv2); + features.add(CpuFeatures::ARM::kARMv6); + features.add(CpuFeatures::ARM::kARMv7); + features.add(CpuFeatures::ARM::kEDSP); + features.add(CpuFeatures::ARM::kVFPv2); + features.add(CpuFeatures::ARM::kVFPv3); +#endif + + // Windows for ARM requires ASIMD. + features.add(CpuFeatures::ARM::kASIMD); + + // Detect additional CPU features by calling `IsProcessorFeaturePresent()`. + static const WinPFPMapping mapping[] = { +#if ASMJIT_ARCH_ARM == 32 + { uint8_t(CpuFeatures::ARM::kVFP_D32) , 18 }, // PF_ARM_VFP_32_REGISTERS_AVAILABLE + { uint8_t(CpuFeatures::ARM::kIDIVT) , 24 }, // PF_ARM_DIVIDE_INSTRUCTION_AVAILABLE + { uint8_t(CpuFeatures::ARM::kVFPv4) , 27 }, // PF_ARM_FMAC_INSTRUCTIONS_AVAILABLE + { uint8_t(CpuFeatures::ARM::kARMv8a) , 29 }, // PF_ARM_V8_INSTRUCTIONS_AVAILABLE +#endif + { uint8_t(CpuFeatures::ARM::kAES) , 30 }, // PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE + { uint8_t(CpuFeatures::ARM::kCRC32) , 31 }, // PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE + { uint8_t(CpuFeatures::ARM::kLSE) , 34 } // PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE + + }; + detectPFPFeatures(cpu, mapping, ASMJIT_ARRAY_SIZE(mapping)); + + // Windows provides several instructions under a single flag: + if (features.hasAES()) { + features.add(CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + } + + expandARMFeaturesByVersion(cpu); +} + +// CpuInfo - Detect - ARM [Linux] +// ============================== + +#elif defined(__linux__) + +struct LinuxHWCapMapping { + uint8_t featureId; + uint8_t hwCapBit; +}; + +static ASMJIT_FAVOR_SIZE void detectHWCaps(CpuInfo& cpu, unsigned long type, const LinuxHWCapMapping* mapping, size_t size) noexcept { + unsigned long mask = getauxval(type); + for (size_t i = 0; i < size; i++) + cpu.features().addIf(Support::bitTest(mask, mapping[i].hwCapBit), mapping[i].featureId); +} + +#if ASMJIT_ARCH_ARM == 32 + +// `AT_HWCAP` provides ARMv7 (and less) related flags. +static const LinuxHWCapMapping hwCapMapping[] = { + { uint8_t(CpuFeatures::ARM::kVFPv2) , 6 }, // HWCAP_VFP + { uint8_t(CpuFeatures::ARM::kEDSP) , 7 }, // HWCAP_EDSP + { uint8_t(CpuFeatures::ARM::kASIMD) , 12 }, // HWCAP_NEON + { uint8_t(CpuFeatures::ARM::kVFPv3) , 13 }, // HWCAP_VFPv3 + { uint8_t(CpuFeatures::ARM::kVFPv4) , 16 }, // HWCAP_VFPv4 + { uint8_t(CpuFeatures::ARM::kIDIVA) , 17 }, // HWCAP_IDIVA + { uint8_t(CpuFeatures::ARM::kIDIVT) , 18 }, // HWCAP_IDIVT + { uint8_t(CpuFeatures::ARM::kVFP_D32) , 19 } // HWCAP_VFPD32 +}; + +// `AT_HWCAP2` provides ARMv8+ related flags. +static const LinuxHWCapMapping hwCap2Mapping[] = { + { uint8_t(CpuFeatures::ARM::kAES) , 0 }, // HWCAP2_AES + { uint8_t(CpuFeatures::ARM::kPMULL) , 1 }, // HWCAP2_PMULL + { uint8_t(CpuFeatures::ARM::kSHA1) , 2 }, // HWCAP2_SHA1 + { uint8_t(CpuFeatures::ARM::kSHA2) , 3 }, // HWCAP2_SHA2 + { uint8_t(CpuFeatures::ARM::kCRC32) , 4 } // HWCAP2_CRC32 +}; + +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + cpu._wasDetected = true; + + populateBaseARMFeatures(cpu); + + CpuFeatures::ARM& features = cpu.features().arm(); + + detectHWCaps(cpu, AT_HWCAP, hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); + detectHWCaps(cpu, AT_HWCAP2, hwCap2Mapping, ASMJIT_ARRAY_SIZE(hwCap2Mapping)); + + // VFPv3 implies VFPv2. + if (features.hasVFPv3()) + features.add(CpuFeatures::ARM::kVFPv2); + + // VFPv2 implies ARMv6. + if (features.hasVFPv2()) + features.add(CpuFeatures::ARM::kARMv6); + + // ARMv7 provides VFPv3|ASIMD. + if (features.hasVFPv3() || features.hasASIMD()) + features.add(CpuFeatures::ARM::kARMv7); + + // ARMv8 provives AES, CRC32, PMULL, SHA1, and SHA2. + if (features.hasAES() || features.hasCRC32() || features.hasPMULL() || features.hasSHA1() || features.hasSHA2()) + features.add(CpuFeatures::ARM::kARMv8a); +} + +#else + +// `AT_HWCAP` provides ARMv8+ related flags. +static const LinuxHWCapMapping hwCapMapping[] = { + /* + { uint8_t(CpuFeatures::ARM::k) , 0 }, // HWCAP_FP + */ + { uint8_t(CpuFeatures::ARM::kASIMD) , 1 }, // HWCAP_ASIMD + /* + { uint8_t(CpuFeatures::ARM::k) , 2 }, // HWCAP_EVTSTRM + */ + { uint8_t(CpuFeatures::ARM::kAES) , 3 }, // HWCAP_AES + { uint8_t(CpuFeatures::ARM::kPMULL) , 4 }, // HWCAP_PMULL + { uint8_t(CpuFeatures::ARM::kSHA1) , 5 }, // HWCAP_SHA1 + { uint8_t(CpuFeatures::ARM::kSHA2) , 6 }, // HWCAP_SHA2 + { uint8_t(CpuFeatures::ARM::kCRC32) , 7 }, // HWCAP_CRC32 + { uint8_t(CpuFeatures::ARM::kLSE) , 8 }, // HWCAP_ATOMICS + { uint8_t(CpuFeatures::ARM::kFP16CONV) , 9 }, // HWCAP_FPHP + { uint8_t(CpuFeatures::ARM::kFP16FULL) , 10 }, // HWCAP_ASIMDHP + { uint8_t(CpuFeatures::ARM::kCPUID) , 11 }, // HWCAP_CPUID + { uint8_t(CpuFeatures::ARM::kRDM) , 12 }, // HWCAP_ASIMDRDM + { uint8_t(CpuFeatures::ARM::kFJCVTZS) , 13 }, // HWCAP_JSCVT + { uint8_t(CpuFeatures::ARM::kFCMA) , 14 }, // HWCAP_FCMA + /* + { uint8_t(CpuFeatures::ARM::k) , 15 }, // HWCAP_LRCPC + { uint8_t(CpuFeatures::ARM::k) , 16 }, // HWCAP_DCPOP + */ + { uint8_t(CpuFeatures::ARM::kSHA3) , 17 }, // HWCAP_SHA3 + { uint8_t(CpuFeatures::ARM::kSM3) , 18 }, // HWCAP_SM3 + { uint8_t(CpuFeatures::ARM::kSM4) , 19 }, // HWCAP_SM4 + { uint8_t(CpuFeatures::ARM::kDOTPROD) , 20 }, // HWCAP_ASIMDDP + { uint8_t(CpuFeatures::ARM::kSHA512) , 21 }, // HWCAP_SHA512 + { uint8_t(CpuFeatures::ARM::kSVE) , 22 }, // HWCAP_SVE + { uint8_t(CpuFeatures::ARM::kFP16FML) , 23 }, // HWCAP_ASIMDFHM + { uint8_t(CpuFeatures::ARM::kDIT) , 24 }, // HWCAP_DIT + /* + { uint8_t(CpuFeatures::ARM::k) , 25 }, // HWCAP_USCAT + { uint8_t(CpuFeatures::ARM::k) , 26 }, // HWCAP_ILRCPC + */ + { uint8_t(CpuFeatures::ARM::kFLAGM) , 27 }, // HWCAP_FLAGM + { uint8_t(CpuFeatures::ARM::kSSBS) , 28 }, // HWCAP_SSBS + { uint8_t(CpuFeatures::ARM::kSB) , 29 } // HWCAP_SB + /* + { uint8_t(CpuFeatures::ARM::k) , 30 }, // HWCAP_PACA + { uint8_t(CpuFeatures::ARM::k) , 31 } // HWCAP_PACG + */ +}; + +// `AT_HWCAP2` provides ARMv8+ related flags. +static const LinuxHWCapMapping hwCapMapping2[] = { + /* + { uint8_t(CpuFeatures::ARM::k) , 0 }, // HWCAP2_DCPODP + */ + { uint8_t(CpuFeatures::ARM::kSVE2) , 1 }, // HWCAP2_SVE2 + { uint8_t(CpuFeatures::ARM::kSVE2_AES) , 2 }, // HWCAP2_SVEAES + { uint8_t(CpuFeatures::ARM::kSVE_PMULL) , 3 }, // HWCAP2_SVEPMULL + { uint8_t(CpuFeatures::ARM::kSVE2_BITPERM), 4 }, // HWCAP2_SVEBITPERM + { uint8_t(CpuFeatures::ARM::kSVE2_SHA3) , 5 }, // HWCAP2_SVESHA3 + { uint8_t(CpuFeatures::ARM::kSVE2_SM4) , 6 }, // HWCAP2_SVESM4 + { uint8_t(CpuFeatures::ARM::kALTNZCV) , 7 }, // HWCAP2_FLAGM2 + { uint8_t(CpuFeatures::ARM::kFRINT) , 8 }, // HWCAP2_FRINT + { uint8_t(CpuFeatures::ARM::kSVE_I8MM) , 9 }, // HWCAP2_SVEI8MM + { uint8_t(CpuFeatures::ARM::kSVE_F32MM) , 10 }, // HWCAP2_SVEF32MM + { uint8_t(CpuFeatures::ARM::kSVE_F64MM) , 11 }, // HWCAP2_SVEF64MM + { uint8_t(CpuFeatures::ARM::kSVE_BF16) , 12 }, // HWCAP2_SVEBF16 + { uint8_t(CpuFeatures::ARM::kI8MM) , 13 }, // HWCAP2_I8MM + { uint8_t(CpuFeatures::ARM::kBF16) , 14 }, // HWCAP2_BF16 + { uint8_t(CpuFeatures::ARM::kDGH) , 15 }, // HWCAP2_DGH + { uint8_t(CpuFeatures::ARM::kRNG) , 16 }, // HWCAP2_RNG + { uint8_t(CpuFeatures::ARM::kBTI) , 17 }, // HWCAP2_BTI + { uint8_t(CpuFeatures::ARM::kMTE) , 18 } // HWCAP2_MTE +}; + +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + cpu._wasDetected = true; + populateBaseARMFeatures(cpu); + + detectHWCaps(cpu, AT_HWCAP, hwCapMapping, ASMJIT_ARRAY_SIZE(hwCapMapping)); + detectHWCaps(cpu, AT_HWCAP2, hwCapMapping2, ASMJIT_ARRAY_SIZE(hwCapMapping2)); +} + +#endif + +// CpuInfo - Detect - ARM [Apple] +// ============================== + +#elif defined(__APPLE__) + +namespace AppleHWId { + enum CpuFamily : uint32_t { + // Generic ARM. + kCpuFamily_ARM_9 = 0xE73283AEu, + kCpuFamily_ARM_11 = 0x8FF620D8u, + kCpuFamily_ARM_12 = 0xBD1B0AE9u, + kCpuFamily_ARM_13 = 0x0CC90E64u, + kCpuFamily_ARM_14 = 0x96077EF1u, + kCpuFamily_ARM_15 = 0xA8511BCAu, + + // Apple design. + kCpuFamily_SWIFT = 0x1E2D6381u, + kCpuFamily_CYCLONE = 0x37A09642u, + kCpuFamily_TYPHOON = 0x2C91A47Eu, + kCpuFamily_TWISTER = 0x92FB37C8u, + kCpuFamily_HURRICANE = 0x67CEEE93u, + kCpuFamily_MONSOON_MISTRAL = 0xE81E7EF6u, + kCpuFamily_VORTEX_TEMPEST = 0x07D34B9Fu, + kCpuFamily_LIGHTNING_THUNDER = 0x462504D2u, + kCpuFamily_FIRESTORM_ICESTORM = 0x1B588BB3u + }; +}; + +static ASMJIT_FAVOR_SIZE uint32_t queryARMCpuFamilyId() noexcept { + uint32_t result = 0; + size_t size = sizeof(result); + + int res = sysctlbyname("hw.cpufamily", &result, &size, nullptr, 0); + if (res != 0) + return 0; + else + return result; +} + +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + cpu._wasDetected = true; + populateBaseARMFeatures(cpu); + + uint32_t cpuFamilyId = queryARMCpuFamilyId(); + CpuFeatures::ARM& features = cpu.features().arm(); + + switch (cpuFamilyId) { + case AppleHWId::kCpuFamily_ARM_9: + case AppleHWId::kCpuFamily_ARM_11: + case AppleHWId::kCpuFamily_ARM_12: + break; + + // ARM Cortex A8. + case AppleHWId::kCpuFamily_ARM_13: + break; + + // ARM Cortex A9. + case AppleHWId::kCpuFamily_ARM_14: + break; + + // ARM Cortex A7 - ARMv7k. + case AppleHWId::kCpuFamily_ARM_15: + features.add(CpuFeatures::ARM::kARMv7); + break; + + // Apple A6/A6X - ARMv7s. + case AppleHWId::kCpuFamily_SWIFT: + features.add(CpuFeatures::ARM::kARMv7); + break; + + // Apple A7 - ARMv8.0-A. + case AppleHWId::kCpuFamily_CYCLONE: + features.add(CpuFeatures::ARM::kARMv8a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + break; + + // Apple A8 - ARMv8.0-A. + case AppleHWId::kCpuFamily_TYPHOON: + features.add(CpuFeatures::ARM::kARMv8a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + break; + + // Apple A9 - ARMv8.0-A. + case AppleHWId::kCpuFamily_TWISTER: + features.add(CpuFeatures::ARM::kARMv8a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + break; + + // Apple A10 - ARMv8.1-A. + case AppleHWId::kCpuFamily_HURRICANE: + features.add(CpuFeatures::ARM::kARMv8_1a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kRDM, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + + break; + + // Apple A11 - ARMv8.2-A. + case AppleHWId::kCpuFamily_MONSOON_MISTRAL: + features.add(CpuFeatures::ARM::kARMv8_2a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kFP16FULL, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + break; + + // Apple A12 - ARMv8.3-A. + case AppleHWId::kCpuFamily_VORTEX_TEMPEST: + features.add(CpuFeatures::ARM::kARMv8_3a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kFP16FULL, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2); + break; + + // Apple A13 - ARMv8.4-A. + case AppleHWId::kCpuFamily_LIGHTNING_THUNDER: + features.add(CpuFeatures::ARM::kARMv8_4a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kFP16FML, + CpuFeatures::ARM::kFP16FULL, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2, + CpuFeatures::ARM::kSHA3, + CpuFeatures::ARM::kSHA512); + break; + + // Apple A14/M1 - ARMv8.5-A. + case AppleHWId::kCpuFamily_FIRESTORM_ICESTORM: + features.add(CpuFeatures::ARM::kARMv8_4a, + CpuFeatures::ARM::kAES, + CpuFeatures::ARM::kALTNZCV, + CpuFeatures::ARM::kFP16FML, + CpuFeatures::ARM::kFP16FULL, + CpuFeatures::ARM::kFRINT, + CpuFeatures::ARM::kSB, + CpuFeatures::ARM::kSHA1, + CpuFeatures::ARM::kSHA2, + CpuFeatures::ARM::kSHA3, + CpuFeatures::ARM::kSHA512, + CpuFeatures::ARM::kSSBS); + break; + + default: + cpu._wasDetected = false; + break; + } + + expandARMFeaturesByVersion(cpu); +} + +// CpuInfo - Detect - ARM [Unknown] +// ================================ + +#else + +#if ASMJIT_ARCH_ARM == 64 + #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with AArch64 CPU)") +#else + #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown OS with ARM CPU)") +#endif + +static ASMJIT_FAVOR_SIZE void detectARMCpu(CpuInfo& cpu) noexcept { + populateBaseARMFeatures(cpu); + detectARMFeaturesViaCompilerFlags(cpu); + expandARMFeaturesByVersion(cpu); +} +#endif + +#endif + +// CpuInfo - Detect - Host +// ======================= + +static uint32_t cpuInfoInitialized; +static CpuInfo cpuInfoGlobal(Globals::NoInit); + +const CpuInfo& CpuInfo::host() noexcept { + // This should never cause a problem as the resulting information should always be the same. In the worst case we + // would just overwrite it non-atomically. + if (!cpuInfoInitialized) { + CpuInfo cpuInfoLocal; + + cpuInfoLocal._arch = Arch::kHost; + cpuInfoLocal._subArch = SubArch::kHost; + +#if ASMJIT_ARCH_X86 + detectX86Cpu(cpuInfoLocal); +#elif ASMJIT_ARCH_ARM + detectARMCpu(cpuInfoLocal); +#else + #pragma message("[asmjit] Disabling runtime CPU detection - unsupported OS/CPU combination (Unknown CPU)") +#endif + + cpuInfoLocal._hwThreadCount = detectHWThreadCount(); + cpuInfoGlobal = cpuInfoLocal; + cpuInfoInitialized = 1; + } + + return cpuInfoGlobal; +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/cpuinfo.h b/src/asmjit/core/cpuinfo.h new file mode 100644 index 0000000..4af5c3a --- /dev/null +++ b/src/asmjit/core/cpuinfo.h @@ -0,0 +1,813 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_CPUINFO_H_INCLUDED +#define ASMJIT_CORE_CPUINFO_H_INCLUDED + +#include "../core/archtraits.h" +#include "../core/environment.h" +#include "../core/globals.h" +#include "../core/string.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! CPU features information. +//! +//! Each feature is represented by a single bit in an embedded bit array. +class CpuFeatures { +public: + //! A word that is used to represents feature bits. + typedef Support::BitWord BitWord; + //! Iterator that can iterate all CPU features set. + typedef Support::BitVectorIterator Iterator; + + //! \name Constants + //! \{ + + //! \cond INTERNAL + enum : uint32_t { + kMaxFeatures = 256, + kNumBitWords = kMaxFeatures / Support::kBitWordSizeInBits + }; + //! \endcond + + //! \} + + //! \name Data + //! \{ + + //! CPU features data. + struct Data { + //! \name Members + //! \{ + + //! Data bits. + Support::Array _bits; + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline bool operator==(const Data& other) noexcept { return eq(other); } + inline bool operator!=(const Data& other) noexcept { return !eq(other); } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns true if there are no features set. + inline bool empty() const noexcept { return _bits.aggregate(0) == 0; } + + //! Returns all features as array of bitwords (see \ref Support::BitWord). + inline BitWord* bits() noexcept { return _bits.data(); } + //! Returns all features as array of bitwords (const). + inline const BitWord* bits() const noexcept { return _bits.data(); } + + //! Returns the number of BitWords returned by \ref bits(). + inline size_t bitWordCount() const noexcept { return kNumBitWords; } + + //! Returns \ref Support::BitVectorIterator, that can be used to iterate over all features efficiently. + inline Iterator iterator() const noexcept { return Iterator(_bits.data(), kNumBitWords); } + + //! Tests whether the feature `featureId` is present. + template + ASMJIT_FORCE_INLINE bool has(const FeatureId& featureId) const noexcept { + ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); + + uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; + uint32_t bit = uint32_t(featureId) % Support::kBitWordSizeInBits; + + return bool((_bits[idx] >> bit) & 0x1); + } + + //! Tests whether all features as defined by `other` are present. + ASMJIT_FORCE_INLINE bool hasAll(const Data& other) const noexcept { + for (uint32_t i = 0; i < kNumBitWords; i++) + if ((_bits[i] & other._bits[i]) != other._bits[i]) + return false; + return true; + } + + //! \} + + //! \name Manipulation + //! \{ + + inline void reset() noexcept { _bits.fill(0); } + + //! Adds the given CPU `featureId` to the list of features. + template + ASMJIT_FORCE_INLINE void add(const FeatureId& featureId) noexcept { + ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); + + uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; + uint32_t bit = uint32_t(featureId) % Support::kBitWordSizeInBits; + + _bits[idx] |= BitWord(1) << bit; + } + + template + ASMJIT_FORCE_INLINE void add(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + add(featureId); + add(std::forward(otherFeatureIds)...); + } + + template + ASMJIT_FORCE_INLINE void addIf(bool condition, const FeatureId& featureId) noexcept { + ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); + + uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; + uint32_t bit = uint32_t(featureId) % Support::kBitWordSizeInBits; + + _bits[idx] |= BitWord(condition) << bit; + } + + template + ASMJIT_FORCE_INLINE void addIf(bool condition, const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + addIf(condition, featureId); + addIf(condition, std::forward(otherFeatureIds)...); + } + + //! Removes the given CPU `featureId` from the list of features. + template + ASMJIT_FORCE_INLINE void remove(const FeatureId& featureId) noexcept { + ASMJIT_ASSERT(uint32_t(featureId) < kMaxFeatures); + + uint32_t idx = uint32_t(featureId) / Support::kBitWordSizeInBits; + uint32_t bit = uint32_t(featureId) % Support::kBitWordSizeInBits; + + _bits[idx] &= ~(BitWord(1) << bit); + } + + template + ASMJIT_FORCE_INLINE void remove(const FeatureId& featureId, Args&&... otherFeatureIds) noexcept { + remove(featureId); + remove(std::forward(otherFeatureIds)...); + } + + //! Tests whether this CPU features data matches `other`. + ASMJIT_FORCE_INLINE bool eq(const Data& other) const noexcept { return _bits == other._bits; } + + //! \} + + }; + + //! X86 specific features data. + struct X86 : public Data { + //! X86 CPU feature identifiers. + enum Id : uint8_t { + // @EnumValuesBegin{"enum": "CpuFeatures::X86"}@ + kNone, //!< No feature (never set, used internally). + + kMT, //!< CPU has multi-threading capabilities. + kNX, //!< CPU has Not-Execute-Bit aka DEP (data-execution prevention). + k3DNOW, //!< CPU has 3DNOW (3DNOW base instructions) [AMD]. + k3DNOW2, //!< CPU has 3DNOW2 (enhanced 3DNOW) [AMD]. + kADX, //!< CPU has ADX (multi-precision add-carry instruction extensions). + kAESNI, //!< CPU has AESNI (AES encode/decode instructions). + kALTMOVCR8, //!< CPU has LOCK MOV R<->CR0 (supports `MOV R<->CR8` via `LOCK MOV R<->CR0` in 32-bit mode) [AMD]. + kAMX_BF16, //!< CPU has AMX_BF16 (advanced matrix extensions - BF16 instructions). + kAMX_INT8, //!< CPU has AMX_INT8 (advanced matrix extensions - INT8 instructions). + kAMX_TILE, //!< CPU has AMX_TILE (advanced matrix extensions). + kAVX, //!< CPU has AVX (advanced vector extensions). + kAVX2, //!< CPU has AVX2 (advanced vector extensions 2). + kAVX512_4FMAPS, //!< CPU has AVX512_FMAPS (FMA packed single). + kAVX512_4VNNIW, //!< CPU has AVX512_VNNIW (vector NN instructions word variable precision). + kAVX512_BF16, //!< CPU has AVX512_BF16 (BFLOAT16 support instruction). + kAVX512_BITALG, //!< CPU has AVX512_BITALG (VPOPCNT[B|W], VPSHUFBITQMB). + kAVX512_BW, //!< CPU has AVX512_BW (packed BYTE|WORD). + kAVX512_CDI, //!< CPU has AVX512_CDI (conflict detection). + kAVX512_DQ, //!< CPU has AVX512_DQ (packed DWORD|QWORD). + kAVX512_ERI, //!< CPU has AVX512_ERI (exponential and reciprocal). + kAVX512_F, //!< CPU has AVX512_F (AVX512 foundation). + kAVX512_FP16, //!< CPU has AVX512_FP16 (FP16 extensions). + kAVX512_IFMA, //!< CPU has AVX512_IFMA (integer fused-multiply-add using 52-bit precision). + kAVX512_PFI, //!< CPU has AVX512_PFI (prefetch instructions). + kAVX512_VBMI, //!< CPU has AVX512_VBMI (vector byte manipulation). + kAVX512_VBMI2, //!< CPU has AVX512_VBMI2 (vector byte manipulation 2). + kAVX512_VL, //!< CPU has AVX512_VL (vector length extensions). + kAVX512_VNNI, //!< CPU has AVX512_VNNI (vector neural network instructions). + kAVX512_VP2INTERSECT, //!< CPU has AVX512_VP2INTERSECT + kAVX512_VPOPCNTDQ, //!< CPU has AVX512_VPOPCNTDQ (VPOPCNT[D|Q] instructions). + kAVX_VNNI, //!< CPU has AVX_VNNI (VEX encoding of vpdpbusd/vpdpbusds/vpdpwssd/vpdpwssds). + kBMI, //!< CPU has BMI (bit manipulation instructions #1). + kBMI2, //!< CPU has BMI2 (bit manipulation instructions #2). + kCET_IBT, //!< CPU has CET-IBT (indirect branch tracking). + kCET_SS, //!< CPU has CET-SS. + kCLDEMOTE, //!< CPU has CLDEMOTE (cache line demote). + kCLFLUSH, //!< CPU has CLFUSH (Cache Line flush). + kCLFLUSHOPT, //!< CPU has CLFUSHOPT (Cache Line flush - optimized). + kCLWB, //!< CPU has CLWB. + kCLZERO, //!< CPU has CLZERO. + kCMOV, //!< CPU has CMOV (CMOV and FCMOV instructions). + kCMPXCHG16B, //!< CPU has CMPXCHG16B (compare-exchange 16 bytes) [X86_64]. + kCMPXCHG8B, //!< CPU has CMPXCHG8B (compare-exchange 8 bytes). + kENCLV, //!< CPU has ENCLV. + kENQCMD, //!< CPU has ENQCMD (enqueue stores). + kERMS, //!< CPU has ERMS (enhanced REP MOVSB/STOSB). + kF16C, //!< CPU has F16C. + kFMA, //!< CPU has FMA (fused-multiply-add 3 operand form). + kFMA4, //!< CPU has FMA4 (fused-multiply-add 4 operand form). + kFPU, //!< CPU has FPU (FPU support). + kFSGSBASE, //!< CPU has FSGSBASE. + kFXSR, //!< CPU has FXSR (FXSAVE/FXRSTOR instructions). + kFXSROPT, //!< CPU has FXSROTP (FXSAVE/FXRSTOR is optimized). + kGEODE, //!< CPU has GEODE extensions (3DNOW additions). + kGFNI, //!< CPU has GFNI (Galois field instructions). + kHLE, //!< CPU has HLE. + kHRESET, //!< CPU has HRESET. + kI486, //!< CPU has I486 features (I486+ support). + kLAHFSAHF, //!< CPU has LAHF/SAHF (LAHF/SAHF in 64-bit mode) [X86_64]. + kLWP, //!< CPU has LWP (lightweight profiling) [AMD]. + kLZCNT, //!< CPU has LZCNT (LZCNT instruction). + kMCOMMIT, //!< CPU has MCOMMIT (MCOMMIT instruction). + kMMX, //!< CPU has MMX (MMX base instructions). + kMMX2, //!< CPU has MMX2 (MMX extensions or MMX2). + kMONITOR, //!< CPU has MONITOR (MONITOR/MWAIT instructions). + kMONITORX, //!< CPU has MONITORX (MONITORX/MWAITX instructions). + kMOVBE, //!< CPU has MOVBE (move with byte-order swap). + kMOVDIR64B, //!< CPU has MOVDIR64B (move 64 bytes as direct store). + kMOVDIRI, //!< CPU has MOVDIRI (move dword/qword as direct store). + kMPX, //!< CPU has MPX (memory protection extensions). + kMSR, //!< CPU has MSR (RDMSR/WRMSR instructions). + kMSSE, //!< CPU has MSSE (misaligned SSE support). + kOSXSAVE, //!< CPU has OSXSAVE (XSAVE enabled by OS). + kOSPKE, //!< CPU has OSPKE (PKE enabled by OS). + kPCLMULQDQ, //!< CPU has PCLMULQDQ (packed carry-less multiplication). + kPCONFIG, //!< CPU has PCONFIG (PCONFIG instruction). + kPOPCNT, //!< CPU has POPCNT (POPCNT instruction). + kPREFETCHW, //!< CPU has PREFETCHW. + kPREFETCHWT1, //!< CPU has PREFETCHWT1. + kPTWRITE, //!< CPU has PTWRITE. + kRDPID, //!< CPU has RDPID. + kRDPRU, //!< CPU has RDPRU. + kRDRAND, //!< CPU has RDRAND. + kRDSEED, //!< CPU has RDSEED. + kRDTSC, //!< CPU has RDTSC. + kRDTSCP, //!< CPU has RDTSCP. + kRTM, //!< CPU has RTM. + kSERIALIZE, //!< CPU has SERIALIZE. + kSHA, //!< CPU has SHA (SHA-1 and SHA-256 instructions). + kSKINIT, //!< CPU has SKINIT (SKINIT/STGI instructions) [AMD]. + kSMAP, //!< CPU has SMAP (supervisor-mode access prevention). + kSMEP, //!< CPU has SMEP (supervisor-mode execution prevention). + kSMX, //!< CPU has SMX (safer mode extensions). + kSNP, //!< CPU has SNP. + kSSE, //!< CPU has SSE. + kSSE2, //!< CPU has SSE2. + kSSE3, //!< CPU has SSE3. + kSSE4_1, //!< CPU has SSE4.1. + kSSE4_2, //!< CPU has SSE4.2. + kSSE4A, //!< CPU has SSE4A [AMD]. + kSSSE3, //!< CPU has SSSE3. + kSVM, //!< CPU has SVM (virtualization) [AMD]. + kTBM, //!< CPU has TBM (trailing bit manipulation) [AMD]. + kTSX, //!< CPU has TSX. + kTSXLDTRK, //!< CPU has TSXLDTRK. + kUINTR, //!< CPU has UINTR (user interrupts). + kVAES, //!< CPU has VAES (vector AES 256|512 bit support). + kVMX, //!< CPU has VMX (virtualization) [INTEL]. + kVPCLMULQDQ, //!< CPU has VPCLMULQDQ (vector PCLMULQDQ 256|512-bit support). + kWAITPKG, //!< CPU has WAITPKG (UMONITOR, UMWAIT, TPAUSE). + kWBNOINVD, //!< CPU has WBNOINVD. + kXOP, //!< CPU has XOP (XOP instructions) [AMD]. + kXSAVE, //!< CPU has XSAVE. + kXSAVEC, //!< CPU has XSAVEC. + kXSAVEOPT, //!< CPU has XSAVEOPT. + kXSAVES, //!< CPU has XSAVES. + // @EnumValuesEnd@ + + kMaxValue = kXSAVES + }; + + #define ASMJIT_X86_FEATURE(FEATURE) \ + inline bool has##FEATURE() const noexcept { return has(X86::k##FEATURE); } + + ASMJIT_X86_FEATURE(MT) + ASMJIT_X86_FEATURE(NX) + ASMJIT_X86_FEATURE(3DNOW) + ASMJIT_X86_FEATURE(3DNOW2) + ASMJIT_X86_FEATURE(ADX) + ASMJIT_X86_FEATURE(AESNI) + ASMJIT_X86_FEATURE(ALTMOVCR8) + ASMJIT_X86_FEATURE(AMX_BF16) + ASMJIT_X86_FEATURE(AMX_INT8) + ASMJIT_X86_FEATURE(AMX_TILE) + ASMJIT_X86_FEATURE(AVX) + ASMJIT_X86_FEATURE(AVX2) + ASMJIT_X86_FEATURE(AVX512_4FMAPS) + ASMJIT_X86_FEATURE(AVX512_4VNNIW) + ASMJIT_X86_FEATURE(AVX512_BF16) + ASMJIT_X86_FEATURE(AVX512_BITALG) + ASMJIT_X86_FEATURE(AVX512_BW) + ASMJIT_X86_FEATURE(AVX512_CDI) + ASMJIT_X86_FEATURE(AVX512_DQ) + ASMJIT_X86_FEATURE(AVX512_ERI) + ASMJIT_X86_FEATURE(AVX512_F) + ASMJIT_X86_FEATURE(AVX512_FP16) + ASMJIT_X86_FEATURE(AVX512_IFMA) + ASMJIT_X86_FEATURE(AVX512_PFI) + ASMJIT_X86_FEATURE(AVX512_VBMI) + ASMJIT_X86_FEATURE(AVX512_VBMI2) + ASMJIT_X86_FEATURE(AVX512_VL) + ASMJIT_X86_FEATURE(AVX512_VNNI) + ASMJIT_X86_FEATURE(AVX512_VP2INTERSECT) + ASMJIT_X86_FEATURE(AVX512_VPOPCNTDQ) + ASMJIT_X86_FEATURE(AVX_VNNI) + ASMJIT_X86_FEATURE(BMI) + ASMJIT_X86_FEATURE(BMI2) + ASMJIT_X86_FEATURE(CET_IBT) + ASMJIT_X86_FEATURE(CET_SS) + ASMJIT_X86_FEATURE(CLDEMOTE) + ASMJIT_X86_FEATURE(CLFLUSH) + ASMJIT_X86_FEATURE(CLFLUSHOPT) + ASMJIT_X86_FEATURE(CLWB) + ASMJIT_X86_FEATURE(CLZERO) + ASMJIT_X86_FEATURE(CMOV) + ASMJIT_X86_FEATURE(CMPXCHG16B) + ASMJIT_X86_FEATURE(CMPXCHG8B) + ASMJIT_X86_FEATURE(ENCLV) + ASMJIT_X86_FEATURE(ENQCMD) + ASMJIT_X86_FEATURE(ERMS) + ASMJIT_X86_FEATURE(F16C) + ASMJIT_X86_FEATURE(FMA) + ASMJIT_X86_FEATURE(FMA4) + ASMJIT_X86_FEATURE(FPU) + ASMJIT_X86_FEATURE(FSGSBASE) + ASMJIT_X86_FEATURE(FXSR) + ASMJIT_X86_FEATURE(FXSROPT) + ASMJIT_X86_FEATURE(GEODE) + ASMJIT_X86_FEATURE(GFNI) + ASMJIT_X86_FEATURE(HLE) + ASMJIT_X86_FEATURE(HRESET) + ASMJIT_X86_FEATURE(I486) + ASMJIT_X86_FEATURE(LAHFSAHF) + ASMJIT_X86_FEATURE(LWP) + ASMJIT_X86_FEATURE(LZCNT) + ASMJIT_X86_FEATURE(MCOMMIT) + ASMJIT_X86_FEATURE(MMX) + ASMJIT_X86_FEATURE(MMX2) + ASMJIT_X86_FEATURE(MONITOR) + ASMJIT_X86_FEATURE(MONITORX) + ASMJIT_X86_FEATURE(MOVBE) + ASMJIT_X86_FEATURE(MOVDIR64B) + ASMJIT_X86_FEATURE(MOVDIRI) + ASMJIT_X86_FEATURE(MPX) + ASMJIT_X86_FEATURE(MSR) + ASMJIT_X86_FEATURE(MSSE) + ASMJIT_X86_FEATURE(OSXSAVE) + ASMJIT_X86_FEATURE(OSPKE) + ASMJIT_X86_FEATURE(PCLMULQDQ) + ASMJIT_X86_FEATURE(PCONFIG) + ASMJIT_X86_FEATURE(POPCNT) + ASMJIT_X86_FEATURE(PREFETCHW) + ASMJIT_X86_FEATURE(PREFETCHWT1) + ASMJIT_X86_FEATURE(PTWRITE) + ASMJIT_X86_FEATURE(RDPID) + ASMJIT_X86_FEATURE(RDPRU) + ASMJIT_X86_FEATURE(RDRAND) + ASMJIT_X86_FEATURE(RDSEED) + ASMJIT_X86_FEATURE(RDTSC) + ASMJIT_X86_FEATURE(RDTSCP) + ASMJIT_X86_FEATURE(RTM) + ASMJIT_X86_FEATURE(SERIALIZE) + ASMJIT_X86_FEATURE(SHA) + ASMJIT_X86_FEATURE(SKINIT) + ASMJIT_X86_FEATURE(SMAP) + ASMJIT_X86_FEATURE(SMEP) + ASMJIT_X86_FEATURE(SMX) + ASMJIT_X86_FEATURE(SNP) + ASMJIT_X86_FEATURE(SSE) + ASMJIT_X86_FEATURE(SSE2) + ASMJIT_X86_FEATURE(SSE3) + ASMJIT_X86_FEATURE(SSE4_1) + ASMJIT_X86_FEATURE(SSE4_2) + ASMJIT_X86_FEATURE(SSE4A) + ASMJIT_X86_FEATURE(SSSE3) + ASMJIT_X86_FEATURE(SVM) + ASMJIT_X86_FEATURE(TBM) + ASMJIT_X86_FEATURE(TSX) + ASMJIT_X86_FEATURE(TSXLDTRK) + ASMJIT_X86_FEATURE(UINTR) + ASMJIT_X86_FEATURE(VAES) + ASMJIT_X86_FEATURE(VMX) + ASMJIT_X86_FEATURE(VPCLMULQDQ) + ASMJIT_X86_FEATURE(WAITPKG) + ASMJIT_X86_FEATURE(WBNOINVD) + ASMJIT_X86_FEATURE(XOP) + ASMJIT_X86_FEATURE(XSAVE) + ASMJIT_X86_FEATURE(XSAVEC) + ASMJIT_X86_FEATURE(XSAVEOPT) + ASMJIT_X86_FEATURE(XSAVES) + + #undef ASMJIT_X86_FEATURE + }; + + //! ARM specific features data. + struct ARM : public Data { + //! ARM CPU feature identifiers. + enum Id : uint8_t { + // @EnumValuesBegin{"enum": "CpuFeatures::ARM"}@ + kNone = 0, //!< No feature (never set, used internally). + kTHUMB, //!< THUMB v1 ISA. + kTHUMBv2, //!< THUMB v2 ISA. + + kARMv6, //!< ARMv6 ISA. + kARMv7, //!< ARMv7 ISA. + kARMv8a, //!< ARMv8-A ISA. + kARMv8_1a, //!< ARMv8.1-A ISA. + kARMv8_2a, //!< ARMv8.2-A ISA. + kARMv8_3a, //!< ARMv8.3-A ISA. + kARMv8_4a, //!< ARMv8.4-A ISA. + kARMv8_5a, //!< ARMv8.5-A ISA. + kARMv8_6a, //!< ARMv8.6-A ISA. + kARMv8_7a, //!< ARMv8.7-A ISA. + + kVFPv2, //!< CPU has VFPv2 instruction set. + kVFPv3, //!< CPU has VFPv3 instruction set. + kVFPv4, //!< CPU has VFPv4 instruction set. + kVFP_D32, //!< CPU has 32 VFP-D (64-bit) registers. + + kAES, //!< CPU has AES (AArch64 only). + kALTNZCV, //!< CPU has ALTNZCV (AArch64 only). + kASIMD, //!< CPU has Advanced SIMD (NEON on ARM/THUMB). + kBF16, //!< CPU has BF16 (AArch64 only). + kBTI, //!< CPU has BTI (branch target identification). + kCPUID, //!< CPU has accessible CPUID register (ID_AA64ZFR0_EL1). + kCRC32, //!< CPU has CRC32 . + kDGH, //!< CPU has DGH (AArch64 only). + kDIT, //!< CPU has data independent timing instructions (DIT). + kDOTPROD, //!< CPU has DOTPROD (SDOT/UDOT). + kEDSP, //!< CPU has EDSP (ARM/THUMB only). + kFCMA, //!< CPU has FCMA (FCADD/FCMLA). + kFJCVTZS, //!< CPU has FJCVTZS (AArch64 only). + kFLAGM, //!< CPU has FLAGM (AArch64 only). + kFP16CONV, //!< CPU has FP16 (half-float) conversion. + kFP16FML, //!< CPU has FMLAL{2}/FMLSL{2} + kFP16FULL, //!< CPU has full support for FP16. + kFRINT, //!< CPU has FRINT[32|64][X|Z] (AArch64 only). + kI8MM, //!< CPU has I8MM (AArch64 only). + kIDIVA, //!< CPU has hardware SDIV and UDIV (ARM mode). + kIDIVT, //!< CPU has hardware SDIV and UDIV (THUMB mode). + kLSE, //!< CPU has large system extensions (LSE) (AArch64 only). + kMTE, //!< CPU has MTE (AArch64 only). + kRCPC_IMMO, //!< CPU has RCPC_IMMO (AArch64 only). + kRDM, //!< CPU has RDM (AArch64 only). + kPMU, //!< CPU has PMU (AArch64 only). + kPMULL, //!< CPU has PMULL (AArch64 only). + kRNG, //!< CPU has random number generation (RNG). + kSB, //!< CPU has speculative barrier SB (AArch64 only). + kSHA1, //!< CPU has SHA1. + kSHA2, //!< CPU has SHA2. + kSHA3, //!< CPU has SHA3. + kSHA512, //!< CPU has SHA512. + kSM3, //!< CPU has SM3. + kSM4, //!< CPU has SM4. + kSSBS, //!< CPU has SSBS. + kSVE, //!< CPU has SVE (AArch64 only). + kSVE_BF16, //!< CPU has SVE-BF16 (AArch64 only). + kSVE_F32MM, //!< CPU has SVE-F32MM (AArch64 only). + kSVE_F64MM, //!< CPU has SVE-F64MM (AArch64 only). + kSVE_I8MM, //!< CPU has SVE-I8MM (AArch64 only). + kSVE_PMULL, //!< CPU has SVE-PMULL (AArch64 only). + kSVE2, //!< CPU has SVE2 (AArch64 only). + kSVE2_AES, //!< CPU has SVE2-AES (AArch64 only). + kSVE2_BITPERM, //!< CPU has SVE2-BITPERM (AArch64 only). + kSVE2_SHA3, //!< CPU has SVE2-SHA3 (AArch64 only). + kSVE2_SM4, //!< CPU has SVE2-SM4 (AArch64 only). + kTME, //!< CPU has transactional memory extensions (TME). + // @EnumValuesEnd@ + + kMaxValue = kTME + }; + + #define ASMJIT_ARM_FEATURE(FEATURE) \ + inline bool has##FEATURE() const noexcept { return has(ARM::k##FEATURE); } + + ASMJIT_ARM_FEATURE(THUMB) + ASMJIT_ARM_FEATURE(THUMBv2) + + ASMJIT_ARM_FEATURE(ARMv6) + ASMJIT_ARM_FEATURE(ARMv7) + ASMJIT_ARM_FEATURE(ARMv8a) + ASMJIT_ARM_FEATURE(ARMv8_1a) + ASMJIT_ARM_FEATURE(ARMv8_2a) + ASMJIT_ARM_FEATURE(ARMv8_3a) + ASMJIT_ARM_FEATURE(ARMv8_4a) + ASMJIT_ARM_FEATURE(ARMv8_5a) + ASMJIT_ARM_FEATURE(ARMv8_6a) + ASMJIT_ARM_FEATURE(ARMv8_7a) + + ASMJIT_ARM_FEATURE(VFPv2) + ASMJIT_ARM_FEATURE(VFPv3) + ASMJIT_ARM_FEATURE(VFPv4) + ASMJIT_ARM_FEATURE(VFP_D32) + + ASMJIT_ARM_FEATURE(AES) + ASMJIT_ARM_FEATURE(ALTNZCV) + ASMJIT_ARM_FEATURE(ASIMD) + ASMJIT_ARM_FEATURE(BF16) + ASMJIT_ARM_FEATURE(BTI) + ASMJIT_ARM_FEATURE(CPUID) + ASMJIT_ARM_FEATURE(CRC32) + ASMJIT_ARM_FEATURE(DGH) + ASMJIT_ARM_FEATURE(DIT) + ASMJIT_ARM_FEATURE(DOTPROD) + ASMJIT_ARM_FEATURE(EDSP) + ASMJIT_ARM_FEATURE(FCMA) + ASMJIT_ARM_FEATURE(FLAGM) + ASMJIT_ARM_FEATURE(FP16CONV) + ASMJIT_ARM_FEATURE(FP16FML) + ASMJIT_ARM_FEATURE(FP16FULL) + ASMJIT_ARM_FEATURE(FRINT) + ASMJIT_ARM_FEATURE(IDIVA) + ASMJIT_ARM_FEATURE(IDIVT) + ASMJIT_ARM_FEATURE(LSE) + ASMJIT_ARM_FEATURE(MTE) + ASMJIT_ARM_FEATURE(FJCVTZS) + ASMJIT_ARM_FEATURE(I8MM) + ASMJIT_ARM_FEATURE(RCPC_IMMO) + ASMJIT_ARM_FEATURE(RDM) + ASMJIT_ARM_FEATURE(PMU) + ASMJIT_ARM_FEATURE(PMULL) + ASMJIT_ARM_FEATURE(RNG) + ASMJIT_ARM_FEATURE(SB) + ASMJIT_ARM_FEATURE(SHA1) + ASMJIT_ARM_FEATURE(SHA2) + ASMJIT_ARM_FEATURE(SHA3) + ASMJIT_ARM_FEATURE(SHA512) + ASMJIT_ARM_FEATURE(SM3) + ASMJIT_ARM_FEATURE(SM4) + ASMJIT_ARM_FEATURE(SSBS) + ASMJIT_ARM_FEATURE(SVE) + ASMJIT_ARM_FEATURE(SVE_BF16) + ASMJIT_ARM_FEATURE(SVE_F32MM) + ASMJIT_ARM_FEATURE(SVE_F64MM) + ASMJIT_ARM_FEATURE(SVE_I8MM) + ASMJIT_ARM_FEATURE(SVE_PMULL) + ASMJIT_ARM_FEATURE(SVE2) + ASMJIT_ARM_FEATURE(SVE2_AES) + ASMJIT_ARM_FEATURE(SVE2_BITPERM) + ASMJIT_ARM_FEATURE(SVE2_SHA3) + ASMJIT_ARM_FEATURE(SVE2_SM4) + ASMJIT_ARM_FEATURE(TME) + + #undef ASMJIT_ARM_FEATURE + }; + + static_assert(uint32_t(X86::kMaxValue) < kMaxFeatures, "The number of X86 CPU features cannot exceed CpuFeatures::kMaxFeatures"); + static_assert(uint32_t(ARM::kMaxValue) < kMaxFeatures, "The number of ARM CPU features cannot exceed CpuFeatures::kMaxFeatures"); + + //! \} + + //! \name Members + //! \{ + + Data _data {}; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline CpuFeatures() noexcept {} + inline CpuFeatures(const CpuFeatures& other) noexcept = default; + inline explicit CpuFeatures(Globals::NoInit_) noexcept {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline CpuFeatures& operator=(const CpuFeatures& other) noexcept = default; + + inline bool operator==(const CpuFeatures& other) noexcept { return eq(other); } + inline bool operator!=(const CpuFeatures& other) noexcept { return !eq(other); } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns true if there are no features set. + inline bool empty() const noexcept { return _data.empty(); } + + //! Casts this base class into a derived type `T`. + template + inline T& data() noexcept { return static_cast(_data); } + + //! Casts this base class into a derived type `T` (const). + template + inline const T& data() const noexcept { return static_cast(_data); } + + //! Returns CpuFeatures::Data as \ref CpuFeatures::X86. + inline X86& x86() noexcept { return data(); } + //! Returns CpuFeatures::Data as \ref CpuFeatures::X86 (const). + inline const X86& x86() const noexcept { return data(); } + + //! Returns CpuFeatures::Data as \ref CpuFeatures::ARM. + inline ARM& arm() noexcept { return data(); } + //! Returns CpuFeatures::Data as \ref CpuFeatures::ARM (const). + inline const ARM& arm() const noexcept { return data(); } + + //! Returns all features as array of bitwords (see \ref Support::BitWord). + inline BitWord* bits() noexcept { return _data.bits(); } + //! Returns all features as array of bitwords (const). + inline const BitWord* bits() const noexcept { return _data.bits(); } + //! Returns the number of BitWords returned by \ref bits(). + inline size_t bitWordCount() const noexcept { return _data.bitWordCount(); } + + //! Returns \ref Support::BitVectorIterator, that can be used to iterate over all features efficiently. + inline Iterator iterator() const noexcept { return _data.iterator(); } + + //! Tests whether the feature `featureId` is present. + template + inline bool has(const FeatureId& featureId) const noexcept { return _data.has(featureId); } + + //! Tests whether all features as defined by `other` are present. + inline bool hasAll(const CpuFeatures& other) const noexcept { return _data.hasAll(other._data); } + + //! \} + + //! \name Manipulation + //! \{ + + inline void reset() noexcept { _data.reset(); } + + //! Adds the given CPU `featureId` to the list of features. + template + inline void add(Args&&... args) noexcept { return _data.add(std::forward(args)...); } + + //! Adds the given CPU `featureId` to the list of features if `condition` is true. + template + inline void addIf(bool condition, Args&&... args) noexcept { return _data.addIf(condition, std::forward(args)...); } + + //! Removes the given CPU `featureId` from the list of features. + template + inline void remove(Args&&... args) noexcept { return _data.remove(std::forward(args)...); } + + //! Tests whether this CPU features matches `other`. + inline bool eq(const CpuFeatures& other) const noexcept { return _data.eq(other._data); } + + //! \} +}; + +//! CPU information. +class CpuInfo { +public: + //! \name Members + //! \{ + + //! Architecture. + Arch _arch; + //! Sub-architecture. + SubArch _subArch; + //! True if the CPU was detected, false if the detection failed or it's not available. + bool _wasDetected; + //! Reserved for future use. + uint8_t _reserved; + //! CPU family ID. + uint32_t _familyId; + //! CPU model ID. + uint32_t _modelId; + //! CPU brand ID. + uint32_t _brandId; + //! CPU stepping. + uint32_t _stepping; + //! Processor type. + uint32_t _processorType; + //! Maximum number of addressable IDs for logical processors. + uint32_t _maxLogicalProcessors; + //! Cache line size (in bytes). + uint32_t _cacheLineSize; + //! Number of hardware threads. + uint32_t _hwThreadCount; + + //! CPU vendor string. + FixedString<16> _vendor; + //! CPU brand string. + FixedString<64> _brand; + //! CPU features. + CpuFeatures _features; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline CpuInfo() noexcept { reset(); } + inline CpuInfo(const CpuInfo& other) noexcept = default; + + inline explicit CpuInfo(Globals::NoInit_) noexcept + : _features(Globals::NoInit) {}; + + //! Returns the host CPU information. + ASMJIT_API static const CpuInfo& host() noexcept; + + //! Initializes CpuInfo architecture and sub-architecture members to `arch` and `subArch`, respectively. + inline void initArch(Arch arch, SubArch subArch = SubArch::kUnknown) noexcept { + _arch = arch; + _subArch = subArch; + } + + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline CpuInfo& operator=(const CpuInfo& other) noexcept = default; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the CPU architecture this information relates to. + inline Arch arch() const noexcept { return _arch; } + + //! Returns the CPU sub-architecture this information relates to. + inline SubArch subArch() const noexcept { return _subArch; } + + //! Returns whether the CPU was detected successfully. + //! + //! If the returned value is false it means that AsmJit either failed to detect the CPU or it doesn't have + //! implementation targeting the host architecture and operating system. + inline bool wasDetected() const noexcept { return _wasDetected; } + + //! Returns the CPU family ID. + //! + //! Family identifier matches the FamilyId read by using CPUID on X86 architecture. + inline uint32_t familyId() const noexcept { return _familyId; } + + //! Returns the CPU model ID. + //! + //! Family identifier matches the ModelId read by using CPUID on X86 architecture. + + inline uint32_t modelId() const noexcept { return _modelId; } + //! Returns the CPU brand id. + //! + //! Family identifier matches the BrandId read by using CPUID on X86 architecture. + inline uint32_t brandId() const noexcept { return _brandId; } + + //! Returns the CPU stepping. + //! + //! Family identifier matches the Stepping information read by using CPUID on X86 architecture. + inline uint32_t stepping() const noexcept { return _stepping; } + + //! Returns the processor type. + //! + //! Family identifier matches the ProcessorType read by using CPUID on X86 architecture. + inline uint32_t processorType() const noexcept { return _processorType; } + + //! Returns the maximum number of logical processors. + inline uint32_t maxLogicalProcessors() const noexcept { return _maxLogicalProcessors; } + + //! Returns the size of a cache line flush. + inline uint32_t cacheLineSize() const noexcept { return _cacheLineSize; } + + //! Returns number of hardware threads available. + inline uint32_t hwThreadCount() const noexcept { return _hwThreadCount; } + + //! Returns a CPU vendor string. + inline const char* vendor() const noexcept { return _vendor.str; } + //! Tests whether the CPU vendor string is equal to `s`. + inline bool isVendor(const char* s) const noexcept { return _vendor.eq(s); } + + //! Returns a CPU brand string. + inline const char* brand() const noexcept { return _brand.str; } + + //! Returns CPU features. + inline CpuFeatures& features() noexcept { return _features; } + //! Returns CPU features (const). + inline const CpuFeatures& features() const noexcept { return _features; } + + //! Tests whether the CPU has the given `feature`. + template + inline bool hasFeature(const FeatureId& featureId) const noexcept { return _features.has(featureId); } + + //! Adds the given CPU `featureId` to the list of features. + template + inline void addFeature(Args&&... args) noexcept { return _features.add(std::forward(args)...); } + + //! Removes the given CPU `featureId` from the list of features. + template + inline void removeFeature(Args&&... args) noexcept { return _features.remove(std::forward(args)...); } + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_CPUINFO_H_INCLUDED diff --git a/src/asmjit/core/emithelper.cpp b/src/asmjit/core/emithelper.cpp new file mode 100644 index 0000000..bcdf098 --- /dev/null +++ b/src/asmjit/core/emithelper.cpp @@ -0,0 +1,323 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/archtraits.h" +#include "../core/emithelper_p.h" +#include "../core/formatter.h" +#include "../core/funcargscontext_p.h" +#include "../core/radefs_p.h" + +// Can be used for debugging... +// #define ASMJIT_DUMP_ARGS_ASSIGNMENT + +ASMJIT_BEGIN_NAMESPACE + +// BaseEmitHelper - Formatting +// =========================== + +#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT +static void dumpFuncValue(String& sb, Arch arch, const FuncValue& value) noexcept { + Formatter::formatTypeId(sb, value.typeId()); + sb.append('@'); + + if (value.isIndirect()) + sb.append('['); + + if (value.isReg()) + Formatter::formatRegister(sb, 0, nullptr, arch, value.regType(), value.regId()); + else if (value.isStack()) + sb.appendFormat("[%d]", value.stackOffset()); + else + sb.append(""); + + if (value.isIndirect()) + sb.append(']'); +} + +static void dumpAssignment(String& sb, const FuncArgsContext& ctx) noexcept { + typedef FuncArgsContext::Var Var; + + Arch arch = ctx.arch(); + uint32_t varCount = ctx.varCount(); + + for (uint32_t i = 0; i < varCount; i++) { + const Var& var = ctx.var(i); + const FuncValue& dst = var.out; + const FuncValue& cur = var.cur; + + sb.appendFormat("Var%u: ", i); + dumpFuncValue(sb, arch, dst); + sb.append(" <- "); + dumpFuncValue(sb, arch, cur); + + if (var.isDone()) + sb.append(" {Done}"); + + sb.append('\n'); + } +} +#endif + +// BaseEmitHelper - EmitArgsAssignment +// =================================== + +ASMJIT_FAVOR_SIZE Error BaseEmitHelper::emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args) { + typedef FuncArgsContext::Var Var; + typedef FuncArgsContext::WorkData WorkData; + + enum WorkFlags : uint32_t { + kWorkNone = 0x00, + kWorkDidSome = 0x01, + kWorkPending = 0x02, + kWorkPostponed = 0x04 + }; + + Arch arch = frame.arch(); + const ArchTraits& archTraits = ArchTraits::byArch(arch); + + RAConstraints constraints; + FuncArgsContext ctx; + + ASMJIT_PROPAGATE(constraints.init(arch)); + ASMJIT_PROPAGATE(ctx.initWorkData(frame, args, &constraints)); + +#ifdef ASMJIT_DUMP_ARGS_ASSIGNMENT + { + String sb; + dumpAssignment(sb, ctx); + printf("%s\n", sb.data()); + } +#endif + + auto& workData = ctx._workData; + uint32_t varCount = ctx._varCount; + uint32_t saVarId = ctx._saVarId; + + BaseReg sp = BaseReg(_emitter->_gpSignature, archTraits.spRegId()); + BaseReg sa = sp; + + if (frame.hasDynamicAlignment()) { + if (frame.hasPreservedFP()) + sa.setId(archTraits.fpRegId()); + else + sa.setId(saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId()); + } + + // Register to stack and stack to stack moves must be first as now we have + // the biggest chance of having as many as possible unassigned registers. + + if (ctx._stackDstMask) { + // Base address of all arguments passed by stack. + BaseMem baseArgPtr(sa, int32_t(frame.saOffset(sa.id()))); + BaseMem baseStackPtr(sp, 0); + + for (uint32_t varId = 0; varId < varCount; varId++) { + Var& var = ctx._vars[varId]; + + if (!var.out.isStack()) + continue; + + FuncValue& cur = var.cur; + FuncValue& out = var.out; + + ASMJIT_ASSERT(cur.isReg() || cur.isStack()); + BaseReg reg; + + BaseMem dstStackPtr = baseStackPtr.cloneAdjusted(out.stackOffset()); + BaseMem srcStackPtr = baseArgPtr.cloneAdjusted(cur.stackOffset()); + + if (cur.isIndirect()) { + if (cur.isStack()) { + // TODO: Indirect stack. + return DebugUtils::errored(kErrorInvalidAssignment); + } + else { + srcStackPtr.setBaseId(cur.regId()); + } + } + + if (cur.isReg() && !cur.isIndirect()) { + WorkData& wd = workData[archTraits.regTypeToGroup(cur.regType())]; + uint32_t regId = cur.regId(); + + reg.setSignatureAndId(archTraits.regTypeToSignature(cur.regType()), regId); + wd.unassign(varId, regId); + } + else { + // Stack to reg move - tricky since we move stack to stack we can decide which register to use. In general + // we follow the rule that IntToInt moves will use GP regs with possibility to signature or zero extend, + // and all other moves will either use GP or VEC regs depending on the size of the move. + OperandSignature signature = getSuitableRegForMemToMemMove(arch, out.typeId(), cur.typeId()); + if (ASMJIT_UNLIKELY(!signature.isValid())) + return DebugUtils::errored(kErrorInvalidState); + + WorkData& wd = workData[signature.regGroup()]; + RegMask availableRegs = wd.availableRegs(); + if (ASMJIT_UNLIKELY(!availableRegs)) + return DebugUtils::errored(kErrorInvalidState); + + uint32_t availableId = Support::ctz(availableRegs); + reg.setSignatureAndId(signature, availableId); + + ASMJIT_PROPAGATE(emitArgMove(reg, out.typeId(), srcStackPtr, cur.typeId())); + } + + if (cur.isIndirect() && cur.isReg()) + workData[RegGroup::kGp].unassign(varId, cur.regId()); + + // Register to stack move. + ASMJIT_PROPAGATE(emitRegMove(dstStackPtr, reg, cur.typeId())); + var.markDone(); + } + } + + // Shuffle all registers that are currently assigned accordingly to target assignment. + + uint32_t workFlags = kWorkNone; + for (;;) { + for (uint32_t varId = 0; varId < varCount; varId++) { + Var& var = ctx._vars[varId]; + if (var.isDone() || !var.cur.isReg()) + continue; + + FuncValue& cur = var.cur; + FuncValue& out = var.out; + + RegGroup curGroup = archTraits.regTypeToGroup(cur.regType()); + RegGroup outGroup = archTraits.regTypeToGroup(out.regType()); + + uint32_t curId = cur.regId(); + uint32_t outId = out.regId(); + + if (curGroup != outGroup) { + // TODO: Conversion is not supported. + return DebugUtils::errored(kErrorInvalidAssignment); + } + else { + WorkData& wd = workData[outGroup]; + if (!wd.isAssigned(outId)) { +EmitMove: + ASMJIT_PROPAGATE( + emitArgMove( + BaseReg(archTraits.regTypeToSignature(out.regType()), outId), out.typeId(), + BaseReg(archTraits.regTypeToSignature(cur.regType()), curId), cur.typeId())); + + wd.reassign(varId, outId, curId); + cur.initReg(out.regType(), outId, out.typeId()); + + if (outId == out.regId()) + var.markDone(); + workFlags |= kWorkDidSome | kWorkPending; + } + else { + uint32_t altId = wd._physToVarId[outId]; + Var& altVar = ctx._vars[altId]; + + if (!altVar.out.isInitialized() || (altVar.out.isReg() && altVar.out.regId() == curId)) { + // Only few architectures provide swap operations, and only for few register groups. + if (archTraits.hasInstRegSwap(curGroup)) { + RegType highestType = Support::max(cur.regType(), altVar.cur.regType()); + if (Support::isBetween(highestType, RegType::kGp8Lo, RegType::kGp16)) + highestType = RegType::kGp32; + + OperandSignature signature = archTraits.regTypeToSignature(highestType); + ASMJIT_PROPAGATE( + emitRegSwap(BaseReg(signature, outId), BaseReg(signature, curId))); + + wd.swap(varId, curId, altId, outId); + cur.setRegId(outId); + var.markDone(); + altVar.cur.setRegId(curId); + + if (altVar.out.isInitialized()) + altVar.markDone(); + workFlags |= kWorkDidSome; + } + else { + // If there is a scratch register it can be used to perform the swap. + RegMask availableRegs = wd.availableRegs(); + if (availableRegs) { + RegMask inOutRegs = wd.dstRegs(); + if (availableRegs & ~inOutRegs) + availableRegs &= ~inOutRegs; + outId = Support::ctz(availableRegs); + goto EmitMove; + } + else { + workFlags |= kWorkPending; + } + } + } + else { + workFlags |= kWorkPending; + } + } + } + } + + if (!(workFlags & kWorkPending)) + break; + + // If we did nothing twice it means that something is really broken. + if ((workFlags & (kWorkDidSome | kWorkPostponed)) == kWorkPostponed) + return DebugUtils::errored(kErrorInvalidState); + + workFlags = (workFlags & kWorkDidSome) ? kWorkNone : kWorkPostponed; + } + + // Load arguments passed by stack into registers. This is pretty simple and + // it never requires multiple iterations like the previous phase. + + if (ctx._hasStackSrc) { + uint32_t iterCount = 1; + if (frame.hasDynamicAlignment() && !frame.hasPreservedFP()) + sa.setId(saVarId < varCount ? ctx._vars[saVarId].cur.regId() : frame.saRegId()); + + // Base address of all arguments passed by stack. + BaseMem baseArgPtr(sa, int32_t(frame.saOffset(sa.id()))); + + for (uint32_t iter = 0; iter < iterCount; iter++) { + for (uint32_t varId = 0; varId < varCount; varId++) { + Var& var = ctx._vars[varId]; + if (var.isDone()) + continue; + + if (var.cur.isStack()) { + ASMJIT_ASSERT(var.out.isReg()); + + uint32_t outId = var.out.regId(); + RegType outType = var.out.regType(); + + RegGroup group = archTraits.regTypeToGroup(outType); + WorkData& wd = workData[group]; + + if (outId == sa.id() && group == RegGroup::kGp) { + // This register will be processed last as we still need `saRegId`. + if (iterCount == 1) { + iterCount++; + continue; + } + wd.unassign(wd._physToVarId[outId], outId); + } + + BaseReg dstReg = BaseReg(archTraits.regTypeToSignature(outType), outId); + BaseMem srcMem = baseArgPtr.cloneAdjusted(var.cur.stackOffset()); + + ASMJIT_PROPAGATE(emitArgMove( + dstReg, var.out.typeId(), + srcMem, var.cur.typeId())); + + wd.assign(varId, outId); + var.cur.initReg(outType, outId, var.cur.typeId(), FuncValue::kFlagIsDone); + } + } + } + } + + return kErrorOk; +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/emithelper_p.h b/src/asmjit/core/emithelper_p.h new file mode 100644 index 0000000..0333959 --- /dev/null +++ b/src/asmjit/core/emithelper_p.h @@ -0,0 +1,58 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_EMITHELPER_P_H_INCLUDED +#define ASMJIT_CORE_EMITHELPER_P_H_INCLUDED + +#include "../core/emitter.h" +#include "../core/operand.h" +#include "../core/type.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_core +//! \{ + +//! Helper class that provides utilities for each supported architecture. +class BaseEmitHelper { +public: + BaseEmitter* _emitter; + + inline explicit BaseEmitHelper(BaseEmitter* emitter = nullptr) noexcept + : _emitter(emitter) {} + + inline BaseEmitter* emitter() const noexcept { return _emitter; } + inline void setEmitter(BaseEmitter* emitter) noexcept { _emitter = emitter; } + + //! Emits a pure move operation between two registers or the same type or between a register and its home + //! slot. This function does not handle register conversion. + virtual Error emitRegMove( + const Operand_& dst_, + const Operand_& src_, TypeId typeId, const char* comment = nullptr) = 0; + + //! Emits swap between two registers. + virtual Error emitRegSwap( + const BaseReg& a, + const BaseReg& b, const char* comment = nullptr) = 0; + + //! Emits move from a function argument (either register or stack) to a register. + //! + //! This function can handle the necessary conversion from one argument to another, and from one register type + //! to another, if it's possible. Any attempt of conversion that requires third register of a different group + //! (for example conversion from K to MMX on X86/X64) will fail. + virtual Error emitArgMove( + const BaseReg& dst_, TypeId dstTypeId, + const Operand_& src_, TypeId srcTypeId, const char* comment = nullptr) = 0; + + Error emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args); +}; + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_EMITHELPER_P_H_INCLUDED diff --git a/src/asmjit/core/emitter.cpp b/src/asmjit/core/emitter.cpp new file mode 100644 index 0000000..92d67a7 --- /dev/null +++ b/src/asmjit/core/emitter.cpp @@ -0,0 +1,333 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/emitterutils_p.h" +#include "../core/errorhandler.h" +#include "../core/logger.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// BaseEmitter - Construction & Destruction +// ======================================== + +BaseEmitter::BaseEmitter(EmitterType emitterType) noexcept + : _emitterType(emitterType) {} + +BaseEmitter::~BaseEmitter() noexcept { + if (_code) { + _addEmitterFlags(EmitterFlags::kDestroyed); + _code->detach(this); + } +} + +// BaseEmitter - Finalize +// ====================== + +Error BaseEmitter::finalize() { + // Does nothing by default, overridden by `BaseBuilder` and `BaseCompiler`. + return kErrorOk; +} + +// BaseEmitter - Internals +// ======================= + +static constexpr EmitterFlags kEmitterPreservedFlags = EmitterFlags::kOwnLogger | EmitterFlags::kOwnErrorHandler; + +static ASMJIT_NOINLINE void BaseEmitter_updateForcedOptions(BaseEmitter* self) noexcept { + bool emitComments = false; + bool hasDiagnosticOptions = false; + + if (self->emitterType() == EmitterType::kAssembler) { + // Assembler: Don't emit comments if logger is not attached. + emitComments = self->_code != nullptr && self->_logger != nullptr; + hasDiagnosticOptions = self->hasDiagnosticOption(DiagnosticOptions::kValidateAssembler); + } + else { + // Builder/Compiler: Always emit comments, we cannot assume they won't be used. + emitComments = self->_code != nullptr; + hasDiagnosticOptions = self->hasDiagnosticOption(DiagnosticOptions::kValidateIntermediate); + } + + if (emitComments) + self->_addEmitterFlags(EmitterFlags::kLogComments); + else + self->_clearEmitterFlags(EmitterFlags::kLogComments); + + // The reserved option tells emitter (Assembler/Builder/Compiler) that there may be either a border + // case (CodeHolder not attached, for example) or that logging or validation is required. + if (self->_code == nullptr || self->_logger || hasDiagnosticOptions) + self->_forcedInstOptions |= InstOptions::kReserved; + else + self->_forcedInstOptions &= ~InstOptions::kReserved; +} + +// BaseEmitter - Diagnostic Options +// ================================ + +void BaseEmitter::addDiagnosticOptions(DiagnosticOptions options) noexcept { + _diagnosticOptions |= options; + BaseEmitter_updateForcedOptions(this); +} + +void BaseEmitter::clearDiagnosticOptions(DiagnosticOptions options) noexcept { + _diagnosticOptions &= ~options; + BaseEmitter_updateForcedOptions(this); +} + +// BaseEmitter - Logging +// ===================== + +void BaseEmitter::setLogger(Logger* logger) noexcept { +#ifndef ASMJIT_NO_LOGGING + if (logger) { + _logger = logger; + _addEmitterFlags(EmitterFlags::kOwnLogger); + } + else { + _logger = nullptr; + _clearEmitterFlags(EmitterFlags::kOwnLogger); + if (_code) + _logger = _code->logger(); + } + BaseEmitter_updateForcedOptions(this); +#else + DebugUtils::unused(logger); +#endif +} + +// BaseEmitter - Error Handling +// ============================ + +void BaseEmitter::setErrorHandler(ErrorHandler* errorHandler) noexcept { + if (errorHandler) { + _errorHandler = errorHandler; + _addEmitterFlags(EmitterFlags::kOwnErrorHandler); + } + else { + _errorHandler = nullptr; + _clearEmitterFlags(EmitterFlags::kOwnErrorHandler); + if (_code) + _errorHandler = _code->errorHandler(); + } +} + +Error BaseEmitter::reportError(Error err, const char* message) { + ErrorHandler* eh = _errorHandler; + if (eh) { + if (!message) + message = DebugUtils::errorAsString(err); + eh->handleError(err, message, this); + } + return err; +} + +// BaseEmitter - Labels +// ==================== + +Label BaseEmitter::labelByName(const char* name, size_t nameSize, uint32_t parentId) noexcept { + return Label(_code ? _code->labelIdByName(name, nameSize, parentId) : Globals::kInvalidId); +} + +bool BaseEmitter::isLabelValid(uint32_t labelId) const noexcept { + return _code && labelId < _code->labelCount(); +} + +// BaseEmitter - Emit (Low-Level) +// ============================== + +using EmitterUtils::noExt; + +Error BaseEmitter::_emitI(InstId instId) { + return _emit(instId, noExt[0], noExt[1], noExt[2], noExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0) { + return _emit(instId, o0, noExt[1], noExt[2], noExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0, const Operand_& o1) { + return _emit(instId, o0, o1, noExt[2], noExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2) { + return _emit(instId, o0, o1, o2, noExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3) { + Operand_ opExt[3] = { o3 }; + return _emit(instId, o0, o1, o2, opExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4) { + Operand_ opExt[3] = { o3, o4 }; + return _emit(instId, o0, o1, o2, opExt); +} + +Error BaseEmitter::_emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5) { + Operand_ opExt[3] = { o3, o4, o5 }; + return _emit(instId, o0, o1, o2, opExt); +} + +Error BaseEmitter::_emitOpArray(InstId instId, const Operand_* operands, size_t opCount) { + const Operand_* op = operands; + Operand_ opExt[3]; + + switch (opCount) { + case 0: + return _emit(instId, noExt[0], noExt[1], noExt[2], noExt); + + case 1: + return _emit(instId, op[0], noExt[1], noExt[2], noExt); + + case 2: + return _emit(instId, op[0], op[1], noExt[2], noExt); + + case 3: + return _emit(instId, op[0], op[1], op[2], noExt); + + case 4: + opExt[0] = op[3]; + opExt[1].reset(); + opExt[2].reset(); + return _emit(instId, op[0], op[1], op[2], opExt); + + case 5: + opExt[0] = op[3]; + opExt[1] = op[4]; + opExt[2].reset(); + return _emit(instId, op[0], op[1], op[2], opExt); + + case 6: + return _emit(instId, op[0], op[1], op[2], op + 3); + + default: + return DebugUtils::errored(kErrorInvalidArgument); + } +} + +// BaseEmitter - Emit Utilities +// ============================ + +Error BaseEmitter::emitProlog(const FuncFrame& frame) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + return _funcs.emitProlog(this, frame); +} + +Error BaseEmitter::emitEpilog(const FuncFrame& frame) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + return _funcs.emitEpilog(this, frame); +} + +Error BaseEmitter::emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args) { + if (ASMJIT_UNLIKELY(!_code)) + return DebugUtils::errored(kErrorNotInitialized); + + return _funcs.emitArgsAssignment(this, frame, args); +} + +// BaseEmitter - Comment +// ===================== + +Error BaseEmitter::commentf(const char* fmt, ...) { + if (!hasEmitterFlag(EmitterFlags::kLogComments)) { + if (!hasEmitterFlag(EmitterFlags::kAttached)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + return kErrorOk; + } + +#ifndef ASMJIT_NO_LOGGING + StringTmp<1024> sb; + + va_list ap; + va_start(ap, fmt); + Error err = sb.appendVFormat(fmt, ap); + va_end(ap); + + ASMJIT_PROPAGATE(err); + return comment(sb.data(), sb.size()); +#else + DebugUtils::unused(fmt); + return kErrorOk; +#endif +} + +Error BaseEmitter::commentv(const char* fmt, va_list ap) { + if (!hasEmitterFlag(EmitterFlags::kLogComments)) { + if (!hasEmitterFlag(EmitterFlags::kAttached)) + return reportError(DebugUtils::errored(kErrorNotInitialized)); + return kErrorOk; + } + +#ifndef ASMJIT_NO_LOGGING + StringTmp<1024> sb; + Error err = sb.appendVFormat(fmt, ap); + + ASMJIT_PROPAGATE(err); + return comment(sb.data(), sb.size()); +#else + DebugUtils::unused(fmt, ap); + return kErrorOk; +#endif +} + +// BaseEmitter - Events +// ==================== + +Error BaseEmitter::onAttach(CodeHolder* code) noexcept { + _code = code; + _environment = code->environment(); + _addEmitterFlags(EmitterFlags::kAttached); + + const ArchTraits& archTraits = ArchTraits::byArch(code->arch()); + RegType nativeRegType = Environment::is32Bit(code->arch()) ? RegType::kGp32 : RegType::kGp64; + _gpSignature = archTraits.regTypeToSignature(nativeRegType); + + onSettingsUpdated(); + return kErrorOk; +} + +Error BaseEmitter::onDetach(CodeHolder* code) noexcept { + DebugUtils::unused(code); + + if (!hasOwnLogger()) + _logger = nullptr; + + if (!hasOwnErrorHandler()) + _errorHandler = nullptr; + + _clearEmitterFlags(~kEmitterPreservedFlags); + _forcedInstOptions = InstOptions::kReserved; + _privateData = 0; + + _environment.reset(); + _gpSignature.reset(); + + _instOptions = InstOptions::kNone; + _extraReg.reset(); + _inlineComment = nullptr; + + return kErrorOk; +} + +void BaseEmitter::onSettingsUpdated() noexcept { + // Only called when attached to CodeHolder by CodeHolder. + ASMJIT_ASSERT(_code != nullptr); + + if (!hasOwnLogger()) + _logger = _code->logger(); + + if (!hasOwnErrorHandler()) + _errorHandler = _code->errorHandler(); + + BaseEmitter_updateForcedOptions(this); +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/emitter.h b/src/asmjit/core/emitter.h new file mode 100644 index 0000000..b8afd6b --- /dev/null +++ b/src/asmjit/core/emitter.h @@ -0,0 +1,741 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_EMITTER_H_INCLUDED +#define ASMJIT_CORE_EMITTER_H_INCLUDED + +#include "../core/archtraits.h" +#include "../core/codeholder.h" +#include "../core/formatter.h" +#include "../core/inst.h" +#include "../core/operand.h" +#include "../core/type.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +class ConstPool; +class FuncFrame; +class FuncArgsAssignment; + +//! Align mode, used by \ref BaseEmitter::align(). +enum class AlignMode : uint8_t { + //! Align executable code. + kCode = 0, + //! Align non-executable code. + kData = 1, + //! Align by a sequence of zeros. + kZero = 2, + + //! Maximum value of `AlignMode`. + kMaxValue = kZero +}; + +//! Emitter type used by \ref BaseEmitter. +enum class EmitterType : uint8_t { + //! Unknown or uninitialized. + kNone = 0, + //! Emitter inherits from \ref BaseAssembler. + kAssembler = 1, + //! Emitter inherits from \ref BaseBuilder. + kBuilder = 2, + //! Emitter inherits from \ref BaseCompiler. + kCompiler = 3, + + //! Maximum value of `EmitterType`. + kMaxValue = kCompiler +}; + +//! Emitter flags, used by \ref BaseEmitter. +enum class EmitterFlags : uint8_t { + //! No flags. + kNone = 0u, + //! Emitter is attached to CodeHolder. + kAttached = 0x01u, + //! The emitter must emit comments. + kLogComments = 0x08u, + //! The emitter has its own \ref Logger (not propagated from \ref CodeHolder). + kOwnLogger = 0x10u, + //! The emitter has its own \ref ErrorHandler (not propagated from \ref CodeHolder). + kOwnErrorHandler = 0x20u, + //! The emitter was finalized. + kFinalized = 0x40u, + //! The emitter was destroyed. + //! + //! This flag is used for a very short time when an emitter is being destroyed by + //! CodeHolder. + kDestroyed = 0x80u +}; +ASMJIT_DEFINE_ENUM_FLAGS(EmitterFlags) + +//! Encoding options. +enum class EncodingOptions : uint32_t { + //! No encoding options. + kNone = 0, + + //! Emit instructions that are optimized for size, if possible. + //! + //! Default: false. + //! + //! X86 Specific + //! ------------ + //! + //! When this option is set it the assembler will try to fix instructions if possible into operation equivalent + //! instructions that take less bytes by taking advantage of implicit zero extension. For example instruction + //! like `mov r64, imm` and `and r64, imm` can be translated to `mov r32, imm` and `and r32, imm` when the + //! immediate constant is lesser than `2^31`. + kOptimizeForSize = 0x00000001u, + + //! Emit optimized code-alignment sequences. + //! + //! Default: false. + //! + //! X86 Specific + //! ------------ + //! + //! Default align sequence used by X86 architecture is one-byte (0x90) opcode that is often shown by disassemblers + //! as NOP. However there are more optimized align sequences for 2-11 bytes that may execute faster on certain CPUs. + //! If this feature is enabled AsmJit will generate specialized sequences for alignment between 2 to 11 bytes. + kOptimizedAlign = 0x00000002u, + + //! Emit jump-prediction hints. + //! + //! Default: false. + //! + //! X86 Specific + //! ------------ + //! + //! Jump prediction is usually based on the direction of the jump. If the jump is backward it is usually predicted as + //! taken; and if the jump is forward it is usually predicted as not-taken. The reason is that loops generally use + //! backward jumps and conditions usually use forward jumps. However this behavior can be overridden by using + //! instruction prefixes. If this option is enabled these hints will be emitted. + //! + //! This feature is disabled by default, because the only processor that used to take into consideration prediction + //! hints was P4. Newer processors implement heuristics for branch prediction and ignore static hints. This means + //! that this feature can be only used for annotation purposes. + kPredictedJumps = 0x00000010u +}; +ASMJIT_DEFINE_ENUM_FLAGS(EncodingOptions) + +//! Diagnostic options are used to tell emitters and their passes to perform diagnostics when emitting or processing +//! user code. These options control validation and extra diagnostics that can be performed by higher level emitters. +//! +//! Instruction Validation +//! ---------------------- +//! +//! \ref BaseAssembler implementation perform by default only basic checks that are necessary to identify all +//! variations of an instruction so the correct encoding can be selected. This is fine for production-ready code +//! as the assembler doesn't have to perform checks that would slow it down. However, sometimes these checks are +//! beneficial especially when the project that uses AsmJit is in a development phase, in which mistakes happen +//! often. To make the experience of using AsmJit seamless it offers validation features that can be controlled +//! by \ref DiagnosticOptions. +//! +//! Compiler Diagnostics +//! -------------------- +//! +//! Diagnostic options work with \ref BaseCompiler passes (precisely with its register allocation pass). These options +//! can be used to enable logging of all operations that the Compiler does. +enum class DiagnosticOptions : uint32_t { + //! No validation options. + kNone = 0, + + //! Perform strict validation in \ref BaseAssembler::emit() implementations. + //! + //! This flag ensures that each instruction is checked before it's encoded into a binary representation. This flag + //! is only relevant for \ref BaseAssembler implementations, but can be set in any other emitter type, in that case + //! if that emitter needs to create an assembler on its own, for the purpose of \ref BaseEmitter::finalize() it + //! would propagate this flag to such assembler so all instructions passed to it are explicitly validated. + //! + //! Default: false. + kValidateAssembler = 0x00000001u, + + //! Perform strict validation in \ref BaseBuilder::emit() and \ref BaseCompiler::emit() implementations. + //! + //! This flag ensures that each instruction is checked before an \ref InstNode representing the instruction is + //! created by \ref BaseBuilder or \ref BaseCompiler. This option could be more useful than \ref kValidateAssembler + //! in cases in which there is an invalid instruction passed to an assembler, which was invalid much earlier, most + //! likely when such instruction was passed to Builder/Compiler. + //! + //! This is a separate option that was introduced, because it's possible to manipulate the instruction stream + //! emitted by \ref BaseBuilder and \ref BaseCompiler - this means that it's allowed to emit invalid instructions + //! (for example with missing operands) that will be fixed later before finalizing it. + //! + //! Default: false. + kValidateIntermediate = 0x00000002u, + + //! Annotate all nodes processed by register allocator (Compiler/RA). + //! + //! \note Annotations don't need debug options, however, some debug options like `kRADebugLiveness` may influence + //! their output (for example the mentioned option would add liveness information to per-instruction annotation). + kRAAnnotate = 0x00000080u, + + //! Debug CFG generation and other related algorithms / operations (Compiler/RA). + kRADebugCFG = 0x00000100u, + + //! Debug liveness analysis (Compiler/RA). + kRADebugLiveness = 0x00000200u, + + //! Debug register allocation assignment (Compiler/RA). + kRADebugAssignment = 0x00000400u, + + //! Debug the removal of code part of unreachable blocks. + kRADebugUnreachable = 0x00000800u, + + //! Enable all debug options (Compiler/RA). + kRADebugAll = 0x0000FF00u, +}; +ASMJIT_DEFINE_ENUM_FLAGS(DiagnosticOptions) + +//! Provides a base foundation to emitting code - specialized by \ref BaseAssembler and \ref BaseBuilder. +class ASMJIT_VIRTAPI BaseEmitter { +public: + ASMJIT_BASE_CLASS(BaseEmitter) + + //! \name Members + //! \{ + + //! See \ref EmitterType. + EmitterType _emitterType = EmitterType::kNone; + //! See \ref EmitterFlags. + EmitterFlags _emitterFlags = EmitterFlags::kNone; + //! Validation flags in case validation is used. + //! + //! \note Validation flags are specific to the emitter and they are setup at construction time and then never + //! changed. + ValidationFlags _validationFlags = ValidationFlags::kNone; + //! Validation options. + DiagnosticOptions _diagnosticOptions = DiagnosticOptions::kNone; + + //! All supported architectures in a bit-mask, where LSB is the bit with a zero index. + uint64_t _archMask = 0; + + //! Encoding options. + EncodingOptions _encodingOptions = EncodingOptions::kNone; + + //! Forced instruction options, combined with \ref _instOptions by \ref emit(). + InstOptions _forcedInstOptions = InstOptions::kReserved; + //! Internal private data used freely by any emitter. + uint32_t _privateData = 0; + + //! CodeHolder the emitter is attached to. + CodeHolder* _code = nullptr; + //! Attached \ref Logger. + Logger* _logger = nullptr; + //! Attached \ref ErrorHandler. + ErrorHandler* _errorHandler = nullptr; + + //! Describes the target environment, matches \ref CodeHolder::environment(). + Environment _environment {}; + //! Native GP register signature and signature related information. + OperandSignature _gpSignature {}; + + //! Next instruction options (affects the next instruction). + InstOptions _instOptions = InstOptions::kNone; + //! Extra register (op-mask {k} on AVX-512) (affects the next instruction). + RegOnly _extraReg {}; + //! Inline comment of the next instruction (affects the next instruction). + const char* _inlineComment = nullptr; + + //! Function callbacks used by emitter implementation. + //! + //! These are typically shared between Assembler/Builder/Compiler of a single backend. + struct Funcs { + typedef Error (ASMJIT_CDECL* EmitProlog)(BaseEmitter* emitter, const FuncFrame& frame); + typedef Error (ASMJIT_CDECL* EmitEpilog)(BaseEmitter* emitter, const FuncFrame& frame); + typedef Error (ASMJIT_CDECL* EmitArgsAssignment)(BaseEmitter* emitter, const FuncFrame& frame, const FuncArgsAssignment& args); + + typedef Error (ASMJIT_CDECL* FormatInstruction)( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + const BaseInst& inst, const Operand_* operands, size_t opCount) ASMJIT_NOEXCEPT_TYPE; + + typedef Error (ASMJIT_CDECL* ValidateFunc)(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) ASMJIT_NOEXCEPT_TYPE; + + //! Emit prolog implementation. + EmitProlog emitProlog; + //! Emit epilog implementation. + EmitEpilog emitEpilog; + //! Emit arguments assignment implementation. + EmitArgsAssignment emitArgsAssignment; + //! Instruction formatter implementation. + FormatInstruction formatInstruction; + //! Instruction validation implementation. + ValidateFunc validate; + + //! Resets all functions to nullptr. + inline void reset() noexcept { + emitProlog = nullptr; + emitEpilog = nullptr; + emitArgsAssignment = nullptr; + validate = nullptr; + } + }; + + Funcs _funcs {}; + + //! \} + + //! \name Construction & Destruction + //! \{ + + ASMJIT_API explicit BaseEmitter(EmitterType emitterType) noexcept; + ASMJIT_API virtual ~BaseEmitter() noexcept; + + //! \} + + //! \name Cast + //! \{ + + template + inline T* as() noexcept { return reinterpret_cast(this); } + + template + inline const T* as() const noexcept { return reinterpret_cast(this); } + + //! \} + + //! \name Emitter Type & Flags + //! \{ + + //! Returns the type of this emitter, see `EmitterType`. + inline EmitterType emitterType() const noexcept { return _emitterType; } + //! Returns emitter flags , see `Flags`. + inline EmitterFlags emitterFlags() const noexcept { return _emitterFlags; } + + //! Tests whether the emitter inherits from `BaseAssembler`. + inline bool isAssembler() const noexcept { return _emitterType == EmitterType::kAssembler; } + //! Tests whether the emitter inherits from `BaseBuilder`. + //! + //! \note Both Builder and Compiler emitters would return `true`. + inline bool isBuilder() const noexcept { return uint32_t(_emitterType) >= uint32_t(EmitterType::kBuilder); } + //! Tests whether the emitter inherits from `BaseCompiler`. + inline bool isCompiler() const noexcept { return _emitterType == EmitterType::kCompiler; } + + //! Tests whether the emitter has the given `flag` enabled. + inline bool hasEmitterFlag(EmitterFlags flag) const noexcept { return Support::test(_emitterFlags, flag); } + //! Tests whether the emitter is finalized. + inline bool isFinalized() const noexcept { return hasEmitterFlag(EmitterFlags::kFinalized); } + //! Tests whether the emitter is destroyed (only used during destruction). + inline bool isDestroyed() const noexcept { return hasEmitterFlag(EmitterFlags::kDestroyed); } + + inline void _addEmitterFlags(EmitterFlags flags) noexcept { _emitterFlags |= flags; } + inline void _clearEmitterFlags(EmitterFlags flags) noexcept { _emitterFlags &= _emitterFlags & ~flags; } + + //! \} + + //! \name Target Information + //! \{ + + //! Returns the CodeHolder this emitter is attached to. + inline CodeHolder* code() const noexcept { return _code; } + + //! Returns the target environment. + //! + //! The returned \ref Environment reference matches \ref CodeHolder::environment(). + inline const Environment& environment() const noexcept { return _environment; } + + //! Tests whether the target architecture is 32-bit. + inline bool is32Bit() const noexcept { return environment().is32Bit(); } + //! Tests whether the target architecture is 64-bit. + inline bool is64Bit() const noexcept { return environment().is64Bit(); } + + //! Returns the target architecture type. + inline Arch arch() const noexcept { return environment().arch(); } + //! Returns the target architecture sub-type. + inline SubArch subArch() const noexcept { return environment().subArch(); } + + //! Returns the target architecture's GP register size (4 or 8 bytes). + inline uint32_t registerSize() const noexcept { return environment().registerSize(); } + + //! \} + + //! \name Initialization & Finalization + //! \{ + + //! Tests whether the emitter is initialized (i.e. attached to \ref CodeHolder). + inline bool isInitialized() const noexcept { return _code != nullptr; } + + //! Finalizes this emitter. + //! + //! Materializes the content of the emitter by serializing it to the attached \ref CodeHolder through an architecture + //! specific \ref BaseAssembler. This function won't do anything if the emitter inherits from \ref BaseAssembler as + //! assemblers emit directly to a \ref CodeBuffer held by \ref CodeHolder. However, if this is an emitter that + //! inherits from \ref BaseBuilder or \ref BaseCompiler then these emitters need the materialization phase as they + //! store their content in a representation not visible to \ref CodeHolder. + ASMJIT_API virtual Error finalize(); + + //! \} + + //! \name Logging + //! \{ + + //! Tests whether the emitter has a logger. + inline bool hasLogger() const noexcept { return _logger != nullptr; } + + //! Tests whether the emitter has its own logger. + //! + //! Own logger means that it overrides the possible logger that may be used by \ref CodeHolder this emitter is + //! attached to. + inline bool hasOwnLogger() const noexcept { return hasEmitterFlag(EmitterFlags::kOwnLogger); } + + //! Returns the logger this emitter uses. + //! + //! The returned logger is either the emitter's own logger or it's logger used by \ref CodeHolder this emitter + //! is attached to. + inline Logger* logger() const noexcept { return _logger; } + + //! Sets or resets the logger of the emitter. + //! + //! If the `logger` argument is non-null then the logger will be considered emitter's own logger, see \ref + //! hasOwnLogger() for more details. If the given `logger` is null then the emitter will automatically use logger + //! that is attached to the \ref CodeHolder this emitter is attached to. + ASMJIT_API void setLogger(Logger* logger) noexcept; + + //! Resets the logger of this emitter. + //! + //! The emitter will bail to using a logger attached to \ref CodeHolder this emitter is attached to, or no logger + //! at all if \ref CodeHolder doesn't have one. + inline void resetLogger() noexcept { return setLogger(nullptr); } + + //! \} + + //! \name Error Handling + //! \{ + + //! Tests whether the emitter has an error handler attached. + inline bool hasErrorHandler() const noexcept { return _errorHandler != nullptr; } + + //! Tests whether the emitter has its own error handler. + //! + //! Own error handler means that it overrides the possible error handler that may be used by \ref CodeHolder this + //! emitter is attached to. + inline bool hasOwnErrorHandler() const noexcept { return hasEmitterFlag(EmitterFlags::kOwnErrorHandler); } + + //! Returns the error handler this emitter uses. + //! + //! The returned error handler is either the emitter's own error handler or it's error handler used by + //! \ref CodeHolder this emitter is attached to. + inline ErrorHandler* errorHandler() const noexcept { return _errorHandler; } + + //! Sets or resets the error handler of the emitter. + ASMJIT_API void setErrorHandler(ErrorHandler* errorHandler) noexcept; + + //! Resets the error handler. + inline void resetErrorHandler() noexcept { setErrorHandler(nullptr); } + + //! Handles the given error in the following way: + //! 1. If the emitter has \ref ErrorHandler attached, it calls its \ref ErrorHandler::handleError() member function + //! first, and then returns the error. The `handleError()` function may throw. + //! 2. if the emitter doesn't have \ref ErrorHandler, the error is simply returned. + ASMJIT_API Error reportError(Error err, const char* message = nullptr); + + //! \} + + //! \name Encoding Options + //! \{ + + //! Returns encoding options. + inline EncodingOptions encodingOptions() const noexcept { return _encodingOptions; } + //! Tests whether the encoding `option` is set. + inline bool hasEncodingOption(EncodingOptions option) const noexcept { return Support::test(_encodingOptions, option); } + + //! Enables the given encoding `options`. + inline void addEncodingOptions(EncodingOptions options) noexcept { _encodingOptions |= options; } + //! Disables the given encoding `options`. + inline void clearEncodingOptions(EncodingOptions options) noexcept { _encodingOptions &= ~options; } + + //! \} + + //! \name Diagnostic Options + //! \{ + + //! Returns the emitter's diagnostic options. + inline DiagnosticOptions diagnosticOptions() const noexcept { return _diagnosticOptions; } + + //! Tests whether the given `option` is present in the emitter's diagnostic options. + inline bool hasDiagnosticOption(DiagnosticOptions option) const noexcept { return Support::test(_diagnosticOptions, option); } + + //! Activates the given diagnostic `options`. + //! + //! This function is used to activate explicit validation options that will be then used by all emitter + //! implementations. There are in general two possibilities: + //! + //! - Architecture specific assembler is used. In this case a \ref DiagnosticOptions::kValidateAssembler can be + //! used to turn on explicit validation that will be used before an instruction is emitted. This means that + //! internally an extra step will be performed to make sure that the instruction is correct. This is needed, + //! because by default assemblers prefer speed over strictness. + //! + //! This option should be used in debug builds as it's pretty expensive. + //! + //! - Architecture specific builder or compiler is used. In this case the user can turn on + //! \ref DiagnosticOptions::kValidateIntermediate option that adds explicit validation step before the Builder + //! or Compiler creates an \ref InstNode to represent an emitted instruction. Error will be returned if the + //! instruction is ill-formed. In addition, also \ref DiagnosticOptions::kValidateAssembler can be used, which + //! would not be consumed by Builder / Compiler directly, but it would be propagated to an architecture specific + //! \ref BaseAssembler implementation it creates during \ref BaseEmitter::finalize(). + ASMJIT_API void addDiagnosticOptions(DiagnosticOptions options) noexcept; + + //! Deactivates the given validation `options`. + //! + //! See \ref addDiagnosticOptions() and \ref DiagnosticOptions for more details. + ASMJIT_API void clearDiagnosticOptions(DiagnosticOptions options) noexcept; + + //! \} + + //! \name Instruction Options + //! \{ + + //! Returns forced instruction options. + //! + //! Forced instruction options are merged with next instruction options before the instruction is encoded. These + //! options have some bits reserved that are used by error handling, logging, and instruction validation purposes. + //! Other options are globals that affect each instruction. + inline InstOptions forcedInstOptions() const noexcept { return _forcedInstOptions; } + + //! Returns options of the next instruction. + inline InstOptions instOptions() const noexcept { return _instOptions; } + //! Returns options of the next instruction. + inline void setInstOptions(InstOptions options) noexcept { _instOptions = options; } + //! Adds options of the next instruction. + inline void addInstOptions(InstOptions options) noexcept { _instOptions |= options; } + //! Resets options of the next instruction. + inline void resetInstOptions() noexcept { _instOptions = InstOptions::kNone; } + + //! Tests whether the extra register operand is valid. + inline bool hasExtraReg() const noexcept { return _extraReg.isReg(); } + //! Returns an extra operand that will be used by the next instruction (architecture specific). + inline const RegOnly& extraReg() const noexcept { return _extraReg; } + //! Sets an extra operand that will be used by the next instruction (architecture specific). + inline void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + //! Sets an extra operand that will be used by the next instruction (architecture specific). + inline void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } + //! Resets an extra operand that will be used by the next instruction (architecture specific). + inline void resetExtraReg() noexcept { _extraReg.reset(); } + + //! Returns comment/annotation of the next instruction. + inline const char* inlineComment() const noexcept { return _inlineComment; } + //! Sets comment/annotation of the next instruction. + //! + //! \note This string is set back to null by `_emit()`, but until that it has to remain valid as the Emitter is not + //! required to make a copy of it (and it would be slow to do that for each instruction). + inline void setInlineComment(const char* s) noexcept { _inlineComment = s; } + //! Resets the comment/annotation to nullptr. + inline void resetInlineComment() noexcept { _inlineComment = nullptr; } + + //! \} + + //! \name Sections + //! \{ + + virtual Error section(Section* section) = 0; + + //! \} + + //! \name Labels + //! \{ + + //! Creates a new label. + virtual Label newLabel() = 0; + //! Creates a new named label. + virtual Label newNamedLabel(const char* name, size_t nameSize = SIZE_MAX, LabelType type = LabelType::kGlobal, uint32_t parentId = Globals::kInvalidId) = 0; + + //! Creates a new anonymous label with a name, which can only be used for debugging purposes. + inline Label newAnonymousLabel(const char* name, size_t nameSize = SIZE_MAX) { return newNamedLabel(name, nameSize, LabelType::kAnonymous); } + //! Creates a new external label. + inline Label newExternalLabel(const char* name, size_t nameSize = SIZE_MAX) { return newNamedLabel(name, nameSize, LabelType::kExternal); } + + //! Returns `Label` by `name`. + //! + //! Returns invalid Label in case that the name is invalid or label was not found. + //! + //! \note This function doesn't trigger ErrorHandler in case the name is invalid or no such label exist. You must + //! always check the validity of the `Label` returned. + ASMJIT_API Label labelByName(const char* name, size_t nameSize = SIZE_MAX, uint32_t parentId = Globals::kInvalidId) noexcept; + + //! Binds the `label` to the current position of the current section. + //! + //! \note Attempt to bind the same label multiple times will return an error. + virtual Error bind(const Label& label) = 0; + + //! Tests whether the label `id` is valid (i.e. registered). + ASMJIT_API bool isLabelValid(uint32_t labelId) const noexcept; + //! Tests whether the `label` is valid (i.e. registered). + inline bool isLabelValid(const Label& label) const noexcept { return isLabelValid(label.id()); } + + //! \} + + //! \name Emit + //! \{ + + // NOTE: These `emit()` helpers are designed to address a code-bloat generated by C++ compilers to call a function + // having many arguments. Each parameter to `_emit()` requires some code to pass it, which means that if we default + // to 5 arguments in `_emit()` and instId the C++ compiler would have to generate a virtual function call having 5 + // parameters and additional `this` argument, which is quite a lot. Since by default most instructions have 2 to 3 + // operands it's better to introduce helpers that pass from 0 to 6 operands that help to reduce the size of emit(...) + // function call. + + //! Emits an instruction (internal). + ASMJIT_API Error _emitI(InstId instId); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4); + //! \overload + ASMJIT_API Error _emitI(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_& o3, const Operand_& o4, const Operand_& o5); + + //! Emits an instruction `instId` with the given `operands`. + template + ASMJIT_FORCE_INLINE Error emit(InstId instId, Args&&... operands) { + return _emitI(instId, Support::ForwardOp::forward(operands)...); + } + + ASMJIT_FORCE_INLINE Error emitOpArray(InstId instId, const Operand_* operands, size_t opCount) { + return _emitOpArray(instId, operands, opCount); + } + + ASMJIT_FORCE_INLINE Error emitInst(const BaseInst& inst, const Operand_* operands, size_t opCount) { + setInstOptions(inst.options()); + setExtraReg(inst.extraReg()); + return _emitOpArray(inst.id(), operands, opCount); + } + + //! \cond INTERNAL + //! Emits an instruction - all 6 operands must be defined. + virtual Error _emit(InstId instId, const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* oExt) = 0; + //! Emits instruction having operands stored in array. + ASMJIT_API virtual Error _emitOpArray(InstId instId, const Operand_* operands, size_t opCount); + //! \endcond + + //! \} + + //! \name Emit Utilities + //! \{ + + ASMJIT_API Error emitProlog(const FuncFrame& frame); + ASMJIT_API Error emitEpilog(const FuncFrame& frame); + ASMJIT_API Error emitArgsAssignment(const FuncFrame& frame, const FuncArgsAssignment& args); + + //! \} + + //! \name Align + //! \{ + + //! Aligns the current CodeBuffer position to the `alignment` specified. + //! + //! The sequence that is used to fill the gap between the aligned location and the current location depends on the + //! align `mode`, see \ref AlignMode. The `alignment` argument specifies alignment in bytes, so for example when + //! it's `32` it means that the code buffer will be aligned to `32` bytes. + virtual Error align(AlignMode alignMode, uint32_t alignment) = 0; + + //! \} + + //! \name Embed + //! \{ + + //! Embeds raw data into the \ref CodeBuffer. + virtual Error embed(const void* data, size_t dataSize) = 0; + + //! Embeds a typed data array. + //! + //! This is the most flexible function for embedding data as it allows to: + //! + //! - Assign a `typeId` to the data, so the emitter knows the type of items stored in `data`. Binary data should + //! use \ref TypeId::kUInt8. + //! + //! - Repeat the given data `repeatCount` times, so the data can be used as a fill pattern for example, or as a + //! pattern used by SIMD instructions. + virtual Error embedDataArray(TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1) = 0; + + //! Embeds int8_t `value` repeated by `repeatCount`. + inline Error embedInt8(int8_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kInt8, &value, 1, repeatCount); } + //! Embeds uint8_t `value` repeated by `repeatCount`. + inline Error embedUInt8(uint8_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kUInt8, &value, 1, repeatCount); } + //! Embeds int16_t `value` repeated by `repeatCount`. + inline Error embedInt16(int16_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kInt16, &value, 1, repeatCount); } + //! Embeds uint16_t `value` repeated by `repeatCount`. + inline Error embedUInt16(uint16_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kUInt16, &value, 1, repeatCount); } + //! Embeds int32_t `value` repeated by `repeatCount`. + inline Error embedInt32(int32_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kInt32, &value, 1, repeatCount); } + //! Embeds uint32_t `value` repeated by `repeatCount`. + inline Error embedUInt32(uint32_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kUInt32, &value, 1, repeatCount); } + //! Embeds int64_t `value` repeated by `repeatCount`. + inline Error embedInt64(int64_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kInt64, &value, 1, repeatCount); } + //! Embeds uint64_t `value` repeated by `repeatCount`. + inline Error embedUInt64(uint64_t value, size_t repeatCount = 1) { return embedDataArray(TypeId::kUInt64, &value, 1, repeatCount); } + //! Embeds a floating point `value` repeated by `repeatCount`. + inline Error embedFloat(float value, size_t repeatCount = 1) { return embedDataArray(TypeId(TypeUtils::TypeIdOfT::kTypeId), &value, 1, repeatCount); } + //! Embeds a floating point `value` repeated by `repeatCount`. + inline Error embedDouble(double value, size_t repeatCount = 1) { return embedDataArray(TypeId(TypeUtils::TypeIdOfT::kTypeId), &value, 1, repeatCount); } + + //! Embeds a constant pool at the current offset by performing the following: + //! 1. Aligns by using AlignMode::kData to the minimum `pool` alignment. + //! 2. Binds the ConstPool label so it's bound to an aligned location. + //! 3. Emits ConstPool content. + virtual Error embedConstPool(const Label& label, const ConstPool& pool) = 0; + + //! Embeds an absolute `label` address as data. + //! + //! The `dataSize` is an optional argument that can be used to specify the size of the address data. If it's zero + //! (default) the address size is deduced from the target architecture (either 4 or 8 bytes). + virtual Error embedLabel(const Label& label, size_t dataSize = 0) = 0; + + //! Embeds a delta (distance) between the `label` and `base` calculating it as `label - base`. This function was + //! designed to make it easier to embed lookup tables where each index is a relative distance of two labels. + virtual Error embedLabelDelta(const Label& label, const Label& base, size_t dataSize = 0) = 0; + + //! \} + + //! \name Comment + //! \{ + + //! Emits a comment stored in `data` with an optional `size` parameter. + virtual Error comment(const char* data, size_t size = SIZE_MAX) = 0; + + //! Emits a formatted comment specified by `fmt` and variable number of arguments. + ASMJIT_API Error commentf(const char* fmt, ...); + //! Emits a formatted comment specified by `fmt` and `ap`. + ASMJIT_API Error commentv(const char* fmt, va_list ap); + + //! \} + + //! \name Events + //! \{ + + //! Called after the emitter was attached to `CodeHolder`. + virtual Error onAttach(CodeHolder* ASMJIT_NONNULL(code)) noexcept = 0; + //! Called after the emitter was detached from `CodeHolder`. + virtual Error onDetach(CodeHolder* ASMJIT_NONNULL(code)) noexcept = 0; + + //! Called when \ref CodeHolder has updated an important setting, which involves the following: + //! + //! - \ref Logger has been changed (\ref CodeHolder::setLogger() has been called). + //! + //! - \ref ErrorHandler has been changed (\ref CodeHolder::setErrorHandler() has been called). + //! + //! This function ensures that the settings are properly propagated from \ref CodeHolder to the emitter. + //! + //! \note This function is virtual and can be overridden, however, if you do so, always call \ref + //! BaseEmitter::onSettingsUpdated() within your own implementation to ensure that the emitter is + //! in a consistent state. + ASMJIT_API virtual void onSettingsUpdated() noexcept; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_EMITTER_H_INCLUDED diff --git a/src/asmjit/core/emitterutils.cpp b/src/asmjit/core/emitterutils.cpp new file mode 100644 index 0000000..da12217 --- /dev/null +++ b/src/asmjit/core/emitterutils.cpp @@ -0,0 +1,129 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/assembler.h" +#include "../core/emitterutils_p.h" +#include "../core/formatter_p.h" +#include "../core/logger.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +namespace EmitterUtils { + +#ifndef ASMJIT_NO_LOGGING + +Error finishFormattedLine(String& sb, const FormatOptions& formatOptions, const uint8_t* binData, size_t binSize, size_t offsetSize, size_t immSize, const char* comment) noexcept { + ASMJIT_ASSERT(binSize >= offsetSize); + const size_t kNoBinSize = SIZE_MAX; + + size_t commentSize = comment ? Support::strLen(comment, Globals::kMaxCommentSize) : 0; + + if ((binSize != 0 && binSize != kNoBinSize) || commentSize) { + char sep = ';'; + size_t padding = Formatter::paddingFromOptions(formatOptions, FormatPaddingGroup::kRegularLine); + + for (size_t i = (binSize == kNoBinSize); i < 2; i++) { + ASMJIT_PROPAGATE(sb.padEnd(padding)); + + if (sep) { + ASMJIT_PROPAGATE(sb.append(sep)); + ASMJIT_PROPAGATE(sb.append(' ')); + } + + // Append binary data or comment. + if (i == 0) { + ASMJIT_PROPAGATE(sb.appendHex(binData, binSize - offsetSize - immSize)); + ASMJIT_PROPAGATE(sb.appendChars('.', offsetSize * 2)); + ASMJIT_PROPAGATE(sb.appendHex(binData + binSize - immSize, immSize)); + if (commentSize == 0) break; + } + else { + ASMJIT_PROPAGATE(sb.append(comment, commentSize)); + } + + sep = '|'; + padding += Formatter::paddingFromOptions(formatOptions, FormatPaddingGroup::kMachineCode); + } + } + + return sb.append('\n'); +} + +void logLabelBound(BaseAssembler* self, const Label& label) noexcept { + Logger* logger = self->logger(); + + StringTmp<512> sb; + size_t binSize = logger->hasFlag(FormatFlags::kMachineCode) ? size_t(0) : SIZE_MAX; + + sb.appendChars(' ', logger->indentation(FormatIndentationGroup::kLabel)); + Formatter::formatLabel(sb, logger->flags(), self, label.id()); + sb.append(':'); + finishFormattedLine(sb, logger->options(), nullptr, binSize, 0, 0, self->_inlineComment); + logger->log(sb.data(), sb.size()); +} + +void logInstructionEmitted( + BaseAssembler* self, + InstId instId, + InstOptions options, + const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt, + uint32_t relSize, uint32_t immSize, uint8_t* afterCursor) { + + Logger* logger = self->logger(); + ASMJIT_ASSERT(logger != nullptr); + + StringTmp<256> sb; + FormatFlags formatFlags = logger->flags(); + + uint8_t* beforeCursor = self->bufferPtr(); + intptr_t emittedSize = (intptr_t)(afterCursor - beforeCursor); + + Operand_ opArray[Globals::kMaxOpCount]; + opArrayFromEmitArgs(opArray, o0, o1, o2, opExt); + + sb.appendChars(' ', logger->indentation(FormatIndentationGroup::kCode)); + self->_funcs.formatInstruction(sb, formatFlags, self, self->arch(), BaseInst(instId, options, self->extraReg()), opArray, Globals::kMaxOpCount); + + if (Support::test(formatFlags, FormatFlags::kMachineCode)) + finishFormattedLine(sb, logger->options(), self->bufferPtr(), size_t(emittedSize), relSize, immSize, self->inlineComment()); + else + finishFormattedLine(sb, logger->options(), nullptr, SIZE_MAX, 0, 0, self->inlineComment()); + logger->log(sb); +} + +Error logInstructionFailed( + BaseEmitter* self, + Error err, + InstId instId, + InstOptions options, + const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) { + + StringTmp<256> sb; + sb.append(DebugUtils::errorAsString(err)); + sb.append(": "); + + Operand_ opArray[Globals::kMaxOpCount]; + opArrayFromEmitArgs(opArray, o0, o1, o2, opExt); + + self->_funcs.formatInstruction(sb, FormatFlags::kRegType, self, self->arch(), BaseInst(instId, options, self->extraReg()), opArray, Globals::kMaxOpCount); + + if (self->inlineComment()) { + sb.append(" ; "); + sb.append(self->inlineComment()); + } + + self->resetInstOptions(); + self->resetExtraReg(); + self->resetInlineComment(); + return self->reportError(err, sb.data()); +} + +#endif + +} // {EmitterUtils} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/emitterutils_p.h b/src/asmjit/core/emitterutils_p.h new file mode 100644 index 0000000..2f7370a --- /dev/null +++ b/src/asmjit/core/emitterutils_p.h @@ -0,0 +1,89 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_EMITTERUTILS_P_H_INCLUDED +#define ASMJIT_CORE_EMITTERUTILS_P_H_INCLUDED + +#include "../core/emitter.h" +#include "../core/operand.h" + +ASMJIT_BEGIN_NAMESPACE + +class BaseAssembler; +class FormatOptions; + +//! \cond INTERNAL +//! \addtogroup asmjit_core +//! \{ + +//! Utilities used by various emitters, mostly Assembler implementations. +namespace EmitterUtils { + +//! Default paddings used by Emitter utils and Formatter. + +static constexpr Operand noExt[3]; + +enum kOpIndex : uint32_t { + kOp3 = 0, + kOp4 = 1, + kOp5 = 2 +}; + +static ASMJIT_FORCE_INLINE uint32_t opCountFromEmitArgs(const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { + uint32_t opCount = 0; + + if (opExt[kOp3].isNone()) { + if (!o0.isNone()) opCount = 1; + if (!o1.isNone()) opCount = 2; + if (!o2.isNone()) opCount = 3; + } + else { + opCount = 4; + if (!opExt[kOp4].isNone()) { + opCount = 5 + uint32_t(!opExt[kOp5].isNone()); + } + } + + return opCount; +} + +static ASMJIT_FORCE_INLINE void opArrayFromEmitArgs(Operand_ dst[Globals::kMaxOpCount], const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt) noexcept { + dst[0].copyFrom(o0); + dst[1].copyFrom(o1); + dst[2].copyFrom(o2); + dst[3].copyFrom(opExt[kOp3]); + dst[4].copyFrom(opExt[kOp4]); + dst[5].copyFrom(opExt[kOp5]); +} + +#ifndef ASMJIT_NO_LOGGING +Error finishFormattedLine(String& sb, const FormatOptions& formatOptions, const uint8_t* binData, size_t binSize, size_t offsetSize, size_t immSize, const char* comment) noexcept; + +void logLabelBound(BaseAssembler* self, const Label& label) noexcept; + +void logInstructionEmitted( + BaseAssembler* self, + InstId instId, + InstOptions options, + const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt, + uint32_t relSize, uint32_t immSize, uint8_t* afterCursor); + +Error logInstructionFailed( + BaseEmitter* self, + Error err, + InstId instId, + InstOptions options, + const Operand_& o0, const Operand_& o1, const Operand_& o2, const Operand_* opExt); +#endif + +} + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_EMITTERUTILS_P_H_INCLUDED + diff --git a/src/asmjit/core/environment.cpp b/src/asmjit/core/environment.cpp new file mode 100644 index 0000000..9a694af --- /dev/null +++ b/src/asmjit/core/environment.cpp @@ -0,0 +1,46 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/environment.h" + +ASMJIT_BEGIN_NAMESPACE + +// X86 Target +// ---------- +// +// - 32-bit - Linux, OSX, BSD, and apparently also Haiku guarantee 16-byte +// stack alignment. Other operating systems are assumed to have +// 4-byte alignment by default for safety reasons. +// - 64-bit - stack must be aligned to 16 bytes. +// +// ARM Target +// ---------- +// +// - 32-bit - Stack must be aligned to 8 bytes. +// - 64-bit - Stack must be aligned to 16 bytes (hardware requirement). +uint32_t Environment::stackAlignment() const noexcept { + if (is64Bit()) { + // Assume 16-byte alignment on any 64-bit target. + return 16; + } + else { + // The following platforms use 16-byte alignment in 32-bit mode. + if (isPlatformLinux() || + isPlatformBSD() || + isPlatformApple() || + isPlatformHaiku()) { + return 16u; + } + + if (isFamilyARM()) + return 8; + + // Bail to 4-byte alignment if we don't know. + return 4; + } +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/environment.h b/src/asmjit/core/environment.h new file mode 100644 index 0000000..7e328a9 --- /dev/null +++ b/src/asmjit/core/environment.h @@ -0,0 +1,508 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_ENVIRONMENT_H_INCLUDED +#define ASMJIT_CORE_ENVIRONMENT_H_INCLUDED + +#include "../core/archtraits.h" + +#if defined(__APPLE__) + #include +#endif + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_core +//! \{ + +//! Vendor. +//! +//! \note AsmJit doesn't use vendor information at the moment. It's provided for future use, if required. +enum class Vendor : uint8_t { + //! Unknown or uninitialized platform vendor. + kUnknown = 0, + + //! Maximum value of `PlatformVendor`. + kMaxValue = kUnknown, + + //! Platform vendor detected at compile-time. + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#else + kUnknown +#endif +}; + +//! Platform - runtime environment or operating system. +enum class Platform : uint8_t { + //! Unknown or uninitialized platform. + kUnknown = 0, + + //! Windows OS. + kWindows, + + //! Other platform that is not Windows, most likely POSIX based. + kOther, + + //! Linux OS. + kLinux, + //! GNU/Hurd OS. + kHurd, + + //! FreeBSD OS. + kFreeBSD, + //! OpenBSD OS. + kOpenBSD, + //! NetBSD OS. + kNetBSD, + //! DragonFly BSD OS. + kDragonFlyBSD, + + //! Haiku OS. + kHaiku, + + //! Apple OSX. + kOSX, + //! Apple iOS. + kIOS, + //! Apple TVOS. + kTVOS, + //! Apple WatchOS. + kWatchOS, + + //! Emscripten platform. + kEmscripten, + + //! Maximum value of `Platform`. + kMaxValue = kEmscripten, + + //! Platform detected at compile-time (platform of the host). + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#elif defined(__EMSCRIPTEN__) + kEmscripten +#elif defined(_WIN32) + kWindows +#elif defined(__linux__) + kLinux +#elif defined(__gnu_hurd__) + kHurd +#elif defined(__FreeBSD__) + kFreeBSD +#elif defined(__OpenBSD__) + kOpenBSD +#elif defined(__NetBSD__) + kNetBSD +#elif defined(__DragonFly__) + kDragonFlyBSD +#elif defined(__HAIKU__) + kHaiku +#elif defined(__APPLE__) && TARGET_OS_OSX + kOSX +#elif defined(__APPLE__) && TARGET_OS_TV + kTVOS +#elif defined(__APPLE__) && TARGET_OS_WATCH + kWatchOS +#elif defined(__APPLE__) && TARGET_OS_IPHONE + kIOS +#else + kOther +#endif +}; + +//! Platform ABI (application binary interface). +enum class PlatformABI : uint8_t { + //! Unknown or uninitialied environment. + kUnknown = 0, + //! Microsoft ABI. + kMSVC, + //! GNU ABI. + kGNU, + //! Android Environment / ABI. + kAndroid, + //! Cygwin ABI. + kCygwin, + + //! Maximum value of `PlatformABI`. + kMaxValue, + + //! Host ABI detected at compile-time. + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#elif defined(_MSC_VER) + kMSVC +#elif defined(__CYGWIN__) + kCygwin +#elif defined(__MINGW32__) || defined(__GLIBC__) + kGNU +#elif defined(__ANDROID__) + kAndroid +#else + kUnknown +#endif +}; + +//! Object format. +//! +//! \note AsmJit doesn't really use anything except \ref ObjectFormat::kUnknown and \ref ObjectFormat::kJIT at +//! the moment. Object file formats are provided for future extensibility and a possibility to generate object +//! files at some point. +enum class ObjectFormat : uint8_t { + //! Unknown or uninitialized object format. + kUnknown = 0, + + //! JIT code generation object, most likely \ref JitRuntime or a custom + //! \ref Target implementation. + kJIT, + + //! Executable and linkable format (ELF). + kELF, + //! Common object file format. + kCOFF, + //! Extended COFF object format. + kXCOFF, + //! Mach object file format. + kMachO, + + //! Maximum value of `ObjectFormat`. + kMaxValue +}; + +//! Represents an environment, which is usually related to a \ref Target. +//! +//! Environment has usually an 'arch-subarch-vendor-os-abi' format, which is sometimes called "Triple" (historically +//! it used to be 3 only parts) or "Tuple", which is a convention used by Debian Linux. +//! +//! AsmJit doesn't support all possible combinations or architectures and ABIs, however, it models the environment +//! similarly to other compilers for future extensibility. +class Environment { +public: + //! \name Members + //! \{ + + //! Architecture. + Arch _arch; + //! Sub-architecture type. + SubArch _subArch; + //! Vendor type. + Vendor _vendor; + //! Platform. + Platform _platform; + //! Platform ABI. + PlatformABI _platformABI; + //! Object format. + ObjectFormat _objectFormat; + //! Reserved for future use, must be zero. + uint8_t _reserved[2]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline Environment() noexcept : + _arch(Arch::kUnknown), + _subArch(SubArch::kUnknown), + _vendor(Vendor::kUnknown), + _platform(Platform::kUnknown), + _platformABI(PlatformABI::kUnknown), + _objectFormat(ObjectFormat::kUnknown), + _reserved { 0, 0 } {} + + inline explicit Environment( + Arch arch, + SubArch subArch = SubArch::kUnknown, + Vendor vendor = Vendor::kUnknown, + Platform platform = Platform::kUnknown, + PlatformABI abi = PlatformABI::kUnknown, + ObjectFormat objectFormat = ObjectFormat::kUnknown) noexcept { + + init(arch, subArch, vendor, platform, abi, objectFormat); + } + + inline Environment(const Environment& other) noexcept = default; + + //! Returns the host environment constructed from preprocessor macros defined by the compiler. + //! + //! The returned environment should precisely match the target host architecture, sub-architecture, platform, + //! and ABI. + static inline Environment host() noexcept { + return Environment(Arch::kHost, SubArch::kHost, Vendor::kHost, Platform::kHost, PlatformABI::kHost, ObjectFormat::kUnknown); + } + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline Environment& operator=(const Environment& other) noexcept = default; + + inline bool operator==(const Environment& other) const noexcept { return equals(other); } + inline bool operator!=(const Environment& other) const noexcept { return !equals(other); } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the environment is not set up. + //! + //! Returns true if all members are zero, and thus unknown. + inline bool empty() const noexcept { + // Unfortunately compilers won't optimize fields are checked one by one... + return _packed() == 0; + } + + //! Tests whether the environment is initialized, which means it must have + //! a valid architecture. + inline bool isInitialized() const noexcept { + return _arch != Arch::kUnknown; + } + + inline uint64_t _packed() const noexcept { + uint64_t x; + memcpy(&x, this, 8); + return x; + } + + //! Resets all members of the environment to zero / unknown. + inline void reset() noexcept { + _arch = Arch::kUnknown; + _subArch = SubArch::kUnknown; + _vendor = Vendor::kUnknown; + _platform = Platform::kUnknown; + _platformABI = PlatformABI::kUnknown; + _objectFormat = ObjectFormat::kUnknown; + _reserved[0] = 0; + _reserved[1] = 0; + } + + inline bool equals(const Environment& other) const noexcept { + return _packed() == other._packed(); + } + + //! Returns the architecture. + inline Arch arch() const noexcept { return _arch; } + //! Returns the sub-architecture. + inline SubArch subArch() const noexcept { return _subArch; } + //! Returns vendor. + inline Vendor vendor() const noexcept { return _vendor; } + //! Returns target's platform or operating system. + inline Platform platform() const noexcept { return _platform; } + //! Returns target's ABI. + inline PlatformABI platformABI() const noexcept { return _platformABI; } + //! Returns target's object format. + inline ObjectFormat objectFormat() const noexcept { return _objectFormat; } + + inline void init( + Arch arch, + SubArch subArch = SubArch::kUnknown, + Vendor vendor = Vendor::kUnknown, + Platform platform = Platform::kUnknown, + PlatformABI platformABI = PlatformABI::kUnknown, + ObjectFormat objectFormat = ObjectFormat::kUnknown) noexcept { + + _arch = arch; + _subArch = subArch; + _vendor = vendor; + _platform = platform; + _platformABI = platformABI; + _objectFormat = objectFormat; + _reserved[0] = 0; + _reserved[1] = 0; + } + + inline bool isArchX86() const noexcept { return _arch == Arch::kX86; } + inline bool isArchX64() const noexcept { return _arch == Arch::kX64; } + inline bool isArchARM() const noexcept { return isArchARM(_arch); } + inline bool isArchThumb() const noexcept { return isArchThumb(_arch); } + inline bool isArchAArch64() const noexcept { return isArchAArch64(_arch); } + inline bool isArchMIPS32() const noexcept { return isArchMIPS32(_arch); } + inline bool isArchMIPS64() const noexcept { return isArchMIPS64(_arch); } + inline bool isArchRISCV32() const noexcept { return _arch == Arch::kRISCV32; } + inline bool isArchRISCV64() const noexcept { return _arch == Arch::kRISCV64; } + + //! Tests whether the architecture is 32-bit. + inline bool is32Bit() const noexcept { return is32Bit(_arch); } + //! Tests whether the architecture is 64-bit. + inline bool is64Bit() const noexcept { return is64Bit(_arch); } + + //! Tests whether the architecture is little endian. + inline bool isLittleEndian() const noexcept { return isLittleEndian(_arch); } + //! Tests whether the architecture is big endian. + inline bool isBigEndian() const noexcept { return isBigEndian(_arch); } + + //! Tests whether this architecture is of X86 family. + inline bool isFamilyX86() const noexcept { return isFamilyX86(_arch); } + //! Tests whether this architecture family is ARM, THUMB, or AArch64. + inline bool isFamilyARM() const noexcept { return isFamilyARM(_arch); } + //! Tests whether this architecture family is AArch32 (ARM or THUMB). + inline bool isFamilyAArch32() const noexcept { return isFamilyAArch32(_arch); } + //! Tests whether this architecture family is AArch64. + inline bool isFamilyAArch64() const noexcept { return isFamilyAArch64(_arch); } + //! Tests whether this architecture family is MISP or MIPS64. + inline bool isFamilyMIPS() const noexcept { return isFamilyMIPS(_arch); } + //! Tests whether this architecture family is RISC-V (both 32-bit and 64-bit). + inline bool isFamilyRISCV() const noexcept { return isFamilyRISCV(_arch); } + + //! Tests whether the environment platform is Windows. + inline bool isPlatformWindows() const noexcept { return _platform == Platform::kWindows; } + + //! Tests whether the environment platform is Linux. + inline bool isPlatformLinux() const noexcept { return _platform == Platform::kLinux; } + + //! Tests whether the environment platform is Hurd. + inline bool isPlatformHurd() const noexcept { return _platform == Platform::kHurd; } + + //! Tests whether the environment platform is Haiku. + inline bool isPlatformHaiku() const noexcept { return _platform == Platform::kHaiku; } + + //! Tests whether the environment platform is any BSD. + inline bool isPlatformBSD() const noexcept { + return _platform == Platform::kFreeBSD || + _platform == Platform::kOpenBSD || + _platform == Platform::kNetBSD || + _platform == Platform::kDragonFlyBSD; + } + + //! Tests whether the environment platform is any Apple platform (OSX, iOS, TVOS, WatchOS). + inline bool isPlatformApple() const noexcept { + return _platform == Platform::kOSX || + _platform == Platform::kIOS || + _platform == Platform::kTVOS || + _platform == Platform::kWatchOS; + } + + //! Tests whether the ABI is MSVC. + inline bool isMSVC() const noexcept { return _platformABI == PlatformABI::kMSVC; } + //! Tests whether the ABI is GNU. + inline bool isGNU() const noexcept { return _platformABI == PlatformABI::kGNU; } + + //! Returns a calculated stack alignment for this environment. + ASMJIT_API uint32_t stackAlignment() const noexcept; + + //! Returns a native register size of this architecture. + uint32_t registerSize() const noexcept { return registerSizeFromArch(_arch); } + + //! Sets the architecture to `arch`. + inline void setArch(Arch arch) noexcept { _arch = arch; } + //! Sets the sub-architecture to `subArch`. + inline void setSubArch(SubArch subArch) noexcept { _subArch = subArch; } + //! Sets the vendor to `vendor`. + inline void setVendor(Vendor vendor) noexcept { _vendor = vendor; } + //! Sets the platform to `platform`. + inline void setPlatform(Platform platform) noexcept { _platform = platform; } + //! Sets the ABI to `platformABI`. + inline void setPlatformABI(PlatformABI platformABI) noexcept { _platformABI = platformABI; } + //! Sets the object format to `objectFormat`. + inline void setObjectFormat(ObjectFormat objectFormat) noexcept { _objectFormat = objectFormat; } + + //! \} + + //! \name Static Utilities + //! \{ + + static inline bool isDefinedArch(Arch arch) noexcept { + return uint32_t(arch) <= uint32_t(Arch::kMaxValue); + } + + static inline bool isValidArch(Arch arch) noexcept { + return arch != Arch::kUnknown && uint32_t(arch) <= uint32_t(Arch::kMaxValue); + } + + //! Tests whether the given architecture `arch` is 32-bit. + static inline bool is32Bit(Arch arch) noexcept { + return (uint32_t(arch) & uint32_t(Arch::k32BitMask)) == uint32_t(Arch::k32BitMask); + } + + //! Tests whether the given architecture `arch` is 64-bit. + static inline bool is64Bit(Arch arch) noexcept { + return (uint32_t(arch) & uint32_t(Arch::k32BitMask)) == 0; + } + + //! Tests whether the given architecture `arch` is little endian. + static inline bool isLittleEndian(Arch arch) noexcept { + return uint32_t(arch) < uint32_t(Arch::kBigEndian); + } + + //! Tests whether the given architecture `arch` is big endian. + static inline bool isBigEndian(Arch arch) noexcept { + return uint32_t(arch) >= uint32_t(Arch::kBigEndian); + } + + //! Tests whether the given architecture is Thumb or Thumb_BE. + static inline bool isArchThumb(Arch arch) noexcept { + return arch == Arch::kThumb || arch == Arch::kThumb_BE; + } + + //! Tests whether the given architecture is ARM or ARM_BE. + static inline bool isArchARM(Arch arch) noexcept { + return arch == Arch::kARM || arch == Arch::kARM_BE; + } + + //! Tests whether the given architecture is AArch64 or AArch64_BE. + static inline bool isArchAArch64(Arch arch) noexcept { + return arch == Arch::kAArch64 || arch == Arch::kAArch64_BE; + } + + //! Tests whether the given architecture is MIPS32_LE or MIPS32_BE. + static inline bool isArchMIPS32(Arch arch) noexcept { + return arch == Arch::kMIPS32_LE || arch == Arch::kMIPS32_BE; + } + + //! Tests whether the given architecture is MIPS64_LE or MIPS64_BE. + static inline bool isArchMIPS64(Arch arch) noexcept { + return arch == Arch::kMIPS64_LE || arch == Arch::kMIPS64_BE; + } + + //! Tests whether the given architecture family is X86 or X64. + static inline bool isFamilyX86(Arch arch) noexcept { + return arch == Arch::kX86 || arch == Arch::kX64; + } + + //! Tests whether the given architecture family is ARM, THUMB, or AArch64. + static inline bool isFamilyARM(Arch arch) noexcept { + return isArchARM(arch) || isArchAArch64(arch) || isArchThumb(arch); + } + + //! Tests whether the given architecture family is AArch32 (ARM or THUMB). + static inline bool isFamilyAArch32(Arch arch) noexcept { + return isArchARM(arch) || isArchThumb(arch); + } + + //! Tests whether the given architecture family is AArch64. + static inline bool isFamilyAArch64(Arch arch) noexcept { + return isArchAArch64(arch); + } + + //! Tests whether the given architecture family is MISP or MIPS64. + static inline bool isFamilyMIPS(Arch arch) noexcept { + return isArchMIPS32(arch) || isArchMIPS64(arch); + } + + //! Tests whether the given architecture family is RISC-V (both 32-bit and 64-bit). + static inline bool isFamilyRISCV(Arch arch) noexcept { + return arch == Arch::kRISCV32 || arch == Arch::kRISCV64; + } + + //! Returns a native general purpose register size from the given architecture. + static inline uint32_t registerSizeFromArch(Arch arch) noexcept { + return is32Bit(arch) ? 4u : 8u; + } + + //! \} +}; + +static_assert(sizeof(Environment) == 8, + "Environment must occupy exactly 8 bytes."); + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_ENVIRONMENT_H_INCLUDED diff --git a/src/asmjit/core/errorhandler.cpp b/src/asmjit/core/errorhandler.cpp new file mode 100644 index 0000000..5a7dac5 --- /dev/null +++ b/src/asmjit/core/errorhandler.cpp @@ -0,0 +1,14 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/errorhandler.h" + +ASMJIT_BEGIN_NAMESPACE + +ErrorHandler::ErrorHandler() noexcept {} +ErrorHandler::~ErrorHandler() noexcept {} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/errorhandler.h b/src/asmjit/core/errorhandler.h new file mode 100644 index 0000000..5151d43 --- /dev/null +++ b/src/asmjit/core/errorhandler.h @@ -0,0 +1,228 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_ERRORHANDLER_H_INCLUDED +#define ASMJIT_CORE_ERRORHANDLER_H_INCLUDED + +#include "../core/globals.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_error_handling +//! \{ + +class BaseEmitter; + +//! Error handler can be used to override the default behavior of error handling. +//! +//! It's available to all classes that inherit `BaseEmitter`. Override \ref ErrorHandler::handleError() to implement +//! your own error handler. +//! +//! The following use-cases are supported: +//! +//! - Record the error and continue code generation. This is the simplest approach that can be used to at least log +//! possible errors. +//! - Throw an exception. AsmJit doesn't use exceptions and is completely exception-safe, but it's perfectly legal +//! to throw an exception from the error handler. +//! - Use plain old C's `setjmp()` and `longjmp()`. Asmjit always puts Assembler, Builder and Compiler to +//! a consistent state before calling \ref handleError(), so `longjmp()` can be used without issues to cancel the +//! code generation if an error occurred. This method can be used if exception handling in your project is turned +//! off and you still want some comfort. In most cases it should be safe as AsmJit uses \ref Zone memory and the +//! ownership of memory it allocates always ends with the instance that allocated it. If using this approach please +//! never jump outside the life-time of \ref CodeHolder and \ref BaseEmitter. +//! +//! \ref ErrorHandler can be attached to \ref CodeHolder or \ref BaseEmitter, which has a priority. The example below +//! uses error handler that just prints the error, but lets AsmJit continue: +//! +//! ``` +//! // Error Handling #1 - Logging and returning Error. +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! // Error handler that just prints the error and lets AsmJit ignore it. +//! class SimpleErrorHandler : public ErrorHandler { +//! public: +//! Error err; +//! +//! inline SimpleErrorHandler() : err(kErrorOk) {} +//! +//! void handleError(Error err, const char* message, BaseEmitter* origin) override { +//! this->err = err; +//! fprintf(stderr, "ERROR: %s\n", message); +//! } +//! }; +//! +//! int main() { +//! JitRuntime rt; +//! SimpleErrorHandler eh; +//! +//! CodeHolder code; +//! code.init(rt.environment()); +//! code.setErrorHandler(&eh); +//! +//! // Try to emit instruction that doesn't exist. +//! x86::Assembler a(&code); +//! a.emit(x86::Inst::kIdMov, x86::xmm0, x86::xmm1); +//! +//! if (eh.err) { +//! // Assembler failed! +//! return 1; +//! } +//! +//! return 0; +//! } +//! ``` +//! +//! If error happens during instruction emitting / encoding the assembler behaves transactionally - the output buffer +//! won't advance if encoding failed, thus either a fully encoded instruction or nothing is emitted. The error handling +//! shown above is useful, but it's still not the best way of dealing with errors in AsmJit. The following example +//! shows how to use exception handling to handle errors in a more C++ way: +//! +//! ``` +//! // Error Handling #2 - Throwing an exception. +//! #include +//! #include +//! #include +//! #include +//! +//! using namespace asmjit; +//! +//! // Error handler that throws a user-defined `AsmJitException`. +//! class AsmJitException : public std::exception { +//! public: +//! Error err; +//! std::string message; +//! +//! AsmJitException(Error err, const char* message) noexcept +//! : err(err), +//! message(message) {} +//! +//! const char* what() const noexcept override { return message.c_str(); } +//! }; +//! +//! class ThrowableErrorHandler : public ErrorHandler { +//! public: +//! // Throw is possible, functions that use ErrorHandler are never 'noexcept'. +//! void handleError(Error err, const char* message, BaseEmitter* origin) override { +//! throw AsmJitException(err, message); +//! } +//! }; +//! +//! int main() { +//! JitRuntime rt; +//! ThrowableErrorHandler eh; +//! +//! CodeHolder code; +//! code.init(rt.environment()); +//! code.setErrorHandler(&eh); +//! +//! x86::Assembler a(&code); +//! +//! // Try to emit instruction that doesn't exist. +//! try { +//! a.emit(x86::Inst::kIdMov, x86::xmm0, x86::xmm1); +//! } +//! catch (const AsmJitException& ex) { +//! printf("EXCEPTION THROWN: %s\n", ex.what()); +//! return 1; +//! } +//! +//! return 0; +//! } +//! ``` +//! +//! If C++ exceptions are not what you like or your project turns off them completely there is still a way of reducing +//! the error handling to a minimum by using a standard setjmp/longjmp approach. AsmJit is exception-safe and cleans +//! up everything before calling the ErrorHandler, so any approach is safe. You can simply jump from the error handler +//! without causing any side-effects or memory leaks. The following example demonstrates how it could be done: +//! +//! ``` +//! // Error Handling #3 - Using setjmp/longjmp if exceptions are not allowed. +//! #include +//! #include +//! #include +//! +//! class LongJmpErrorHandler : public asmjit::ErrorHandler { +//! public: +//! inline LongJmpErrorHandler() : err(asmjit::kErrorOk) {} +//! +//! void handleError(asmjit::Error err, const char* message, asmjit::BaseEmitter* origin) override { +//! this->err = err; +//! longjmp(state, 1); +//! } +//! +//! jmp_buf state; +//! asmjit::Error err; +//! }; +//! +//! int main(int argc, char* argv[]) { +//! using namespace asmjit; +//! +//! JitRuntime rt; +//! LongJmpErrorHandler eh; +//! +//! CodeHolder code; +//! code.init(rt.rt.environment()); +//! code.setErrorHandler(&eh); +//! +//! x86::Assembler a(&code); +//! +//! if (!setjmp(eh.state)) { +//! // Try to emit instruction that doesn't exist. +//! a.emit(x86::Inst::kIdMov, x86::xmm0, x86::xmm1); +//! } +//! else { +//! Error err = eh.err; +//! printf("ASMJIT ERROR: 0x%08X [%s]\n", err, DebugUtils::errorAsString(err)); +//! } +//! +//! return 0; +//! } +//! ``` +class ASMJIT_VIRTAPI ErrorHandler { +public: + ASMJIT_BASE_CLASS(ErrorHandler) + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `ErrorHandler` instance. + ASMJIT_API ErrorHandler() noexcept; + //! Destroys the `ErrorHandler` instance. + ASMJIT_API virtual ~ErrorHandler() noexcept; + + //! \} + + //! \name Interface + //! \{ + + //! Error handler (must be reimplemented). + //! + //! Error handler is called after an error happened and before it's propagated to the caller. There are multiple + //! ways how the error handler can be used: + //! + //! 1. User-based error handling without throwing exception or using C's`longjmp()`. This is for users that don't + //! use exceptions and want customized error handling. + //! + //! 2. Throwing an exception. AsmJit doesn't use exceptions and is completely exception-safe, but you can throw + //! exception from your error handler if this way is the preferred way of handling errors in your project. + //! + //! 3. Using plain old C's `setjmp()` and `longjmp()`. Asmjit always puts `BaseEmitter` to a consistent state before + //! calling `handleError()` so `longjmp()` can be used without any issues to cancel the code generation if an + //! error occurred. There is no difference between exceptions and `longjmp()` from AsmJit's perspective, however, + //! never jump outside of `CodeHolder` and `BaseEmitter` scope as you would leak memory. + virtual void handleError(Error err, const char* message, BaseEmitter* origin) = 0; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_ERRORHANDLER_H_INCLUDED + diff --git a/src/asmjit/core/formatter.cpp b/src/asmjit/core/formatter.cpp new file mode 100644 index 0000000..1a9a386 --- /dev/null +++ b/src/asmjit/core/formatter.cpp @@ -0,0 +1,584 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_LOGGING + +#include "../core/archtraits.h" +#include "../core/builder.h" +#include "../core/codeholder.h" +#include "../core/compiler.h" +#include "../core/emitter.h" +#include "../core/formatter_p.h" +#include "../core/string.h" +#include "../core/support.h" +#include "../core/type.h" + +#if !defined(ASMJIT_NO_X86) + #include "../x86/x86formatter_p.h" +#endif + +#if !defined(ASMJIT_NO_AARCH64) + #include "../arm/a64formatter_p.h" +#endif + +ASMJIT_BEGIN_NAMESPACE + +#if defined(ASMJIT_NO_COMPILER) +class VirtReg; +#endif + +namespace Formatter { + +static const char wordNameTable[][8] = { + "db", + "dw", + "dd", + "dq", + "byte", + "half", + "word", + "hword", + "dword", + "qword", + "xword", + "short", + "long", + "quad" +}; + + +Error formatTypeId(String& sb, TypeId typeId) noexcept { + if (typeId == TypeId::kVoid) + return sb.append("void"); + + if (!TypeUtils::isValid(typeId)) + return sb.append("unknown"); + + const char* typeName = "unknown"; + uint32_t typeSize = TypeUtils::sizeOf(typeId); + TypeId scalarType = TypeUtils::scalarOf(typeId); + + switch (scalarType) { + case TypeId::kIntPtr : typeName = "intptr" ; break; + case TypeId::kUIntPtr: typeName = "uintptr"; break; + case TypeId::kInt8 : typeName = "int8" ; break; + case TypeId::kUInt8 : typeName = "uint8" ; break; + case TypeId::kInt16 : typeName = "int16" ; break; + case TypeId::kUInt16 : typeName = "uint16" ; break; + case TypeId::kInt32 : typeName = "int32" ; break; + case TypeId::kUInt32 : typeName = "uint32" ; break; + case TypeId::kInt64 : typeName = "int64" ; break; + case TypeId::kUInt64 : typeName = "uint64" ; break; + case TypeId::kFloat32: typeName = "float32"; break; + case TypeId::kFloat64: typeName = "float64"; break; + case TypeId::kFloat80: typeName = "float80"; break; + case TypeId::kMask8 : typeName = "mask8" ; break; + case TypeId::kMask16 : typeName = "mask16" ; break; + case TypeId::kMask32 : typeName = "mask32" ; break; + case TypeId::kMask64 : typeName = "mask64" ; break; + case TypeId::kMmx32 : typeName = "mmx32" ; break; + case TypeId::kMmx64 : typeName = "mmx64" ; break; + + default: + typeName = "unknown"; + break; + } + + uint32_t baseSize = TypeUtils::sizeOf(scalarType); + if (typeSize > baseSize) { + uint32_t count = typeSize / baseSize; + return sb.appendFormat("%sx%u", typeName, unsigned(count)); + } + else { + return sb.append(typeName); + } +} + +Error formatFeature( + String& sb, + Arch arch, + uint32_t featureId) noexcept { + +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::FormatterInternal::formatFeature(sb, featureId); +#endif + +#if !defined(ASMJIT_NO_AARCH32) && !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyARM(arch)) + return arm::FormatterInternal::formatFeature(sb, featureId); +#endif + + return kErrorInvalidArch; +} + +Error formatLabel( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + uint32_t labelId) noexcept { + + DebugUtils::unused(formatFlags); + + const LabelEntry* le = emitter->code()->labelEntry(labelId); + if (ASMJIT_UNLIKELY(!le)) + return sb.appendFormat("", labelId); + + if (le->hasName()) { + if (le->hasParent()) { + uint32_t parentId = le->parentId(); + const LabelEntry* pe = emitter->code()->labelEntry(parentId); + + if (ASMJIT_UNLIKELY(!pe)) + ASMJIT_PROPAGATE(sb.appendFormat("", labelId)); + else if (ASMJIT_UNLIKELY(!pe->hasName())) + ASMJIT_PROPAGATE(sb.appendFormat("L%u", parentId)); + else + ASMJIT_PROPAGATE(sb.append(pe->name())); + + ASMJIT_PROPAGATE(sb.append('.')); + } + + if (le->type() == LabelType::kAnonymous) + ASMJIT_PROPAGATE(sb.appendFormat("L%u@", labelId)); + return sb.append(le->name()); + } + else { + return sb.appendFormat("L%u", labelId); + } +} + +Error formatRegister( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + RegType regType, + uint32_t regId) noexcept { + +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::FormatterInternal::formatRegister(sb, formatFlags, emitter, arch, regType, regId); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::FormatterInternal::formatRegister(sb, formatFlags, emitter, arch, regType, regId); +#endif + + return kErrorInvalidArch; +} + +Error formatOperand( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + const Operand_& op) noexcept { + +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::FormatterInternal::formatOperand(sb, formatFlags, emitter, arch, op); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::FormatterInternal::formatOperand(sb, formatFlags, emitter, arch, op); +#endif + + return kErrorInvalidArch; +} + +ASMJIT_API Error formatDataType( + String& sb, + FormatFlags formatFlags, + Arch arch, + TypeId typeId) noexcept +{ + DebugUtils::unused(formatFlags); + + if (ASMJIT_UNLIKELY(uint32_t(arch) > uint32_t(Arch::kMaxValue))) + return DebugUtils::errored(kErrorInvalidArch); + + uint32_t typeSize = TypeUtils::sizeOf(typeId); + if (typeSize == 0 || typeSize > 8) + return DebugUtils::errored(kErrorInvalidState); + + uint32_t typeSizeLog2 = Support::ctz(typeSize); + return sb.append(wordNameTable[size_t(ArchTraits::byArch(arch).typeNameIdByIndex(typeSizeLog2))]); +} + +static Error formatDataHelper(String& sb, const char* typeName, uint32_t typeSize, const uint8_t* data, size_t itemCount) noexcept { + sb.append('.'); + sb.append(typeName); + sb.append(' '); + + for (size_t i = 0; i < itemCount; i++) { + uint64_t v = 0; + + if (i != 0) + ASMJIT_PROPAGATE(sb.append(", ", 2)); + + switch (typeSize) { + case 1: v = data[0]; break; + case 2: v = Support::readU16u(data); break; + case 4: v = Support::readU32u(data); break; + case 8: v = Support::readU64u(data); break; + } + + ASMJIT_PROPAGATE(sb.appendUInt(v, 16, typeSize * 2, StringFormatFlags::kAlternate)); + data += typeSize; + } + + return kErrorOk; +} + +Error formatData( + String& sb, + FormatFlags formatFlags, + Arch arch, + TypeId typeId, const void* data, size_t itemCount, size_t repeatCount) noexcept +{ + DebugUtils::unused(formatFlags); + + if (ASMJIT_UNLIKELY(!Environment::isDefinedArch(arch))) + return DebugUtils::errored(kErrorInvalidArch); + + uint32_t typeSize = TypeUtils::sizeOf(typeId); + if (typeSize == 0) + return DebugUtils::errored(kErrorInvalidState); + + if (!Support::isPowerOf2(typeSize)) { + itemCount *= typeSize; + typeSize = 1; + } + + while (typeSize > 8u) { + typeSize >>= 1; + itemCount <<= 1; + } + + uint32_t typeSizeLog2 = Support::ctz(typeSize); + const char* wordName = wordNameTable[size_t(ArchTraits::byArch(arch).typeNameIdByIndex(typeSizeLog2))]; + + if (repeatCount > 1) + ASMJIT_PROPAGATE(sb.appendFormat(".repeat %zu ", repeatCount)); + + return formatDataHelper(sb, wordName, typeSize, static_cast(data), itemCount); +} + +Error formatInstruction( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept { + +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::FormatterInternal::formatInstruction(sb, formatFlags, emitter, arch, inst, operands, opCount); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyARM(arch)) + return a64::FormatterInternal::formatInstruction(sb, formatFlags, emitter, arch, inst, operands, opCount); +#endif + + return kErrorInvalidArch; +} + +#ifndef ASMJIT_NO_BUILDER + +#ifndef ASMJIT_NO_COMPILER +static Error formatFuncValue(String& sb, FormatFlags formatFlags, const BaseEmitter* emitter, FuncValue value) noexcept { + TypeId typeId = value.typeId(); + ASMJIT_PROPAGATE(formatTypeId(sb, typeId)); + + if (value.isAssigned()) { + ASMJIT_PROPAGATE(sb.append('@')); + + if (value.isIndirect()) + ASMJIT_PROPAGATE(sb.append('[')); + + // NOTE: It should be either reg or stack, but never both. We + // use two IFs on purpose so if the FuncValue is both it would + // show in logs. + if (value.isReg()) { + ASMJIT_PROPAGATE(formatRegister(sb, formatFlags, emitter, emitter->arch(), value.regType(), value.regId())); + } + + if (value.isStack()) { + ASMJIT_PROPAGATE(sb.appendFormat("[%d]", int(value.stackOffset()))); + } + + if (value.isIndirect()) + ASMJIT_PROPAGATE(sb.append(']')); + } + + return kErrorOk; +} + +static Error formatFuncValuePack( + String& sb, + FormatFlags formatFlags, + const BaseCompiler* cc, + const FuncValuePack& pack, + const RegOnly* vRegs) noexcept { + + size_t count = pack.count(); + if (!count) + return sb.append("void"); + + if (count > 1) + sb.append('['); + + for (uint32_t valueIndex = 0; valueIndex < count; valueIndex++) { + const FuncValue& value = pack[valueIndex]; + if (!value) + break; + + if (valueIndex) + ASMJIT_PROPAGATE(sb.append(", ")); + + ASMJIT_PROPAGATE(formatFuncValue(sb, formatFlags, cc, value)); + + if (vRegs) { + const VirtReg* virtReg = nullptr; + static const char nullReg[] = ""; + + if (vRegs[valueIndex].isReg() && cc->isVirtIdValid(vRegs[valueIndex].id())) + virtReg = cc->virtRegById(vRegs[valueIndex].id()); + + ASMJIT_PROPAGATE(sb.appendFormat(" %s", virtReg ? virtReg->name() : nullReg)); + } + } + + if (count > 1) + sb.append(']'); + + return kErrorOk; +} + +static Error formatFuncRets( + String& sb, + FormatFlags formatFlags, + const BaseCompiler* cc, + const FuncDetail& fd) noexcept { + + return formatFuncValuePack(sb, formatFlags, cc, fd.retPack(), nullptr); +} + +static Error formatFuncArgs( + String& sb, + FormatFlags formatFlags, + const BaseCompiler* cc, + const FuncDetail& fd, + const FuncNode::ArgPack* argPacks) noexcept { + + uint32_t argCount = fd.argCount(); + if (!argCount) + return sb.append("void"); + + for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) { + if (argIndex) + ASMJIT_PROPAGATE(sb.append(", ")); + + ASMJIT_PROPAGATE(formatFuncValuePack(sb, formatFlags, cc, fd.argPack(argIndex), argPacks[argIndex]._data)); + } + + return kErrorOk; +} +#endif + +Error formatNode( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder, + const BaseNode* node) noexcept { + + if (node->hasPosition() && formatOptions.hasFlag(FormatFlags::kPositions)) + ASMJIT_PROPAGATE(sb.appendFormat("<%05u> ", node->position())); + + size_t startLineIndex = sb.size(); + + switch (node->type()) { + case NodeType::kInst: + case NodeType::kJump: { + const InstNode* instNode = node->as(); + ASMJIT_PROPAGATE(builder->_funcs.formatInstruction(sb, formatOptions.flags(), builder, + builder->arch(), + instNode->baseInst(), instNode->operands(), instNode->opCount())); + break; + } + + case NodeType::kSection: { + const SectionNode* sectionNode = node->as(); + if (builder->_code->isSectionValid(sectionNode->id())) { + const Section* section = builder->_code->sectionById(sectionNode->id()); + ASMJIT_PROPAGATE(sb.appendFormat(".section %s", section->name())); + } + break; + } + + case NodeType::kLabel: { + const LabelNode* labelNode = node->as(); + ASMJIT_PROPAGATE(formatLabel(sb, formatOptions.flags(), builder, labelNode->labelId())); + ASMJIT_PROPAGATE(sb.append(":")); + break; + } + + case NodeType::kAlign: { + const AlignNode* alignNode = node->as(); + ASMJIT_PROPAGATE(sb.appendFormat(".align %u (%s)", + alignNode->alignment(), + alignNode->alignMode() == AlignMode::kCode ? "code" : "data")); + break; + } + + case NodeType::kEmbedData: { + const EmbedDataNode* embedNode = node->as(); + ASMJIT_PROPAGATE(sb.append('.')); + ASMJIT_PROPAGATE(formatDataType(sb, formatOptions.flags(), builder->arch(), embedNode->typeId())); + ASMJIT_PROPAGATE(sb.appendFormat(" {Count=%zu Repeat=%zu TotalSize=%zu}", embedNode->itemCount(), embedNode->repeatCount(), embedNode->dataSize())); + break; + } + + case NodeType::kEmbedLabel: { + const EmbedLabelNode* embedNode = node->as(); + ASMJIT_PROPAGATE(sb.append(".label ")); + ASMJIT_PROPAGATE(formatLabel(sb, formatOptions.flags(), builder, embedNode->labelId())); + break; + } + + case NodeType::kEmbedLabelDelta: { + const EmbedLabelDeltaNode* embedNode = node->as(); + ASMJIT_PROPAGATE(sb.append(".label (")); + ASMJIT_PROPAGATE(formatLabel(sb, formatOptions.flags(), builder, embedNode->labelId())); + ASMJIT_PROPAGATE(sb.append(" - ")); + ASMJIT_PROPAGATE(formatLabel(sb, formatOptions.flags(), builder, embedNode->baseLabelId())); + ASMJIT_PROPAGATE(sb.append(")")); + break; + } + + case NodeType::kConstPool: { + const ConstPoolNode* constPoolNode = node->as(); + ASMJIT_PROPAGATE(sb.appendFormat("[ConstPool Size=%zu Alignment=%zu]", constPoolNode->size(), constPoolNode->alignment())); + break; + }; + + case NodeType::kComment: { + const CommentNode* commentNode = node->as(); + ASMJIT_PROPAGATE(sb.appendFormat("; %s", commentNode->inlineComment())); + break; + } + + case NodeType::kSentinel: { + const SentinelNode* sentinelNode = node->as(); + const char* sentinelName = nullptr; + + switch (sentinelNode->sentinelType()) { + case SentinelType::kFuncEnd: + sentinelName = "[FuncEnd]"; + break; + + default: + sentinelName = "[Sentinel]"; + break; + } + + ASMJIT_PROPAGATE(sb.append(sentinelName)); + break; + } + +#ifndef ASMJIT_NO_COMPILER + case NodeType::kFunc: { + const FuncNode* funcNode = node->as(); + + if (builder->isCompiler()) { + ASMJIT_PROPAGATE(formatLabel(sb, formatOptions.flags(), builder, funcNode->labelId())); + ASMJIT_PROPAGATE(sb.append(": ")); + + ASMJIT_PROPAGATE(formatFuncRets(sb, formatOptions.flags(), static_cast(builder), funcNode->detail())); + ASMJIT_PROPAGATE(sb.append(" Func(")); + ASMJIT_PROPAGATE(formatFuncArgs(sb, formatOptions.flags(), static_cast(builder), funcNode->detail(), funcNode->argPacks())); + ASMJIT_PROPAGATE(sb.append(")")); + } + break; + } + + case NodeType::kFuncRet: { + const FuncRetNode* retNode = node->as(); + ASMJIT_PROPAGATE(sb.append("[FuncRet]")); + + for (uint32_t i = 0; i < 2; i++) { + const Operand_& op = retNode->_opArray[i]; + if (!op.isNone()) { + ASMJIT_PROPAGATE(sb.append(i == 0 ? " " : ", ")); + ASMJIT_PROPAGATE(formatOperand(sb, formatOptions.flags(), builder, builder->arch(), op)); + } + } + break; + } + + case NodeType::kInvoke: { + const InvokeNode* invokeNode = node->as(); + ASMJIT_PROPAGATE(builder->_funcs.formatInstruction(sb, formatOptions.flags(), builder, + builder->arch(), + invokeNode->baseInst(), invokeNode->operands(), invokeNode->opCount())); + break; + } +#endif + + default: { + ASMJIT_PROPAGATE(sb.appendFormat("[UserNode:%u]", node->type())); + break; + } + } + + if (node->hasInlineComment()) { + size_t requiredPadding = paddingFromOptions(formatOptions, FormatPaddingGroup::kRegularLine); + size_t currentPadding = sb.size() - startLineIndex; + + if (currentPadding < requiredPadding) + ASMJIT_PROPAGATE(sb.appendChars(' ', requiredPadding - currentPadding)); + + ASMJIT_PROPAGATE(sb.append("; ")); + ASMJIT_PROPAGATE(sb.append(node->inlineComment())); + } + + return kErrorOk; +} + +Error formatNodeList( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder) noexcept { + + return formatNodeList(sb, formatOptions, builder, builder->firstNode(), nullptr); +} + +Error formatNodeList( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder, + const BaseNode* begin, + const BaseNode* end) noexcept { + + const BaseNode* node = begin; + while (node != end) { + ASMJIT_PROPAGATE(formatNode(sb, formatOptions, builder, node)); + ASMJIT_PROPAGATE(sb.append('\n')); + node = node->next(); + } + return kErrorOk; +} +#endif + +} // {Formatter} + +ASMJIT_END_NAMESPACE + +#endif diff --git a/src/asmjit/core/formatter.h b/src/asmjit/core/formatter.h new file mode 100644 index 0000000..0ee6bde --- /dev/null +++ b/src/asmjit/core/formatter.h @@ -0,0 +1,249 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FORMATTER_H_INCLUDED +#define ASMJIT_CORE_FORMATTER_H_INCLUDED + +#include "../core/globals.h" +#include "../core/inst.h" +#include "../core/string.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_logging +//! \{ + +class BaseBuilder; +class BaseEmitter; +class BaseNode; +struct Operand_; + +//! Format flags used by \ref Logger and \ref FormatOptions. +enum class FormatFlags : uint32_t { + //! No formatting flags. + kNone = 0u, + + //! Show also binary form of each logged instruction (Assembler). + kMachineCode = 0x00000001u, + //! Show a text explanation of some immediate values. + kExplainImms = 0x00000002u, + //! Use hexadecimal notation of immediate values. + kHexImms = 0x00000004u, + //! Use hexadecimal notation of addresses and offsets in addresses. + kHexOffsets = 0x00000008u, + //! Show casts between virtual register types (Compiler output). + kRegCasts = 0x00000010u, + //! Show positions associated with nodes (Compiler output). + kPositions = 0x00000020u, + //! Always format a register type (Compiler output). + kRegType = 0x00000040u +}; +ASMJIT_DEFINE_ENUM_FLAGS(FormatFlags) + +//! Format indentation group, used by \ref FormatOptions. +enum class FormatIndentationGroup : uint32_t { + //! Indentation used for instructions and directives. + kCode = 0u, + //! Indentation used for labels and function nodes. + kLabel = 1u, + //! Indentation used for comments (not inline comments). + kComment = 2u, + + //! \cond INTERNAL + //! Reserved for future use. + kReserved = 3u, + //! \endcond + + //! Maximum value of `FormatIndentationGroup`. + kMaxValue = kReserved +}; + +//! Format padding group, used by \ref FormatOptions. +enum class FormatPaddingGroup : uint32_t { + //! Describes padding of a regular line, which can represent instruction, data, or assembler directives. + kRegularLine = 0, + //! Describes padding of machine code dump that is visible next to the instruction, if enabled. + kMachineCode = 1, + + //! Maximum value of `FormatPaddingGroup`. + kMaxValue = kMachineCode +}; + +//! Formatting options used by \ref Logger and \ref Formatter. +class FormatOptions { +public: + //! \name Members + //! \{ + + //! Format flags. + FormatFlags _flags = FormatFlags::kNone; + //! Indentations for each indentation group. + Support::Array _indentation {}; + //! Paddings for each padding group. + Support::Array _padding {}; + + //! \} + + //! \name Reset + //! \{ + + //! Resets FormatOptions to its default initialized state. + inline void reset() noexcept { + _flags = FormatFlags::kNone; + _indentation.fill(uint8_t(0)); + _padding.fill(uint16_t(0)); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns format flags. + inline FormatFlags flags() const noexcept { return _flags; } + //! Tests whether the given `flag` is set in format flags. + inline bool hasFlag(FormatFlags flag) const noexcept { return Support::test(_flags, flag); } + + //! Resets all format flags to `flags`. + inline void setFlags(FormatFlags flags) noexcept { _flags = flags; } + //! Adds `flags` to format flags. + inline void addFlags(FormatFlags flags) noexcept { _flags |= flags; } + //! Removes `flags` from format flags. + inline void clearFlags(FormatFlags flags) noexcept { _flags &= ~flags; } + + //! Returns indentation for the given indentation `group`. + inline uint8_t indentation(FormatIndentationGroup group) const noexcept { return _indentation[group]; } + //! Sets indentation for the given indentation `group`. + inline void setIndentation(FormatIndentationGroup group, uint32_t n) noexcept { _indentation[group] = uint8_t(n); } + //! Resets indentation for the given indentation `group` to zero. + inline void resetIndentation(FormatIndentationGroup group) noexcept { _indentation[group] = uint8_t(0); } + + //! Returns pading for the given padding `group`. + inline size_t padding(FormatPaddingGroup group) const noexcept { return _padding[group]; } + //! Sets pading for the given padding `group`. + inline void setPadding(FormatPaddingGroup group, size_t n) noexcept { _padding[group] = uint16_t(n); } + //! Resets pading for the given padding `group` to zero, which means that a default padding will be used + //! based on the target architecture properties. + inline void resetPadding(FormatPaddingGroup group) noexcept { _padding[group] = uint16_t(0); } + + //! \} +}; + +//! Provides formatting functionality to format operands, instructions, and nodes. +namespace Formatter { + +#ifndef ASMJIT_NO_LOGGING + +//! Appends a formatted `typeId` to the output string `sb`. +ASMJIT_API Error formatTypeId( + String& sb, + TypeId typeId) noexcept; + +//! Appends a formatted `featureId` to the output string `sb`. +//! +//! See \ref CpuFeatures. +ASMJIT_API Error formatFeature( + String& sb, + Arch arch, + uint32_t featureId) noexcept; + +//! Appends a formatted register to the output string `sb`. +//! +//! \note Emitter is optional, but it's required to format virtual registers, which won't be formatted properly +//! if the `emitter` is not provided. +ASMJIT_API Error formatRegister( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + RegType regType, + uint32_t regId) noexcept; + +//! Appends a formatted label to the output string `sb`. +//! +//! \note Emitter is optional, but it's required to format named labels properly, otherwise the formatted as +//! it is an anonymous label. +ASMJIT_API Error formatLabel( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + uint32_t labelId) noexcept; + +//! Appends a formatted operand to the output string `sb`. +//! +//! \note Emitter is optional, but it's required to format named labels and virtual registers. See +//! \ref formatRegister() and \ref formatLabel() for more details. +ASMJIT_API Error formatOperand( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + const Operand_& op) noexcept; + +//! Appends a formatted data-type to the output string `sb`. +ASMJIT_API Error formatDataType( + String& sb, + FormatFlags formatFlags, + Arch arch, + TypeId typeId) noexcept; + +//! Appends a formatted data to the output string `sb`. +ASMJIT_API Error formatData( + String& sb, + FormatFlags formatFlags, + Arch arch, + TypeId typeId, const void* data, size_t itemCount, size_t repeatCount = 1) noexcept; + +//! Appends a formatted instruction to the output string `sb`. +//! +//! \note Emitter is optional, but it's required to format named labels and virtual registers. See +//! \ref formatRegister() and \ref formatLabel() for more details. +ASMJIT_API Error formatInstruction( + String& sb, + FormatFlags formatFlags, + const BaseEmitter* emitter, + Arch arch, + const BaseInst& inst, const Operand_* operands, size_t opCount) noexcept; + +#ifndef ASMJIT_NO_BUILDER +//! Appends a formatted node to the output string `sb`. +//! +//! The `node` must belong to the provided `builder`. +ASMJIT_API Error formatNode( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder, + const BaseNode* node) noexcept; + +//! Appends formatted nodes to the output string `sb`. +//! +//! All nodes that are part of the given `builder` will be appended. +ASMJIT_API Error formatNodeList( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder) noexcept; + +//! Appends formatted nodes to the output string `sb`. +//! +//! This function works the same as \ref formatNode(), but appends more nodes to the output string, +//! separating each node with a newline '\n' character. +ASMJIT_API Error formatNodeList( + String& sb, + const FormatOptions& formatOptions, + const BaseBuilder* builder, + const BaseNode* begin, + const BaseNode* end) noexcept; +#endif + +#endif + +} // {Formatter} + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FORMATTER_H_INCLUDED diff --git a/src/asmjit/core/formatter_p.h b/src/asmjit/core/formatter_p.h new file mode 100644 index 0000000..6070fd7 --- /dev/null +++ b/src/asmjit/core/formatter_p.h @@ -0,0 +1,34 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FORMATTER_P_H_INCLUDED +#define ASMJIT_CORE_FORMATTER_P_H_INCLUDED + +#include "../core/formatter.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_logging +//! \{ + +namespace Formatter { + +static ASMJIT_FORCE_INLINE size_t paddingFromOptions(const FormatOptions& formatOptions, FormatPaddingGroup group) noexcept { + static constexpr uint16_t _defaultPaddingTable[uint32_t(FormatPaddingGroup::kMaxValue) + 1] = { 44, 26 }; + static_assert(uint32_t(FormatPaddingGroup::kMaxValue) + 1 == 2, "If a new group is defined it must be added here"); + + size_t padding = formatOptions.padding(group); + return padding ? padding : size_t(_defaultPaddingTable[uint32_t(group)]); +} + +} // {Formatter} + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FORMATTER_H_P_INCLUDED diff --git a/src/asmjit/core/func.cpp b/src/asmjit/core/func.cpp new file mode 100644 index 0000000..04dc2aa --- /dev/null +++ b/src/asmjit/core/func.cpp @@ -0,0 +1,286 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/archtraits.h" +#include "../core/func.h" +#include "../core/operand.h" +#include "../core/type.h" +#include "../core/funcargscontext_p.h" + +#if !defined(ASMJIT_NO_X86) + #include "../x86/x86func_p.h" +#endif + +#if !defined(ASMJIT_NO_AARCH64) + #include "../arm/a64func_p.h" +#endif + +ASMJIT_BEGIN_NAMESPACE + +// CallConv - Init & Reset +// ======================= + +ASMJIT_FAVOR_SIZE Error CallConv::init(CallConvId ccId, const Environment& environment) noexcept { + reset(); + +#if !defined(ASMJIT_NO_X86) + if (environment.isFamilyX86()) + return x86::FuncInternal::initCallConv(*this, ccId, environment); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (environment.isFamilyAArch64()) + return a64::FuncInternal::initCallConv(*this, ccId, environment); +#endif + + return DebugUtils::errored(kErrorInvalidArgument); +} + +// FuncDetail - Init / Reset +// ========================= + +ASMJIT_FAVOR_SIZE Error FuncDetail::init(const FuncSignature& signature, const Environment& environment) noexcept { + CallConvId ccId = signature.callConvId(); + uint32_t argCount = signature.argCount(); + + if (ASMJIT_UNLIKELY(argCount > Globals::kMaxFuncArgs)) + return DebugUtils::errored(kErrorInvalidArgument); + + CallConv& cc = _callConv; + ASMJIT_PROPAGATE(cc.init(ccId, environment)); + + uint32_t registerSize = Environment::registerSizeFromArch(cc.arch()); + uint32_t deabstractDelta = TypeUtils::deabstractDeltaOfSize(registerSize); + + const TypeId* signatureArgs = signature.args(); + for (uint32_t argIndex = 0; argIndex < argCount; argIndex++) { + FuncValuePack& argPack = _args[argIndex]; + argPack[0].initTypeId(TypeUtils::deabstract(signatureArgs[argIndex], deabstractDelta)); + } + + _argCount = uint8_t(argCount); + _vaIndex = uint8_t(signature.vaIndex()); + + TypeId ret = signature.ret(); + if (ret != TypeId::kVoid) + _rets[0].initTypeId(TypeUtils::deabstract(ret, deabstractDelta)); + +#if !defined(ASMJIT_NO_X86) + if (environment.isFamilyX86()) + return x86::FuncInternal::initFuncDetail(*this, signature, registerSize); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (environment.isFamilyAArch64()) + return a64::FuncInternal::initFuncDetail(*this, signature, registerSize); +#endif + + // We should never bubble here as if `cc.init()` succeeded then there has to be an implementation for the current + // architecture. However, stay safe. + return DebugUtils::errored(kErrorInvalidArgument); +} + +// FuncFrame - Init +// ================ + +ASMJIT_FAVOR_SIZE Error FuncFrame::init(const FuncDetail& func) noexcept { + Arch arch = func.callConv().arch(); + if (!Environment::isValidArch(arch)) + return DebugUtils::errored(kErrorInvalidArch); + + const ArchTraits& archTraits = ArchTraits::byArch(arch); + + // Initializing FuncFrame means making a copy of some properties of `func`. Properties like `_localStackSize` will + // be set by the user before the frame is finalized. + reset(); + + _arch = arch; + _spRegId = uint8_t(archTraits.spRegId()); + _saRegId = uint8_t(BaseReg::kIdBad); + + uint32_t naturalStackAlignment = func.callConv().naturalStackAlignment(); + uint32_t minDynamicAlignment = Support::max(naturalStackAlignment, 16); + + if (minDynamicAlignment == naturalStackAlignment) + minDynamicAlignment <<= 1; + + _naturalStackAlignment = uint8_t(naturalStackAlignment); + _minDynamicAlignment = uint8_t(minDynamicAlignment); + _redZoneSize = uint8_t(func.redZoneSize()); + _spillZoneSize = uint8_t(func.spillZoneSize()); + _finalStackAlignment = uint8_t(_naturalStackAlignment); + + if (func.hasFlag(CallConvFlags::kCalleePopsStack)) { + _calleeStackCleanup = uint16_t(func.argStackSize()); + } + + // Initial masks of dirty and preserved registers. + for (RegGroup group : RegGroupVirtValues{}) { + _dirtyRegs[group] = func.usedRegs(group); + _preservedRegs[group] = func.preservedRegs(group); + } + + // Exclude stack pointer - this register is never included in saved GP regs. + _preservedRegs[RegGroup::kGp] &= ~Support::bitMask(archTraits.spRegId()); + + // The size and alignment of save/restore area of registers for each virtual register group + _saveRestoreRegSize = func.callConv()._saveRestoreRegSize; + _saveRestoreAlignment = func.callConv()._saveRestoreAlignment; + + return kErrorOk; +} + +// FuncFrame - Finalize +// ==================== + +ASMJIT_FAVOR_SIZE Error FuncFrame::finalize() noexcept { + if (!Environment::isValidArch(arch())) + return DebugUtils::errored(kErrorInvalidArch); + + const ArchTraits& archTraits = ArchTraits::byArch(arch()); + + uint32_t registerSize = _saveRestoreRegSize[RegGroup::kGp]; + uint32_t vectorSize = _saveRestoreRegSize[RegGroup::kVec]; + uint32_t returnAddressSize = archTraits.hasLinkReg() ? 0u : registerSize; + + // The final stack alignment must be updated accordingly to call and local stack alignments. + uint32_t stackAlignment = _finalStackAlignment; + ASMJIT_ASSERT(stackAlignment == Support::max(_naturalStackAlignment, + _callStackAlignment, + _localStackAlignment)); + + bool hasFP = hasPreservedFP(); + bool hasDA = hasDynamicAlignment(); + + uint32_t kSp = archTraits.spRegId(); + uint32_t kFp = archTraits.fpRegId(); + uint32_t kLr = archTraits.linkRegId(); + + // Make frame pointer dirty if the function uses it. + if (hasFP) { + _dirtyRegs[RegGroup::kGp] |= Support::bitMask(kFp); + + // Currently required by ARM, if this works differently across architectures we would have to generalize most + // likely in CallConv. + if (kLr != BaseReg::kIdBad) + _dirtyRegs[RegGroup::kGp] |= Support::bitMask(kLr); + } + + // These two are identical if the function doesn't align its stack dynamically. + uint32_t saRegId = _saRegId; + if (saRegId == BaseReg::kIdBad) + saRegId = kSp; + + // Fix stack arguments base-register from SP to FP in case it was not picked before and the function performs + // dynamic stack alignment. + if (hasDA && saRegId == kSp) + saRegId = kFp; + + // Mark as dirty any register but SP if used as SA pointer. + if (saRegId != kSp) + _dirtyRegs[RegGroup::kGp] |= Support::bitMask(saRegId); + + _spRegId = uint8_t(kSp); + _saRegId = uint8_t(saRegId); + + // Setup stack size used to save preserved registers. + uint32_t saveRestoreSizes[2] {}; + for (RegGroup group : RegGroupVirtValues{}) + saveRestoreSizes[size_t(!archTraits.hasInstPushPop(group))] + += Support::alignUp(Support::popcnt(savedRegs(group)) * saveRestoreRegSize(group), saveRestoreAlignment(group)); + + _pushPopSaveSize = uint16_t(saveRestoreSizes[0]); + _extraRegSaveSize = uint16_t(saveRestoreSizes[1]); + + uint32_t v = 0; // The beginning of the stack frame relative to SP after prolog. + v += callStackSize(); // Count 'callStackSize' <- This is used to call functions. + v = Support::alignUp(v, stackAlignment); // Align to function's stack alignment. + + _localStackOffset = v; // Store 'localStackOffset' <- Function's local stack starts here. + v += localStackSize(); // Count 'localStackSize' <- Function's local stack ends here. + + // If the function's stack must be aligned, calculate the alignment necessary to store vector registers, and set + // `FuncAttributes::kAlignedVecSR` to inform PEI that it can use instructions that perform aligned stores/loads. + if (stackAlignment >= vectorSize && _extraRegSaveSize) { + addAttributes(FuncAttributes::kAlignedVecSR); + v = Support::alignUp(v, vectorSize); // Align 'extraRegSaveOffset'. + } + + _extraRegSaveOffset = v; // Store 'extraRegSaveOffset' <- Non-GP save/restore starts here. + v += _extraRegSaveSize; // Count 'extraRegSaveSize' <- Non-GP save/restore ends here. + + // Calculate if dynamic alignment (DA) slot (stored as offset relative to SP) is required and its offset. + if (hasDA && !hasFP) { + _daOffset = v; // Store 'daOffset' <- DA pointer would be stored here. + v += registerSize; // Count 'daOffset'. + } + else { + _daOffset = FuncFrame::kTagInvalidOffset; + } + + // Link Register + // ------------- + // + // The stack is aligned after the function call as the return address is stored in a link register. Some + // architectures may require to always have aligned stack after PUSH/POP operation, which is represented + // by ArchTraits::stackAlignmentConstraint(). + // + // No Link Register (X86/X64) + // -------------------------- + // + // The return address should be stored after GP save/restore regs. It has the same size as `registerSize` + // (basically the native register/pointer size). We don't adjust it now as `v` now contains the exact size + // that the function requires to adjust (call frame + stack frame, vec stack size). The stack (if we consider + // this size) is misaligned now, as it's always aligned before the function call - when `call()` is executed + // it pushes the current EIP|RIP onto the stack, and misaligns it by 12 or 8 bytes (depending on the + // architecture). So count number of bytes needed to align it up to the function's CallFrame (the beginning). + if (v || hasFuncCalls() || !returnAddressSize) + v += Support::alignUpDiff(v + pushPopSaveSize() + returnAddressSize, stackAlignment); + + _pushPopSaveOffset = v; // Store 'pushPopSaveOffset' <- Function's push/pop save/restore starts here. + _stackAdjustment = v; // Store 'stackAdjustment' <- SA used by 'add SP, SA' and 'sub SP, SA'. + v += _pushPopSaveSize; // Count 'pushPopSaveSize' <- Function's push/pop save/restore ends here. + _finalStackSize = v; // Store 'finalStackSize' <- Final stack used by the function. + + if (!archTraits.hasLinkReg()) + v += registerSize; // Count 'ReturnAddress' <- As CALL pushes onto stack. + + // If the function performs dynamic stack alignment then the stack-adjustment must be aligned. + if (hasDA) + _stackAdjustment = Support::alignUp(_stackAdjustment, stackAlignment); + + // Calculate where the function arguments start relative to SP. + _saOffsetFromSP = hasDA ? FuncFrame::kTagInvalidOffset : v; + + // Calculate where the function arguments start relative to FP or user-provided register. + _saOffsetFromSA = hasFP ? returnAddressSize + registerSize // Return address + frame pointer. + : returnAddressSize + _pushPopSaveSize; // Return address + all push/pop regs. + + return kErrorOk; +} + +// FuncArgsAssignment - UpdateFuncFrame +// ==================================== + +ASMJIT_FAVOR_SIZE Error FuncArgsAssignment::updateFuncFrame(FuncFrame& frame) const noexcept { + Arch arch = frame.arch(); + const FuncDetail* func = funcDetail(); + + if (!func) + return DebugUtils::errored(kErrorInvalidState); + + RAConstraints constraints; + ASMJIT_PROPAGATE(constraints.init(arch)); + + FuncArgsContext ctx; + ASMJIT_PROPAGATE(ctx.initWorkData(frame, *this, &constraints)); + ASMJIT_PROPAGATE(ctx.markDstRegsDirty(frame)); + ASMJIT_PROPAGATE(ctx.markScratchRegs(frame)); + ASMJIT_PROPAGATE(ctx.markStackArgsReg(frame)); + return kErrorOk; +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/func.h b/src/asmjit/core/func.h new file mode 100644 index 0000000..78a41f9 --- /dev/null +++ b/src/asmjit/core/func.h @@ -0,0 +1,1445 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FUNC_H_INCLUDED +#define ASMJIT_CORE_FUNC_H_INCLUDED + +#include "../core/archtraits.h" +#include "../core/environment.h" +#include "../core/operand.h" +#include "../core/type.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_function +//! \{ + +//! Calling convention id. +//! +//! Calling conventions can be divided into the following groups: +//! +//! - Universal - calling conventions are applicable to any target. They will be converted to a target dependent +//! calling convention at runtime by \ref CallConv::init() with some help from \ref Environment. The purpose of +//! these calling conventions is to make using functions less target dependent and closer to C and C++. +//! +//! - Target specific - calling conventions that are used by a particular architecture and ABI. For example +//! Windows 64-bit calling convention and AMD64 SystemV calling convention. +enum class CallConvId : uint8_t { + //! None or invalid (can't be used). + kNone = 0, + + // Universal Calling Conventions + // ----------------------------- + + //! Standard function call or explicit `__cdecl` where it can be specified. + //! + //! This is a universal calling convention, which is used to initialize specific calling connventions based on + //! architecture, platform, and its ABI. + kCDecl = 1, + + //! `__stdcall` on targets that support this calling convention (X86). + //! + //! \note This calling convention is only supported on 32-bit X86. If used on environment that doesn't support + //! this calling convention it will be replaced by \ref CallConvId::kCDecl. + kStdCall = 2, + + //! `__fastcall` on targets that support this calling convention (X86). + //! + //! \note This calling convention is only supported on 32-bit X86. If used on environment that doesn't support + //! this calling convention it will be replaced by \ref CallConvId::kCDecl. + kFastCall = 3, + + //! `__vectorcall` on targets that support this calling convention (X86/X64). + //! + //! \note This calling convention is only supported on 32-bit and 64-bit X86 architecture on Windows platform. + //! If used on environment that doesn't support this calling it will be replaced by \ref CallConvId::kCDecl. + kVectorCall = 4, + + //! `__thiscall` on targets that support this calling convention (X86). + //! + //! \note This calling convention is only supported on 32-bit X86 Windows platform. If used on environment that + //! doesn't support this calling convention it will be replaced by \ref CallConvId::kCDecl. + kThisCall = 5, + + //! `__attribute__((regparm(1)))` convention (GCC and Clang). + kRegParm1 = 6, + //! `__attribute__((regparm(2)))` convention (GCC and Clang). + kRegParm2 = 7, + //! `__attribute__((regparm(3)))` convention (GCC and Clang). + kRegParm3 = 8, + + //! Soft-float calling convention (ARM). + //! + //! Floating point arguments are passed via general purpose registers. + kSoftFloat = 9, + + //! Hard-float calling convention (ARM). + //! + //! Floating point arguments are passed via SIMD registers. + kHardFloat = 10, + + //! AsmJit specific calling convention designed for calling functions inside a multimedia code that don't use many + //! registers internally, but are long enough to be called and not inlined. These functions are usually used to + //! calculate trigonometric functions, logarithms, etc... + kLightCall2 = 16, + kLightCall3 = 17, + kLightCall4 = 18, + + // ABI-Specific Calling Conventions + // -------------------------------- + + //! X64 System-V calling convention. + kX64SystemV = 32, + //! X64 Windows calling convention. + kX64Windows = 33, + + //! Maximum value of `CallConvId`. + kMaxValue = kX64Windows, + + // Host Calling Conventions + // ------------------------ + + //! Host calling convention detected at compile-time. + kHost = +#if defined(_DOXYGEN) + DETECTED_AT_COMPILE_TIME +#elif ASMJIT_ARCH_ARM == 32 && defined(__SOFTFP__) + kSoftFloat +#elif ASMJIT_ARCH_ARM == 32 && !defined(__SOFTFP__) + kHardFloat +#else + kCDecl +#endif +}; + +//! Strategy used by calling conventions to assign registers to function arguments. +//! +//! Calling convention strategy describes how AsmJit should convert function arguments used by \ref FuncSignature +//! into register identifiers and stack offsets. The \ref CallConvStrategy::kDefault strategy assigns registers +//! and then stack whereas \ref CallConvStrategy::kX64Windows strategy does register shadowing as defined by WIN64 +//! calling convention, which is only used by 64-bit Windows. +enum class CallConvStrategy : uint8_t { + //! Default register assignment strategy. + kDefault = 0, + //! Windows 64-bit ABI register assignment strategy. + kX64Windows = 1, + //! Windows 64-bit __vectorcall register assignment strategy. + kX64VectorCall = 2, + + //! Maximum value of `CallConvStrategy`. + kMaxValue = kX64VectorCall +}; + +//! Calling convention flags. +enum class CallConvFlags : uint32_t { + //! No flags. + kNone = 0, + //! Callee is responsible for cleaning up the stack. + kCalleePopsStack = 0x0001u, + //! Pass vector arguments indirectly (as a pointer). + kIndirectVecArgs = 0x0002u, + //! Pass F32 and F64 arguments via VEC128 register. + kPassFloatsByVec = 0x0004u, + //! Pass MMX and vector arguments via stack if the function has variable arguments. + kPassVecByStackIfVA = 0x0008u, + //! MMX registers are passed and returned via GP registers. + kPassMmxByGp = 0x0010u, + //! MMX registers are passed and returned via XMM registers. + kPassMmxByXmm = 0x0020u, + //! Calling convention can be used with variable arguments. + kVarArgCompatible = 0x0080u +}; +ASMJIT_DEFINE_ENUM_FLAGS(CallConvFlags) + +//! Function calling convention. +//! +//! Function calling convention is a scheme that defines how function parameters are passed and how function +//! returns its result. AsmJit defines a variety of architecture and OS specific calling conventions and also +//! provides a compile time detection to make the code-generation easier. +struct CallConv { + //! \name Constants + //! \{ + + enum : uint32_t { + //! Maximum number of register arguments per register group. + //! + //! \note This is not really AsmJit's limitatation, it's just the number that makes sense considering all common + //! calling conventions. Usually even conventions that use registers to pass function arguments are limited to 8 + //! and less arguments passed via registers per group. + kMaxRegArgsPerGroup = 16 + }; + + //! \} + + //! \name Members + //! \{ + + //! Target architecture. + Arch _arch; + //! Calling convention id. + CallConvId _id; + //! Register assignment strategy. + CallConvStrategy _strategy; + + //! Red zone size (AMD64 == 128 bytes). + uint8_t _redZoneSize; + //! Spill zone size (WIN-X64 == 32 bytes). + uint8_t _spillZoneSize; + //! Natural stack alignment as defined by OS/ABI. + uint8_t _naturalStackAlignment; + + //! Calling convention flags. + CallConvFlags _flags; + + //! Size to save/restore per register group. + Support::Array _saveRestoreRegSize; + //! Alignment of save/restore groups. + Support::Array _saveRestoreAlignment; + + //! Mask of all passed registers, per group. + Support::Array _passedRegs; + //! Mask of all preserved registers, per group. + Support::Array _preservedRegs; + + //! Passed registers' order. + union RegOrder { + //! Passed registers, ordered. + uint8_t id[kMaxRegArgsPerGroup]; + //! Packed IDs in `uint32_t` array. + uint32_t packed[(kMaxRegArgsPerGroup + 3) / 4]; + }; + + //! Passed registers' order, per register group. + Support::Array _passedOrder; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Initializes this calling convention to the given `ccId` based on the `environment`. + //! + //! See \ref CallConvId and \ref Environment for more details. + ASMJIT_API Error init(CallConvId ccId, const Environment& environment) noexcept; + + //! Resets this CallConv struct into a defined state. + //! + //! It's recommended to reset the \ref CallConv struct in case you would like create a custom calling convention + //! as it prevents from using an uninitialized data (CallConv doesn't have a constructor that would initialize it, + //! it's just a struct). + inline void reset() noexcept { + memset(this, 0, sizeof(*this)); + memset(_passedOrder.data(), 0xFF, sizeof(_passedOrder)); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the target architecture of this calling convention. + inline Arch arch() const noexcept { return _arch; } + //! Sets the target architecture of this calling convention. + inline void setArch(Arch arch) noexcept { _arch = arch; } + + //! Returns the calling convention id. + inline CallConvId id() const noexcept { return _id; } + //! Sets the calling convention id. + inline void setId(CallConvId ccId) noexcept { _id = ccId; } + + //! Returns the strategy used to assign registers to arguments. + inline CallConvStrategy strategy() const noexcept { return _strategy; } + //! Sets the strategy used to assign registers to arguments. + inline void setStrategy(CallConvStrategy ccStrategy) noexcept { _strategy = ccStrategy; } + + //! Tests whether the calling convention has the given `flag` set. + inline bool hasFlag(CallConvFlags flag) const noexcept { return Support::test(_flags, flag); } + //! Returns the calling convention flags, see `Flags`. + inline CallConvFlags flags() const noexcept { return _flags; } + //! Adds the calling convention flags, see `Flags`. + inline void setFlags(CallConvFlags flag) noexcept { _flags = flag; }; + //! Adds the calling convention flags, see `Flags`. + inline void addFlags(CallConvFlags flags) noexcept { _flags |= flags; }; + + //! Tests whether this calling convention specifies 'RedZone'. + inline bool hasRedZone() const noexcept { return _redZoneSize != 0; } + //! Tests whether this calling convention specifies 'SpillZone'. + inline bool hasSpillZone() const noexcept { return _spillZoneSize != 0; } + + //! Returns size of 'RedZone'. + inline uint32_t redZoneSize() const noexcept { return _redZoneSize; } + //! Returns size of 'SpillZone'. + inline uint32_t spillZoneSize() const noexcept { return _spillZoneSize; } + + //! Sets size of 'RedZone'. + inline void setRedZoneSize(uint32_t size) noexcept { _redZoneSize = uint8_t(size); } + //! Sets size of 'SpillZone'. + inline void setSpillZoneSize(uint32_t size) noexcept { _spillZoneSize = uint8_t(size); } + + //! Returns a natural stack alignment. + inline uint32_t naturalStackAlignment() const noexcept { return _naturalStackAlignment; } + //! Sets a natural stack alignment. + //! + //! This function can be used to override the default stack alignment in case that you know that it's alignment is + //! different. For example it allows to implement custom calling conventions that guarantee higher stack alignment. + inline void setNaturalStackAlignment(uint32_t value) noexcept { _naturalStackAlignment = uint8_t(value); } + + //! Returns the size of a register (or its part) to be saved and restored of the given `group`. + inline uint32_t saveRestoreRegSize(RegGroup group) const noexcept { return _saveRestoreRegSize[group]; } + //! Sets the size of a vector register (or its part) to be saved and restored. + inline void setSaveRestoreRegSize(RegGroup group, uint32_t size) noexcept { _saveRestoreRegSize[group] = uint8_t(size); } + + //! Returns the alignment of a save-restore area of the given `group`. + inline uint32_t saveRestoreAlignment(RegGroup group) const noexcept { return _saveRestoreAlignment[group]; } + //! Sets the alignment of a save-restore area of the given `group`. + inline void setSaveRestoreAlignment(RegGroup group, uint32_t alignment) noexcept { _saveRestoreAlignment[group] = uint8_t(alignment); } + + //! Returns the order of passed registers of the given `group`. + inline const uint8_t* passedOrder(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _passedOrder[size_t(group)].id; + } + + //! Returns the mask of passed registers of the given `group`. + inline RegMask passedRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _passedRegs[size_t(group)]; + } + + inline void _setPassedPacked(RegGroup group, uint32_t p0, uint32_t p1, uint32_t p2, uint32_t p3) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + + _passedOrder[group].packed[0] = p0; + _passedOrder[group].packed[1] = p1; + _passedOrder[group].packed[2] = p2; + _passedOrder[group].packed[3] = p3; + } + + //! Resets the order and mask of passed registers. + inline void setPassedToNone(RegGroup group) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + + _setPassedPacked(group, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu, 0xFFFFFFFFu); + _passedRegs[size_t(group)] = 0u; + } + + //! Sets the order and mask of passed registers. + inline void setPassedOrder(RegGroup group, uint32_t a0, uint32_t a1 = 0xFF, uint32_t a2 = 0xFF, uint32_t a3 = 0xFF, uint32_t a4 = 0xFF, uint32_t a5 = 0xFF, uint32_t a6 = 0xFF, uint32_t a7 = 0xFF) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + + // NOTE: This should always be called with all arguments known at compile time, so even if it looks scary it + // should be translated into few instructions. + _setPassedPacked(group, Support::bytepack32_4x8(a0, a1, a2, a3), + Support::bytepack32_4x8(a4, a5, a6, a7), + 0xFFFFFFFFu, + 0xFFFFFFFFu); + + _passedRegs[group] = (a0 != 0xFF ? 1u << a0 : 0u) | + (a1 != 0xFF ? 1u << a1 : 0u) | + (a2 != 0xFF ? 1u << a2 : 0u) | + (a3 != 0xFF ? 1u << a3 : 0u) | + (a4 != 0xFF ? 1u << a4 : 0u) | + (a5 != 0xFF ? 1u << a5 : 0u) | + (a6 != 0xFF ? 1u << a6 : 0u) | + (a7 != 0xFF ? 1u << a7 : 0u) ; + } + + //! Returns preserved register mask of the given `group`. + inline RegMask preservedRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _preservedRegs[group]; + } + + //! Sets preserved register mask of the given `group`. + inline void setPreservedRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _preservedRegs[group] = regs; + } + + //! \} +}; + +//! Function signature. +//! +//! Contains information about function return type, count of arguments and their TypeIds. Function signature is +//! a low level structure which doesn't contain platform specific or calling convention specific information. +struct FuncSignature { + //! \name Constants + //! \{ + + enum : uint8_t { + //! Doesn't have variable number of arguments (`...`). + kNoVarArgs = 0xFFu + }; + + //! \} + + //! \name Members + //! \{ + + //! Calling convention id. + CallConvId _ccId; + //! Count of arguments. + uint8_t _argCount; + //! Index of a first VA or `kNoVarArgs`. + uint8_t _vaIndex; + //! Return value TypeId. + TypeId _ret; + //! Function arguments TypeIds. + const TypeId* _args; + + //! \} + + //! \name Initializtion & Reset + //! \{ + + //! Initializes the function signature. + inline void init(CallConvId ccId, uint32_t vaIndex, TypeId ret, const TypeId* args, uint32_t argCount) noexcept { + ASMJIT_ASSERT(argCount <= 0xFF); + + _ccId = ccId; + _argCount = uint8_t(argCount); + _vaIndex = uint8_t(vaIndex); + _ret = ret; + _args = args; + } + + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the calling convention. + inline CallConvId callConvId() const noexcept { return _ccId; } + //! Sets the calling convention to `ccId`; + inline void setCallConvId(CallConvId ccId) noexcept { _ccId = ccId; } + + //! Tests whether the function has variable number of arguments (...). + inline bool hasVarArgs() const noexcept { return _vaIndex != kNoVarArgs; } + //! Returns the variable arguments (...) index, `kNoVarArgs` if none. + inline uint32_t vaIndex() const noexcept { return _vaIndex; } + //! Sets the variable arguments (...) index to `index`. + inline void setVaIndex(uint32_t index) noexcept { _vaIndex = uint8_t(index); } + //! Resets the variable arguments index (making it a non-va function). + inline void resetVaIndex() noexcept { _vaIndex = kNoVarArgs; } + + //! Returns the number of function arguments. + inline uint32_t argCount() const noexcept { return _argCount; } + + inline bool hasRet() const noexcept { return _ret != TypeId::kVoid; } + //! Returns the return value type. + inline TypeId ret() const noexcept { return _ret; } + + //! Returns the type of the argument at index `i`. + inline TypeId arg(uint32_t i) const noexcept { + ASMJIT_ASSERT(i < _argCount); + return _args[i]; + } + //! Returns the array of function arguments' types. + inline const TypeId* args() const noexcept { return _args; } + + //! \} +}; + +template +class FuncSignatureT : public FuncSignature { +public: + inline FuncSignatureT(CallConvId ccId = CallConvId::kHost, uint32_t vaIndex = kNoVarArgs) noexcept { + static constexpr TypeId ret_args[] = { (TypeId(TypeUtils::TypeIdOfT::kTypeId))... }; + init(ccId, vaIndex, ret_args[0], ret_args + 1, uint32_t(ASMJIT_ARRAY_SIZE(ret_args) - 1)); + } +}; + +//! Function signature builder. +class FuncSignatureBuilder : public FuncSignature { +public: + TypeId _builderArgList[Globals::kMaxFuncArgs]; + + //! \name Initializtion & Reset + //! \{ + + inline FuncSignatureBuilder(CallConvId ccId = CallConvId::kHost, uint32_t vaIndex = kNoVarArgs) noexcept { + init(ccId, vaIndex, TypeId::kVoid, _builderArgList, 0); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Sets the return type to `retType`. + inline void setRet(TypeId retType) noexcept { _ret = retType; } + //! Sets the return type based on `T`. + template + inline void setRetT() noexcept { setRet(TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + + //! Sets the argument at index `index` to `argType`. + inline void setArg(uint32_t index, TypeId argType) noexcept { + ASMJIT_ASSERT(index < _argCount); + _builderArgList[index] = argType; + } + //! Sets the argument at index `i` to the type based on `T`. + template + inline void setArgT(uint32_t index) noexcept { setArg(index, TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + + //! Appends an argument of `type` to the function prototype. + inline void addArg(TypeId type) noexcept { + ASMJIT_ASSERT(_argCount < Globals::kMaxFuncArgs); + _builderArgList[_argCount++] = type; + } + //! Appends an argument of type based on `T` to the function prototype. + template + inline void addArgT() noexcept { addArg(TypeId(TypeUtils::TypeIdOfT::kTypeId)); } + + //! \} +}; + +//! Argument or return value (or its part) as defined by `FuncSignature`, but with register or stack address +//! (and other metadata) assigned. +struct FuncValue { + //! \name Constants + //! \{ + + enum Bits : uint32_t { + kTypeIdShift = 0, //!< TypeId shift. + kTypeIdMask = 0x000000FFu, //!< TypeId mask. + + kFlagIsReg = 0x00000100u, //!< Passed by register. + kFlagIsStack = 0x00000200u, //!< Passed by stack. + kFlagIsIndirect = 0x00000400u, //!< Passed indirectly by reference (internally a pointer). + kFlagIsDone = 0x00000800u, //!< Used internally by arguments allocator. + + kStackOffsetShift = 12, //!< Stack offset shift. + kStackOffsetMask = 0xFFFFF000u, //!< Stack offset mask (must occupy MSB bits). + + kRegIdShift = 16, //!< RegId shift. + kRegIdMask = 0x00FF0000u, //!< RegId mask. + + kRegTypeShift = 24, //!< RegType shift. + kRegTypeMask = 0xFF000000u //!< RegType mask. + }; + + //! \} + + //! \name Members + //! \{ + + uint32_t _data; + + //! \} + + //! \name Initializtion & Reset + //! + //! These initialize the whole `FuncValue` to either register or stack. Useful when you know all of these + //! properties and wanna just set it up. + //! + //! \{ + + //! Initializes the `typeId` of this `FuncValue`. + inline void initTypeId(TypeId typeId) noexcept { + _data = uint32_t(typeId) << kTypeIdShift; + } + + inline void initReg(RegType regType, uint32_t regId, TypeId typeId, uint32_t flags = 0) noexcept { + _data = (uint32_t(regType) << kRegTypeShift) | (regId << kRegIdShift) | (uint32_t(typeId) << kTypeIdShift) | kFlagIsReg | flags; + } + + inline void initStack(int32_t offset, TypeId typeId) noexcept { + _data = (uint32_t(offset) << kStackOffsetShift) | (uint32_t(typeId) << kTypeIdShift) | kFlagIsStack; + } + + //! Resets the value to its unassigned state. + inline void reset() noexcept { _data = 0; } + + //! \} + + //! \name Assign + //! + //! These initialize only part of `FuncValue`, useful when building `FuncValue` incrementally. The caller + //! should first init the type-id by caliing `initTypeId` and then continue building either register or stack. + //! + //! \{ + + inline void assignRegData(RegType regType, uint32_t regId) noexcept { + ASMJIT_ASSERT((_data & (kRegTypeMask | kRegIdMask)) == 0); + _data |= (uint32_t(regType) << kRegTypeShift) | (regId << kRegIdShift) | kFlagIsReg; + } + + inline void assignStackOffset(int32_t offset) noexcept { + ASMJIT_ASSERT((_data & kStackOffsetMask) == 0); + _data |= (uint32_t(offset) << kStackOffsetShift) | kFlagIsStack; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns true if the value is initialized (explicit bool cast). + inline explicit operator bool() const noexcept { return _data != 0; } + + inline void _replaceValue(uint32_t mask, uint32_t value) noexcept { _data = (_data & ~mask) | value; } + + //! Tests whether the `FuncValue` has a flag `flag` set. + inline bool hasFlag(uint32_t flag) const noexcept { return Support::test(_data, flag); } + //! Adds `flags` to `FuncValue`. + inline void addFlags(uint32_t flags) noexcept { _data |= flags; } + //! Clears `flags` of `FuncValue`. + inline void clearFlags(uint32_t flags) noexcept { _data &= ~flags; } + + //! Tests whether the value is initialized (i.e. contains a valid data). + inline bool isInitialized() const noexcept { return _data != 0; } + //! Tests whether the argument is passed by register. + inline bool isReg() const noexcept { return hasFlag(kFlagIsReg); } + //! Tests whether the argument is passed by stack. + inline bool isStack() const noexcept { return hasFlag(kFlagIsStack); } + //! Tests whether the argument is passed by register. + inline bool isAssigned() const noexcept { return hasFlag(kFlagIsReg | kFlagIsStack); } + //! Tests whether the argument is passed through a pointer (used by WIN64 to pass XMM|YMM|ZMM). + inline bool isIndirect() const noexcept { return hasFlag(kFlagIsIndirect); } + + //! Tests whether the argument was already processed (used internally). + inline bool isDone() const noexcept { return hasFlag(kFlagIsDone); } + + //! Returns a register type of the register used to pass function argument or return value. + inline RegType regType() const noexcept { return RegType((_data & kRegTypeMask) >> kRegTypeShift); } + //! Sets a register type of the register used to pass function argument or return value. + inline void setRegType(RegType regType) noexcept { _replaceValue(kRegTypeMask, uint32_t(regType) << kRegTypeShift); } + + //! Returns a physical id of the register used to pass function argument or return value. + inline uint32_t regId() const noexcept { return (_data & kRegIdMask) >> kRegIdShift; } + //! Sets a physical id of the register used to pass function argument or return value. + inline void setRegId(uint32_t regId) noexcept { _replaceValue(kRegIdMask, regId << kRegIdShift); } + + //! Returns a stack offset of this argument. + inline int32_t stackOffset() const noexcept { return int32_t(_data & kStackOffsetMask) >> kStackOffsetShift; } + //! Sets a stack offset of this argument. + inline void setStackOffset(int32_t offset) noexcept { _replaceValue(kStackOffsetMask, uint32_t(offset) << kStackOffsetShift); } + + //! Tests whether the argument or return value has associated `TypeId`. + inline bool hasTypeId() const noexcept { return Support::test(_data, kTypeIdMask); } + //! Returns a TypeId of this argument or return value. + inline TypeId typeId() const noexcept { return TypeId((_data & kTypeIdMask) >> kTypeIdShift); } + //! Sets a TypeId of this argument or return value. + inline void setTypeId(TypeId typeId) noexcept { _replaceValue(kTypeIdMask, uint32_t(typeId) << kTypeIdShift); } + + //! \} +}; + +//! Contains multiple `FuncValue` instances in an array so functions that use multiple registers for arguments or +//! return values can represent all inputs and outputs. +struct FuncValuePack { +public: + //! \name Members + //! \{ + + //! Values of the pack. + FuncValue _values[Globals::kMaxValuePack]; + + //! \} + + //! \name Initialization & Reset + //! \{ + + //! Resets all values in the pack. + inline void reset() noexcept { + for (size_t i = 0; i < Globals::kMaxValuePack; i++) + _values[i].reset(); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Calculates how many values are in the pack, checking for non-values from the end. + inline uint32_t count() const noexcept { + uint32_t n = Globals::kMaxValuePack; + while (n && !_values[n - 1]) + n--; + return n; + } + + inline FuncValue* values() noexcept { return _values; } + inline const FuncValue* values() const noexcept { return _values; } + + inline void resetValue(size_t index) noexcept { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + _values[index].reset(); + } + + inline bool hasValue(size_t index) noexcept { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + return _values[index].isInitialized(); + } + + inline void assignReg(size_t index, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + ASMJIT_ASSERT(reg.isPhysReg()); + _values[index].initReg(reg.type(), reg.id(), typeId); + } + + inline void assignReg(size_t index, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + _values[index].initReg(regType, regId, typeId); + } + + inline void assignStack(size_t index, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + _values[index].initStack(offset, typeId); + } + + inline FuncValue& operator[](size_t index) { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + return _values[index]; + } + + inline const FuncValue& operator[](size_t index) const { + ASMJIT_ASSERT(index < Globals::kMaxValuePack); + return _values[index]; + } + + //! \} +}; + +//! Attributes are designed in a way that all are initially false, and user or \ref FuncFrame finalizer adds +//! them when necessary. +enum class FuncAttributes : uint32_t { + //! No attributes. + kNoAttributes = 0, + + //! Function has variable number of arguments. + kHasVarArgs = 0x00000001u, + //! Preserve frame pointer (don't omit FP). + kHasPreservedFP = 0x00000010u, + //! Function calls other functions (is not leaf). + kHasFuncCalls = 0x00000020u, + //! Function has aligned save/restore of vector registers. + kAlignedVecSR = 0x00000040u, + //! FuncFrame is finalized and can be used by prolog/epilog inserter (PEI). + kIsFinalized = 0x00000800u, + + // X86 Specific Attributes + // ----------------------- + + //! Enables the use of AVX within the function's body, prolog, and epilog (X86). + //! + //! This flag instructs prolog and epilog emitter to use AVX instead of SSE for manipulating XMM registers. + kX86_AVXEnabled = 0x00010000u, + + //! Enables the use of AVX-512 within the function's body, prolog, and epilog (X86). + //! + //! This flag instructs Compiler register allocator to use additional 16 registers introduced by AVX-512. + //! Additionally, if the functions saves full width of ZMM registers (custom calling conventions only) then + //! the prolog/epilog inserter would use AVX-512 move instructions to emit the save and restore sequence. + kX86_AVX512Enabled = 0x00020000u, + + //! This flag instructs the epilog writer to emit EMMS instruction before RET (X86). + kX86_MMXCleanup = 0x00040000u, + + //! This flag instructs the epilog writer to emit VZEROUPPER instruction before RET (X86). + kX86_AVXCleanup = 0x00080000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(FuncAttributes) + +//! Function detail - \ref CallConv and expanded \ref FuncSignature. +//! +//! Function detail is architecture and OS dependent representation of a function. It contains a materialized +//! calling convention and expanded function signature so all arguments have assigned either register type/id +//! or stack address. +class FuncDetail { +public: + //! \name Constants + //! \{ + + enum : uint8_t { + //! Doesn't have variable number of arguments (`...`). + kNoVarArgs = 0xFFu + }; + + //! \} + + //! \name Members + //! \{ + + //! Calling convention. + CallConv _callConv; + //! Number of function arguments. + uint8_t _argCount; + //! Variable arguments index of `kNoVarArgs`. + uint8_t _vaIndex; + //! Reserved for future use. + uint16_t _reserved; + //! Registers that contain arguments. + Support::Array _usedRegs; + //! Size of arguments passed by stack. + uint32_t _argStackSize; + //! Function return value(s). + FuncValuePack _rets; + //! Function arguments. + FuncValuePack _args[Globals::kMaxFuncArgs]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline FuncDetail() noexcept { reset(); } + inline FuncDetail(const FuncDetail& other) noexcept = default; + + //! Initializes this `FuncDetail` to the given signature. + ASMJIT_API Error init(const FuncSignature& signature, const Environment& environment) noexcept; + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the function's calling convention, see `CallConv`. + inline const CallConv& callConv() const noexcept { return _callConv; } + + //! Returns the associated calling convention flags, see `CallConv::Flags`. + inline CallConvFlags flags() const noexcept { return _callConv.flags(); } + //! Checks whether a CallConv `flag` is set, see `CallConv::Flags`. + inline bool hasFlag(CallConvFlags ccFlag) const noexcept { return _callConv.hasFlag(ccFlag); } + + //! Tests whether the function has a return value. + inline bool hasRet() const noexcept { return bool(_rets[0]); } + //! Returns the number of function arguments. + inline uint32_t argCount() const noexcept { return _argCount; } + + //! Returns function return values. + inline FuncValuePack& retPack() noexcept { return _rets; } + //! Returns function return values. + inline const FuncValuePack& retPack() const noexcept { return _rets; } + + //! Returns a function return value associated with the given `valueIndex`. + inline FuncValue& ret(size_t valueIndex = 0) noexcept { return _rets[valueIndex]; } + //! Returns a function return value associated with the given `valueIndex` (const). + inline const FuncValue& ret(size_t valueIndex = 0) const noexcept { return _rets[valueIndex]; } + + //! Returns function argument packs array. + inline FuncValuePack* argPacks() noexcept { return _args; } + //! Returns function argument packs array (const). + inline const FuncValuePack* argPacks() const noexcept { return _args; } + + //! Returns function argument pack at the given `argIndex`. + inline FuncValuePack& argPack(size_t argIndex) noexcept { + ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); + return _args[argIndex]; + } + + //! Returns function argument pack at the given `argIndex` (const). + inline const FuncValuePack& argPack(size_t argIndex) const noexcept { + ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); + return _args[argIndex]; + } + + //! Returns an argument at `valueIndex` from the argument pack at the given `argIndex`. + inline FuncValue& arg(size_t argIndex, size_t valueIndex = 0) noexcept { + ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); + return _args[argIndex][valueIndex]; + } + + //! Returns an argument at `valueIndex` from the argument pack at the given `argIndex` (const). + inline const FuncValue& arg(size_t argIndex, size_t valueIndex = 0) const noexcept { + ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); + return _args[argIndex][valueIndex]; + } + + //! Resets an argument at the given `argIndex`. + //! + //! If the argument is a parameter pack (has multiple values) all values are reset. + inline void resetArg(size_t argIndex) noexcept { + ASMJIT_ASSERT(argIndex < Globals::kMaxFuncArgs); + _args[argIndex].reset(); + } + + //! Tests whether the function has variable arguments. + inline bool hasVarArgs() const noexcept { return _vaIndex != kNoVarArgs; } + //! Returns an index of a first variable argument. + inline uint32_t vaIndex() const noexcept { return _vaIndex; } + + //! Tests whether the function passes one or more argument by stack. + inline bool hasStackArgs() const noexcept { return _argStackSize != 0; } + //! Returns stack size needed for function arguments passed on the stack. + inline uint32_t argStackSize() const noexcept { return _argStackSize; } + + //! Returns red zone size. + inline uint32_t redZoneSize() const noexcept { return _callConv.redZoneSize(); } + //! Returns spill zone size. + inline uint32_t spillZoneSize() const noexcept { return _callConv.spillZoneSize(); } + //! Returns natural stack alignment. + inline uint32_t naturalStackAlignment() const noexcept { return _callConv.naturalStackAlignment(); } + + //! Returns a mask of all passed registers of the given register `group`. + inline RegMask passedRegs(RegGroup group) const noexcept { return _callConv.passedRegs(group); } + //! Returns a mask of all preserved registers of the given register `group`. + inline RegMask preservedRegs(RegGroup group) const noexcept { return _callConv.preservedRegs(group); } + + //! Returns a mask of all used registers of the given register `group`. + inline RegMask usedRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _usedRegs[size_t(group)]; + } + + //! Adds `regs` to the mask of used registers of the given register `group`. + inline void addUsedRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _usedRegs[size_t(group)] |= regs; + } + + //! \} +}; + +//! Function frame. +//! +//! Function frame is used directly by prolog and epilog insertion (PEI) utils. It provides information necessary to +//! insert a proper and ABI comforming prolog and epilog. Function frame calculation is based on `CallConv` and +//! other function attributes. +//! +//! SSE vs AVX vs AVX-512 +//! --------------------- +//! +//! Function frame provides a way to tell prolog/epilog inserter to use AVX instructions instead of SSE. Use +//! `setAvxEnabled()` and `setAvx512Enabled()` to enable AVX and/or AVX-512, respectively. Enabling AVX-512 +//! is mostly for Compiler as it would use 32 SIMD registers instead of 16 when enabled. +//! +//! \note If your code uses AVX instructions and AVX is not enabled there would be a performance hit in case that +//! some registers had to be saved/restored in function's prolog/epilog, respectively. Thus, it's recommended to +//! always let the function frame know about the use of AVX. +//! +//! Function Frame Structure +//! ------------------------ +//! +//! Various properties can contribute to the size and structure of the function frame. The function frame in most +//! cases won't use all of the properties illustrated (for example Spill Zone and Red Zone are never used together). +//! +//! ``` +//! +-----------------------------+ +//! | Arguments Passed by Stack | +//! +-----------------------------+ +//! | Spill Zone | +//! +-----------------------------+ <- Stack offset (args) starts from here. +//! | Return Address, if Pushed | +//! +-----------------------------+ <- Stack pointer (SP) upon entry. +//! | Save/Restore Stack. | +//! +-----------------------------+-----------------------------+ +//! | Local Stack | | +//! +-----------------------------+ Final Stack | +//! | Call Stack | | +//! +-----------------------------+-----------------------------+ <- SP after prolog. +//! | Red Zone | +//! +-----------------------------+ +//! ``` +class FuncFrame { +public: + //! \name Constants + //! \{ + + enum : uint32_t { + //! Tag used to inform that some offset is invalid. + kTagInvalidOffset = 0xFFFFFFFFu + }; + + //! \} + + //! \name Members + //! \{ + + //! Function attributes. + FuncAttributes _attributes; + + //! Target architecture. + Arch _arch; + //! SP register ID (to access call stack and local stack). + uint8_t _spRegId; + //! SA register ID (to access stack arguments). + uint8_t _saRegId; + + //! Red zone size (copied from CallConv). + uint8_t _redZoneSize; + //! Spill zone size (copied from CallConv). + uint8_t _spillZoneSize; + //! Natural stack alignment (copied from CallConv). + uint8_t _naturalStackAlignment; + //! Minimum stack alignment to turn on dynamic alignment. + uint8_t _minDynamicAlignment; + + //! Call stack alignment. + uint8_t _callStackAlignment; + //! Local stack alignment. + uint8_t _localStackAlignment; + //! Final stack alignment. + uint8_t _finalStackAlignment; + + //! Adjustment of the stack before returning (X86-STDCALL). + uint16_t _calleeStackCleanup; + + //! Call stack size. + uint32_t _callStackSize; + //! Local stack size. + uint32_t _localStackSize; + //! Final stack size (sum of call stack and local stack). + uint32_t _finalStackSize; + + //! Local stack offset (non-zero only if call stack is used). + uint32_t _localStackOffset; + //! Offset relative to SP that contains previous SP (before alignment). + uint32_t _daOffset; + //! Offset of the first stack argument relative to SP. + uint32_t _saOffsetFromSP; + //! Offset of the first stack argument relative to SA (_saRegId or FP). + uint32_t _saOffsetFromSA; + + //! Local stack adjustment in prolog/epilog. + uint32_t _stackAdjustment; + + //! Registers that are dirty. + Support::Array _dirtyRegs; + //! Registers that must be preserved (copied from CallConv). + Support::Array _preservedRegs; + //! Size to save/restore per register group. + Support::Array _saveRestoreRegSize; + //! Alignment of save/restore area per register group. + Support::Array _saveRestoreAlignment; + + //! Stack size required to save registers with push/pop. + uint16_t _pushPopSaveSize; + //! Stack size required to save extra registers that cannot use push/pop. + uint16_t _extraRegSaveSize; + //! Offset where registers saved/restored via push/pop are stored + uint32_t _pushPopSaveOffset; + //! Offset where extra ragisters that cannot use push/pop are stored. + uint32_t _extraRegSaveOffset; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline FuncFrame() noexcept { reset(); } + inline FuncFrame(const FuncFrame& other) noexcept = default; + + ASMJIT_API Error init(const FuncDetail& func) noexcept; + + inline void reset() noexcept { + memset(this, 0, sizeof(FuncFrame)); + _spRegId = BaseReg::kIdBad; + _saRegId = BaseReg::kIdBad; + _daOffset = kTagInvalidOffset; + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the target architecture of the function frame. + inline Arch arch() const noexcept { return _arch; } + + //! Returns function frame attributes, see `Attributes`. + inline FuncAttributes attributes() const noexcept { return _attributes; } + //! Checks whether the FuncFame contains an attribute `attr`. + inline bool hasAttribute(FuncAttributes attr) const noexcept { return Support::test(_attributes, attr); } + //! Adds attributes `attrs` to the FuncFrame. + inline void addAttributes(FuncAttributes attrs) noexcept { _attributes |= attrs; } + //! Clears attributes `attrs` from the FrameFrame. + inline void clearAttributes(FuncAttributes attrs) noexcept { _attributes &= ~attrs; } + + //! Tests whether the function has variable number of arguments. + inline bool hasVarArgs() const noexcept { return hasAttribute(FuncAttributes::kHasVarArgs); } + //! Sets the variable arguments flag. + inline void setVarArgs() noexcept { addAttributes(FuncAttributes::kHasVarArgs); } + //! Resets variable arguments flag. + inline void resetVarArgs() noexcept { clearAttributes(FuncAttributes::kHasVarArgs); } + + //! Tests whether the function preserves frame pointer (EBP|ESP on X86). + inline bool hasPreservedFP() const noexcept { return hasAttribute(FuncAttributes::kHasPreservedFP); } + //! Enables preserved frame pointer. + inline void setPreservedFP() noexcept { addAttributes(FuncAttributes::kHasPreservedFP); } + //! Disables preserved frame pointer. + inline void resetPreservedFP() noexcept { clearAttributes(FuncAttributes::kHasPreservedFP); } + + //! Tests whether the function calls other functions. + inline bool hasFuncCalls() const noexcept { return hasAttribute(FuncAttributes::kHasFuncCalls); } + //! Sets `kFlagHasCalls` to true. + inline void setFuncCalls() noexcept { addAttributes(FuncAttributes::kHasFuncCalls); } + //! Sets `kFlagHasCalls` to false. + inline void resetFuncCalls() noexcept { clearAttributes(FuncAttributes::kHasFuncCalls); } + + //! Tests whether the function has AVX enabled. + inline bool isAvxEnabled() const noexcept { return hasAttribute(FuncAttributes::kX86_AVXEnabled); } + //! Enables AVX use. + inline void setAvxEnabled() noexcept { addAttributes(FuncAttributes::kX86_AVXEnabled); } + //! Disables AVX use. + inline void resetAvxEnabled() noexcept { clearAttributes(FuncAttributes::kX86_AVXEnabled); } + + //! Tests whether the function has AVX-512 enabled. + inline bool isAvx512Enabled() const noexcept { return hasAttribute(FuncAttributes::kX86_AVX512Enabled); } + //! Enables AVX-512 use. + inline void setAvx512Enabled() noexcept { addAttributes(FuncAttributes::kX86_AVX512Enabled); } + //! Disables AVX-512 use. + inline void resetAvx512Enabled() noexcept { clearAttributes(FuncAttributes::kX86_AVX512Enabled); } + + //! Tests whether the function has MMX cleanup - 'emms' instruction in epilog. + inline bool hasMmxCleanup() const noexcept { return hasAttribute(FuncAttributes::kX86_MMXCleanup); } + //! Enables MMX cleanup. + inline void setMmxCleanup() noexcept { addAttributes(FuncAttributes::kX86_MMXCleanup); } + //! Disables MMX cleanup. + inline void resetMmxCleanup() noexcept { clearAttributes(FuncAttributes::kX86_MMXCleanup); } + + //! Tests whether the function has AVX cleanup - 'vzeroupper' instruction in epilog. + inline bool hasAvxCleanup() const noexcept { return hasAttribute(FuncAttributes::kX86_AVXCleanup); } + //! Enables AVX cleanup. + inline void setAvxCleanup() noexcept { addAttributes(FuncAttributes::kX86_AVXCleanup); } + //! Disables AVX cleanup. + inline void resetAvxCleanup() noexcept { clearAttributes(FuncAttributes::kX86_AVXCleanup); } + + //! Tests whether the function uses call stack. + inline bool hasCallStack() const noexcept { return _callStackSize != 0; } + //! Tests whether the function uses local stack. + inline bool hasLocalStack() const noexcept { return _localStackSize != 0; } + //! Tests whether vector registers can be saved and restored by using aligned reads and writes. + inline bool hasAlignedVecSR() const noexcept { return hasAttribute(FuncAttributes::kAlignedVecSR); } + //! Tests whether the function has to align stack dynamically. + inline bool hasDynamicAlignment() const noexcept { return _finalStackAlignment >= _minDynamicAlignment; } + + //! Tests whether the calling convention specifies 'RedZone'. + inline bool hasRedZone() const noexcept { return _redZoneSize != 0; } + //! Tests whether the calling convention specifies 'SpillZone'. + inline bool hasSpillZone() const noexcept { return _spillZoneSize != 0; } + + //! Returns the size of 'RedZone'. + inline uint32_t redZoneSize() const noexcept { return _redZoneSize; } + //! Returns the size of 'SpillZone'. + inline uint32_t spillZoneSize() const noexcept { return _spillZoneSize; } + //! Returns natural stack alignment (guaranteed stack alignment upon entry). + inline uint32_t naturalStackAlignment() const noexcept { return _naturalStackAlignment; } + //! Returns natural stack alignment (guaranteed stack alignment upon entry). + inline uint32_t minDynamicAlignment() const noexcept { return _minDynamicAlignment; } + + //! Tests whether the callee must adjust SP before returning (X86-STDCALL only) + inline bool hasCalleeStackCleanup() const noexcept { return _calleeStackCleanup != 0; } + //! Returns home many bytes of the stack the callee must adjust before returning (X86-STDCALL only) + inline uint32_t calleeStackCleanup() const noexcept { return _calleeStackCleanup; } + + //! Returns call stack alignment. + inline uint32_t callStackAlignment() const noexcept { return _callStackAlignment; } + //! Returns local stack alignment. + inline uint32_t localStackAlignment() const noexcept { return _localStackAlignment; } + //! Returns final stack alignment (the maximum value of call, local, and natural stack alignments). + inline uint32_t finalStackAlignment() const noexcept { return _finalStackAlignment; } + + //! Sets call stack alignment. + //! + //! \note This also updates the final stack alignment. + inline void setCallStackAlignment(uint32_t alignment) noexcept { + _callStackAlignment = uint8_t(alignment); + _finalStackAlignment = Support::max(_naturalStackAlignment, _callStackAlignment, _localStackAlignment); + } + + //! Sets local stack alignment. + //! + //! \note This also updates the final stack alignment. + inline void setLocalStackAlignment(uint32_t value) noexcept { + _localStackAlignment = uint8_t(value); + _finalStackAlignment = Support::max(_naturalStackAlignment, _callStackAlignment, _localStackAlignment); + } + + //! Combines call stack alignment with `alignment`, updating it to the greater value. + //! + //! \note This also updates the final stack alignment. + inline void updateCallStackAlignment(uint32_t alignment) noexcept { + _callStackAlignment = uint8_t(Support::max(_callStackAlignment, alignment)); + _finalStackAlignment = Support::max(_finalStackAlignment, _callStackAlignment); + } + + //! Combines local stack alignment with `alignment`, updating it to the greater value. + //! + //! \note This also updates the final stack alignment. + inline void updateLocalStackAlignment(uint32_t alignment) noexcept { + _localStackAlignment = uint8_t(Support::max(_localStackAlignment, alignment)); + _finalStackAlignment = Support::max(_finalStackAlignment, _localStackAlignment); + } + + //! Returns call stack size. + inline uint32_t callStackSize() const noexcept { return _callStackSize; } + //! Returns local stack size. + inline uint32_t localStackSize() const noexcept { return _localStackSize; } + + //! Sets call stack size. + inline void setCallStackSize(uint32_t size) noexcept { _callStackSize = size; } + //! Sets local stack size. + inline void setLocalStackSize(uint32_t size) noexcept { _localStackSize = size; } + + //! Combines call stack size with `size`, updating it to the greater value. + inline void updateCallStackSize(uint32_t size) noexcept { _callStackSize = Support::max(_callStackSize, size); } + //! Combines local stack size with `size`, updating it to the greater value. + inline void updateLocalStackSize(uint32_t size) noexcept { _localStackSize = Support::max(_localStackSize, size); } + + //! Returns final stack size (only valid after the FuncFrame is finalized). + inline uint32_t finalStackSize() const noexcept { return _finalStackSize; } + + //! Returns an offset to access the local stack (non-zero only if call stack is used). + inline uint32_t localStackOffset() const noexcept { return _localStackOffset; } + + //! Tests whether the function prolog/epilog requires a memory slot for storing unaligned SP. + inline bool hasDAOffset() const noexcept { return _daOffset != kTagInvalidOffset; } + //! Returns a memory offset used to store DA (dynamic alignment) slot (relative to SP). + inline uint32_t daOffset() const noexcept { return _daOffset; } + + inline uint32_t saOffset(uint32_t regId) const noexcept { + return regId == _spRegId ? saOffsetFromSP() + : saOffsetFromSA(); + } + + inline uint32_t saOffsetFromSP() const noexcept { return _saOffsetFromSP; } + inline uint32_t saOffsetFromSA() const noexcept { return _saOffsetFromSA; } + + //! Returns mask of registers of the given register `group` that are modified by the function. The engine would + //! then calculate which registers must be saved & restored by the function by using the data provided by the + //! calling convention. + inline RegMask dirtyRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _dirtyRegs[group]; + } + + //! Sets which registers (as a mask) are modified by the function. + //! + //! \remarks Please note that this will completely overwrite the existing register mask, use `addDirtyRegs()` + //! to modify the existing register mask. + inline void setDirtyRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _dirtyRegs[group] = regs; + } + + //! Adds which registers (as a mask) are modified by the function. + inline void addDirtyRegs(RegGroup group, RegMask regs) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _dirtyRegs[group] |= regs; + } + + //! \overload + inline void addDirtyRegs(const BaseReg& reg) noexcept { + ASMJIT_ASSERT(reg.id() < Globals::kMaxPhysRegs); + addDirtyRegs(reg.group(), Support::bitMask(reg.id())); + } + + //! \overload + template + inline void addDirtyRegs(const BaseReg& reg, Args&&... args) noexcept { + addDirtyRegs(reg); + addDirtyRegs(std::forward(args)...); + } + + inline void setAllDirty() noexcept { + for (size_t i = 0; i < ASMJIT_ARRAY_SIZE(_dirtyRegs); i++) + _dirtyRegs[i] = 0xFFFFFFFFu; + } + + inline void setAllDirty(RegGroup group) noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + _dirtyRegs[group] = 0xFFFFFFFFu; + } + + //! Returns a calculated mask of registers of the given `group` that will be saved and restored in the function's + //! prolog and epilog, respectively. The register mask is calculated from both `dirtyRegs` (provided by user) and + //! `preservedMask` (provided by the calling convention). + inline RegMask savedRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _dirtyRegs[group] & _preservedRegs[group]; + } + + //! Returns the mask of preserved registers of the given register `group`. + //! + //! Preserved registers are those that must survive the function call unmodified. The function can only modify + //! preserved registers it they are saved and restored in funciton's prolog and epilog, respectively. + inline RegMask preservedRegs(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _preservedRegs[group]; + } + + inline uint32_t saveRestoreRegSize(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _saveRestoreRegSize[group]; + } + + inline uint32_t saveRestoreAlignment(RegGroup group) const noexcept { + ASMJIT_ASSERT(group <= RegGroup::kMaxVirt); + return _saveRestoreAlignment[group]; + } + + inline bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } + inline uint32_t saRegId() const noexcept { return _saRegId; } + inline void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } + inline void resetSARegId() { setSARegId(BaseReg::kIdBad); } + + //! Returns stack size required to save/restore registers via push/pop. + inline uint32_t pushPopSaveSize() const noexcept { return _pushPopSaveSize; } + //! Returns an offset to the stack where registers are saved via push/pop. + inline uint32_t pushPopSaveOffset() const noexcept { return _pushPopSaveOffset; } + + //! Returns stack size required to save/restore extra registers that don't use push/pop/ + //! + //! \note On X86 this covers all registers except GP registers, on other architectures it can be always + //! zero (for example AArch64 saves all registers via push/pop like instructions, so this would be zero). + inline uint32_t extraRegSaveSize() const noexcept { return _extraRegSaveSize; } + //! Returns an offset to the stack where extra registers are saved. + inline uint32_t extraRegSaveOffset() const noexcept { return _extraRegSaveOffset; } + + //! Tests whether the functions contains stack adjustment. + inline bool hasStackAdjustment() const noexcept { return _stackAdjustment != 0; } + //! Returns function's stack adjustment used in function's prolog and epilog. + //! + //! If the returned value is zero it means that the stack is not adjusted. This can mean both that the stack + //! is not used and/or the stack is only adjusted by instructions that pust/pop registers into/from stack. + inline uint32_t stackAdjustment() const noexcept { return _stackAdjustment; } + + //! \} + + //! \name Finaliztion + //! \{ + + ASMJIT_API Error finalize() noexcept; + + //! \} +}; + +//! A helper class that can be used to assign a physical register for each function argument. Use with +//! `BaseEmitter::emitArgsAssignment()`. +class FuncArgsAssignment { +public: + //! \name Members + //! \{ + + //! Function detail. + const FuncDetail* _funcDetail; + //! Register that can be used to access arguments passed by stack. + uint8_t _saRegId; + //! Reserved for future use. + uint8_t _reserved[3]; + //! Mapping of each function argument. + FuncValuePack _argPacks[Globals::kMaxFuncArgs]; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline explicit FuncArgsAssignment(const FuncDetail* fd = nullptr) noexcept { reset(fd); } + + inline FuncArgsAssignment(const FuncArgsAssignment& other) noexcept { + memcpy(this, &other, sizeof(*this)); + } + + inline void reset(const FuncDetail* fd = nullptr) noexcept { + _funcDetail = fd; + _saRegId = uint8_t(BaseReg::kIdBad); + memset(_reserved, 0, sizeof(_reserved)); + memset(_argPacks, 0, sizeof(_argPacks)); + } + + //! \} + + //! \name Accessors + //! \{ + + inline const FuncDetail* funcDetail() const noexcept { return _funcDetail; } + inline void setFuncDetail(const FuncDetail* fd) noexcept { _funcDetail = fd; } + + inline bool hasSARegId() const noexcept { return _saRegId != BaseReg::kIdBad; } + inline uint32_t saRegId() const noexcept { return _saRegId; } + inline void setSARegId(uint32_t regId) { _saRegId = uint8_t(regId); } + inline void resetSARegId() { _saRegId = uint8_t(BaseReg::kIdBad); } + + inline FuncValue& arg(size_t argIndex, size_t valueIndex) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + return _argPacks[argIndex][valueIndex]; + } + inline const FuncValue& arg(size_t argIndex, size_t valueIndex) const noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + return _argPacks[argIndex][valueIndex]; + } + + inline bool isAssigned(size_t argIndex, size_t valueIndex) const noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + return _argPacks[argIndex][valueIndex].isAssigned(); + } + + inline void assignReg(size_t argIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + ASMJIT_ASSERT(reg.isPhysReg()); + _argPacks[argIndex][0].initReg(reg.type(), reg.id(), typeId); + } + + inline void assignReg(size_t argIndex, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + _argPacks[argIndex][0].initReg(regType, regId, typeId); + } + + inline void assignStack(size_t argIndex, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + _argPacks[argIndex][0].initStack(offset, typeId); + } + + inline void assignRegInPack(size_t argIndex, size_t valueIndex, const BaseReg& reg, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + ASMJIT_ASSERT(reg.isPhysReg()); + _argPacks[argIndex][valueIndex].initReg(reg.type(), reg.id(), typeId); + } + + inline void assignRegInPack(size_t argIndex, size_t valueIndex, RegType regType, uint32_t regId, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + _argPacks[argIndex][valueIndex].initReg(regType, regId, typeId); + } + + inline void assignStackInPack(size_t argIndex, size_t valueIndex, int32_t offset, TypeId typeId = TypeId::kVoid) noexcept { + ASMJIT_ASSERT(argIndex < ASMJIT_ARRAY_SIZE(_argPacks)); + _argPacks[argIndex][valueIndex].initStack(offset, typeId); + } + + // NOTE: All `assignAll()` methods are shortcuts to assign all arguments at once, however, since registers are + // passed all at once these initializers don't provide any way to pass TypeId and/or to keep any argument between + // the arguments passed unassigned. + inline void _assignAllInternal(size_t argIndex, const BaseReg& reg) noexcept { + assignReg(argIndex, reg); + } + + template + inline void _assignAllInternal(size_t argIndex, const BaseReg& reg, Args&&... args) noexcept { + assignReg(argIndex, reg); + _assignAllInternal(argIndex + 1, std::forward(args)...); + } + + template + inline void assignAll(Args&&... args) noexcept { + _assignAllInternal(0, std::forward(args)...); + } + + //! \} + + //! \name Utilities + //! \{ + + //! Update `FuncFrame` based on function's arguments assignment. + //! + //! \note You MUST call this in orher to use `BaseEmitter::emitArgsAssignment()`, otherwise the FuncFrame would + //! not contain the information necessary to assign all arguments into the registers and/or stack specified. + ASMJIT_API Error updateFuncFrame(FuncFrame& frame) const noexcept; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FUNC_H_INCLUDED + diff --git a/src/asmjit/core/funcargscontext.cpp b/src/asmjit/core/funcargscontext.cpp new file mode 100644 index 0000000..1db50a7 --- /dev/null +++ b/src/asmjit/core/funcargscontext.cpp @@ -0,0 +1,293 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/funcargscontext_p.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_core +//! \{ + +FuncArgsContext::FuncArgsContext() noexcept { + for (RegGroup group : RegGroupVirtValues{}) + _workData[size_t(group)].reset(); +} + +ASMJIT_FAVOR_SIZE Error FuncArgsContext::initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args, const RAConstraints* constraints) noexcept { + Arch arch = frame.arch(); + const FuncDetail& func = *args.funcDetail(); + + _archTraits = &ArchTraits::byArch(arch); + _constraints = constraints; + _arch = arch; + + // Initialize `_archRegs`. + for (RegGroup group : RegGroupVirtValues{}) + _workData[group]._archRegs = _constraints->availableRegs(group); + + if (frame.hasPreservedFP()) + _workData[size_t(RegGroup::kGp)]._archRegs &= ~Support::bitMask(archTraits().fpRegId()); + + // Extract information from all function arguments/assignments and build Var[] array. + uint32_t varId = 0; + for (uint32_t argIndex = 0; argIndex < Globals::kMaxFuncArgs; argIndex++) { + for (uint32_t valueIndex = 0; valueIndex < Globals::kMaxValuePack; valueIndex++) { + const FuncValue& dst_ = args.arg(argIndex, valueIndex); + if (!dst_.isAssigned()) + continue; + + const FuncValue& src_ = func.arg(argIndex, valueIndex); + if (ASMJIT_UNLIKELY(!src_.isAssigned())) + return DebugUtils::errored(kErrorInvalidState); + + Var& var = _vars[varId]; + var.init(src_, dst_); + + FuncValue& src = var.cur; + FuncValue& dst = var.out; + + RegGroup dstGroup = RegGroup::kMaxValue; + uint32_t dstId = BaseReg::kIdBad; + WorkData* dstWd = nullptr; + + // Not supported. + if (src.isIndirect()) + return DebugUtils::errored(kErrorInvalidAssignment); + + if (dst.isReg()) { + RegType dstType = dst.regType(); + if (ASMJIT_UNLIKELY(!archTraits().hasRegType(dstType))) + return DebugUtils::errored(kErrorInvalidRegType); + + // Copy TypeId from source if the destination doesn't have it. The RA used by BaseCompiler would never + // leave TypeId undefined, but users of FuncAPI can just assign phys regs without specifying the type. + if (!dst.hasTypeId()) + dst.setTypeId(archTraits().regTypeToTypeId(dst.regType())); + + dstGroup = archTraits().regTypeToGroup(dstType); + if (ASMJIT_UNLIKELY(dstGroup > RegGroup::kMaxVirt)) + return DebugUtils::errored(kErrorInvalidRegGroup); + + dstWd = &_workData[dstGroup]; + dstId = dst.regId(); + if (ASMJIT_UNLIKELY(dstId >= 32 || !Support::bitTest(dstWd->archRegs(), dstId))) + return DebugUtils::errored(kErrorInvalidPhysId); + + if (ASMJIT_UNLIKELY(Support::bitTest(dstWd->dstRegs(), dstId))) + return DebugUtils::errored(kErrorOverlappedRegs); + + dstWd->_dstRegs |= Support::bitMask(dstId); + dstWd->_dstShuf |= Support::bitMask(dstId); + dstWd->_usedRegs |= Support::bitMask(dstId); + } + else { + if (!dst.hasTypeId()) + dst.setTypeId(src.typeId()); + + OperandSignature signature = getSuitableRegForMemToMemMove(arch, dst.typeId(), src.typeId()); + if (ASMJIT_UNLIKELY(!signature.isValid())) + return DebugUtils::errored(kErrorInvalidState); + _stackDstMask = uint8_t(_stackDstMask | Support::bitMask(signature.regGroup())); + } + + if (src.isReg()) { + uint32_t srcId = src.regId(); + RegGroup srcGroup = archTraits().regTypeToGroup(src.regType()); + + if (dstGroup == srcGroup) { + ASMJIT_ASSERT(dstWd != nullptr); + dstWd->assign(varId, srcId); + + // The best case, register is allocated where it is expected to be. + if (dstId == srcId) + var.markDone(); + } + else { + if (ASMJIT_UNLIKELY(srcGroup > RegGroup::kMaxVirt)) + return DebugUtils::errored(kErrorInvalidState); + + WorkData& srcData = _workData[size_t(srcGroup)]; + srcData.assign(varId, srcId); + } + } + else { + if (dstWd) + dstWd->_numStackArgs++; + _hasStackSrc = true; + } + + varId++; + } + } + + // Initialize WorkData::workRegs. + for (RegGroup group : RegGroupVirtValues{}) { + _workData[group]._workRegs = + (_workData[group].archRegs() & (frame.dirtyRegs(group) | ~frame.preservedRegs(group))) | _workData[group].dstRegs() | _workData[group].assignedRegs(); + } + + // Create a variable that represents `SARegId` if necessary. + bool saRegRequired = _hasStackSrc && frame.hasDynamicAlignment() && !frame.hasPreservedFP(); + + WorkData& gpRegs = _workData[RegGroup::kGp]; + uint32_t saCurRegId = frame.saRegId(); + uint32_t saOutRegId = args.saRegId(); + + if (saCurRegId != BaseReg::kIdBad) { + // Check if the provided `SARegId` doesn't collide with input registers. + if (ASMJIT_UNLIKELY(gpRegs.isAssigned(saCurRegId))) + return DebugUtils::errored(kErrorOverlappedRegs); + } + + if (saOutRegId != BaseReg::kIdBad) { + // Check if the provided `SARegId` doesn't collide with argument assignments. + if (ASMJIT_UNLIKELY(Support::bitTest(gpRegs.dstRegs(), saOutRegId))) + return DebugUtils::errored(kErrorOverlappedRegs); + saRegRequired = true; + } + + if (saRegRequired) { + TypeId ptrTypeId = Environment::is32Bit(arch) ? TypeId::kUInt32 : TypeId::kUInt64; + RegType ptrRegType = Environment::is32Bit(arch) ? RegType::kGp32 : RegType::kGp64; + + _saVarId = uint8_t(varId); + _hasPreservedFP = frame.hasPreservedFP(); + + Var& var = _vars[varId]; + var.reset(); + + if (saCurRegId == BaseReg::kIdBad) { + if (saOutRegId != BaseReg::kIdBad && !gpRegs.isAssigned(saOutRegId)) { + saCurRegId = saOutRegId; + } + else { + RegMask availableRegs = gpRegs.availableRegs(); + if (!availableRegs) + availableRegs = gpRegs.archRegs() & ~gpRegs.workRegs(); + + if (ASMJIT_UNLIKELY(!availableRegs)) + return DebugUtils::errored(kErrorNoMorePhysRegs); + + saCurRegId = Support::ctz(availableRegs); + } + } + + var.cur.initReg(ptrRegType, saCurRegId, ptrTypeId); + gpRegs.assign(varId, saCurRegId); + gpRegs._workRegs |= Support::bitMask(saCurRegId); + + if (saOutRegId != BaseReg::kIdBad) { + var.out.initReg(ptrRegType, saOutRegId, ptrTypeId); + gpRegs._dstRegs |= Support::bitMask(saOutRegId); + gpRegs._workRegs |= Support::bitMask(saOutRegId); + } + else { + var.markDone(); + } + + varId++; + } + + _varCount = varId; + + // Detect register swaps. + for (varId = 0; varId < _varCount; varId++) { + Var& var = _vars[varId]; + if (var.cur.isReg() && var.out.isReg()) { + uint32_t srcId = var.cur.regId(); + uint32_t dstId = var.out.regId(); + + RegGroup group = archTraits().regTypeToGroup(var.cur.regType()); + if (group != archTraits().regTypeToGroup(var.out.regType())) + continue; + + WorkData& wd = _workData[group]; + if (wd.isAssigned(dstId)) { + Var& other = _vars[wd._physToVarId[dstId]]; + if (archTraits().regTypeToGroup(other.out.regType()) == group && other.out.regId() == srcId) { + wd._numSwaps++; + _regSwapsMask = uint8_t(_regSwapsMask | Support::bitMask(group)); + } + } + } + } + + return kErrorOk; +} + +ASMJIT_FAVOR_SIZE Error FuncArgsContext::markDstRegsDirty(FuncFrame& frame) noexcept { + for (RegGroup group : RegGroupVirtValues{}) { + WorkData& wd = _workData[group]; + uint32_t regs = wd.usedRegs() | wd._dstShuf; + + wd._workRegs |= regs; + frame.addDirtyRegs(group, regs); + } + + return kErrorOk; +} + +ASMJIT_FAVOR_SIZE Error FuncArgsContext::markScratchRegs(FuncFrame& frame) noexcept { + uint32_t groupMask = 0; + + // Handle stack to stack moves. + groupMask |= _stackDstMask; + + // Handle register swaps. + groupMask |= _regSwapsMask & ~Support::bitMask(RegGroup::kGp); + + if (!groupMask) + return kErrorOk; + + // Selects one dirty register per affected group that can be used as a scratch register. + for (RegGroup group : RegGroupVirtValues{}) { + if (Support::bitTest(groupMask, group)) { + WorkData& wd = _workData[group]; + + // Initially, pick some clobbered or dirty register. + RegMask workRegs = wd.workRegs(); + RegMask regs = workRegs & ~(wd.usedRegs() | wd._dstShuf); + + // If that didn't work out pick some register which is not in 'used'. + if (!regs) + regs = workRegs & ~wd.usedRegs(); + + // If that didn't work out pick any other register that is allocable. + // This last resort case will, however, result in marking one more + // register dirty. + if (!regs) + regs = wd.archRegs() & ~workRegs; + + // If that didn't work out we will have to use XORs instead of MOVs. + if (!regs) + continue; + + RegMask regMask = Support::blsi(regs); + wd._workRegs |= regMask; + frame.addDirtyRegs(group, regMask); + } + } + + return kErrorOk; +} + +ASMJIT_FAVOR_SIZE Error FuncArgsContext::markStackArgsReg(FuncFrame& frame) noexcept { + if (_saVarId != kVarIdNone) { + const Var& var = _vars[_saVarId]; + frame.setSARegId(var.cur.regId()); + } + else if (frame.hasPreservedFP()) { + frame.setSARegId(archTraits().fpRegId()); + } + + return kErrorOk; +} + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/funcargscontext_p.h b/src/asmjit/core/funcargscontext_p.h new file mode 100644 index 0000000..72ee105 --- /dev/null +++ b/src/asmjit/core/funcargscontext_p.h @@ -0,0 +1,199 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_FUNCARGSCONTEXT_P_H_INCLUDED +#define ASMJIT_CORE_FUNCARGSCONTEXT_P_H_INCLUDED + +#include "../core/archtraits.h" +#include "../core/environment.h" +#include "../core/func.h" +#include "../core/operand.h" +#include "../core/radefs_p.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_core +//! \{ + +static inline OperandSignature getSuitableRegForMemToMemMove(Arch arch, TypeId dstTypeId, TypeId srcTypeId) noexcept { + const ArchTraits& archTraits = ArchTraits::byArch(arch); + + uint32_t dstSize = TypeUtils::sizeOf(dstTypeId); + uint32_t srcSize = TypeUtils::sizeOf(srcTypeId); + uint32_t maxSize = Support::max(dstSize, srcSize); + uint32_t regSize = Environment::registerSizeFromArch(arch); + + OperandSignature signature{0}; + if (maxSize <= regSize || (TypeUtils::isInt(dstTypeId) && TypeUtils::isInt(srcTypeId))) + signature = maxSize <= 4 ? archTraits.regTypeToSignature(RegType::kGp32) + : archTraits.regTypeToSignature(RegType::kGp64); + else if (maxSize <= 8 && archTraits.hasRegType(RegType::kVec64)) + signature = archTraits.regTypeToSignature(RegType::kVec64); + else if (maxSize <= 16 && archTraits.hasRegType(RegType::kVec128)) + signature = archTraits.regTypeToSignature(RegType::kVec128); + else if (maxSize <= 32 && archTraits.hasRegType(RegType::kVec256)) + signature = archTraits.regTypeToSignature(RegType::kVec256); + else if (maxSize <= 64 && archTraits.hasRegType(RegType::kVec512)) + signature = archTraits.regTypeToSignature(RegType::kVec512); + + return signature; +} + +class FuncArgsContext { +public: + enum VarId : uint32_t { + kVarIdNone = 0xFF + }; + + //! Contains information about a single argument or SA register that may need shuffling. + struct Var { + FuncValue cur; + FuncValue out; + + inline void init(const FuncValue& cur_, const FuncValue& out_) noexcept { + cur = cur_; + out = out_; + } + + //! Reset the value to its unassigned state. + inline void reset() noexcept { + cur.reset(); + out.reset(); + } + + inline bool isDone() const noexcept { return cur.isDone(); } + inline void markDone() noexcept { cur.addFlags(FuncValue::kFlagIsDone); } + }; + + struct WorkData { + //! All allocable registers provided by the architecture. + RegMask _archRegs; + //! All registers that can be used by the shuffler. + RegMask _workRegs; + //! Registers used by the shuffler (all). + RegMask _usedRegs; + //! Assigned registers. + RegMask _assignedRegs; + //! Destination registers assigned to arguments or SA. + RegMask _dstRegs; + //! Destination registers that require shuffling. + RegMask _dstShuf; + //! Number of register swaps. + uint8_t _numSwaps; + //! Number of stack loads. + uint8_t _numStackArgs; + //! Reserved (only used as padding). + uint8_t _reserved[6]; + //! Physical ID to variable ID mapping. + uint8_t _physToVarId[32]; + + inline void reset() noexcept { + _archRegs = 0; + _workRegs = 0; + _usedRegs = 0; + _assignedRegs = 0; + _dstRegs = 0; + _dstShuf = 0; + _numSwaps = 0; + _numStackArgs = 0; + memset(_reserved, 0, sizeof(_reserved)); + memset(_physToVarId, kVarIdNone, 32); + } + + inline bool isAssigned(uint32_t regId) const noexcept { + ASMJIT_ASSERT(regId < 32); + return Support::bitTest(_assignedRegs, regId); + } + + inline void assign(uint32_t varId, uint32_t regId) noexcept { + ASMJIT_ASSERT(!isAssigned(regId)); + ASMJIT_ASSERT(_physToVarId[regId] == kVarIdNone); + + _physToVarId[regId] = uint8_t(varId); + _assignedRegs ^= Support::bitMask(regId); + } + + inline void reassign(uint32_t varId, uint32_t newId, uint32_t oldId) noexcept { + ASMJIT_ASSERT( isAssigned(oldId)); + ASMJIT_ASSERT(!isAssigned(newId)); + ASMJIT_ASSERT(_physToVarId[oldId] == varId); + ASMJIT_ASSERT(_physToVarId[newId] == kVarIdNone); + + _physToVarId[oldId] = uint8_t(kVarIdNone); + _physToVarId[newId] = uint8_t(varId); + _assignedRegs ^= Support::bitMask(newId) ^ Support::bitMask(oldId); + } + + inline void swap(uint32_t aVarId, uint32_t aRegId, uint32_t bVarId, uint32_t bRegId) noexcept { + ASMJIT_ASSERT(isAssigned(aRegId)); + ASMJIT_ASSERT(isAssigned(bRegId)); + ASMJIT_ASSERT(_physToVarId[aRegId] == aVarId); + ASMJIT_ASSERT(_physToVarId[bRegId] == bVarId); + + _physToVarId[aRegId] = uint8_t(bVarId); + _physToVarId[bRegId] = uint8_t(aVarId); + } + + inline void unassign(uint32_t varId, uint32_t regId) noexcept { + ASMJIT_ASSERT(isAssigned(regId)); + ASMJIT_ASSERT(_physToVarId[regId] == varId); + + DebugUtils::unused(varId); + _physToVarId[regId] = uint8_t(kVarIdNone); + _assignedRegs ^= Support::bitMask(regId); + } + + inline RegMask archRegs() const noexcept { return _archRegs; } + inline RegMask workRegs() const noexcept { return _workRegs; } + inline RegMask usedRegs() const noexcept { return _usedRegs; } + inline RegMask assignedRegs() const noexcept { return _assignedRegs; } + inline RegMask dstRegs() const noexcept { return _dstRegs; } + inline RegMask availableRegs() const noexcept { return _workRegs & ~_assignedRegs; } + }; + + //! Architecture traits. + const ArchTraits* _archTraits = nullptr; + //! Architecture constraints. + const RAConstraints* _constraints = nullptr; + //! Target architecture. + Arch _arch = Arch::kUnknown; + //! Has arguments passed via stack (SRC). + bool _hasStackSrc = false; + //! Has preserved frame-pointer (FP). + bool _hasPreservedFP = false; + //! Has arguments assigned to stack (DST). + uint8_t _stackDstMask = 0; + //! Register swap groups (bit-mask). + uint8_t _regSwapsMask = 0; + uint8_t _saVarId = kVarIdNone; + uint32_t _varCount = 0; + Support::Array _workData; + Var _vars[Globals::kMaxFuncArgs * Globals::kMaxValuePack + 1]; + + FuncArgsContext() noexcept; + + inline const ArchTraits& archTraits() const noexcept { return *_archTraits; } + inline Arch arch() const noexcept { return _arch; } + + inline uint32_t varCount() const noexcept { return _varCount; } + inline size_t indexOf(const Var* var) const noexcept { return (size_t)(var - _vars); } + + inline Var& var(size_t varId) noexcept { return _vars[varId]; } + inline const Var& var(size_t varId) const noexcept { return _vars[varId]; } + + Error initWorkData(const FuncFrame& frame, const FuncArgsAssignment& args, const RAConstraints* constraints) noexcept; + Error markScratchRegs(FuncFrame& frame) noexcept; + Error markDstRegsDirty(FuncFrame& frame) noexcept; + Error markStackArgsReg(FuncFrame& frame) noexcept; +}; + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_FUNCARGSCONTEXT_P_H_INCLUDED diff --git a/src/asmjit/core/globals.cpp b/src/asmjit/core/globals.cpp new file mode 100644 index 0000000..2bbd0c0 --- /dev/null +++ b/src/asmjit/core/globals.cpp @@ -0,0 +1,133 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/globals.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// DebugUtils - Error As String +// ============================ + +ASMJIT_FAVOR_SIZE const char* DebugUtils::errorAsString(Error err) noexcept { +#ifndef ASMJIT_NO_TEXT + // @EnumStringBegin{"enum": "ErrorCode", "output": "sError", "strip": "kError"}@ + static const char sErrorString[] = + "Ok\0" + "OutOfMemory\0" + "InvalidArgument\0" + "InvalidState\0" + "InvalidArch\0" + "NotInitialized\0" + "AlreadyInitialized\0" + "FeatureNotEnabled\0" + "TooManyHandles\0" + "TooLarge\0" + "NoCodeGenerated\0" + "InvalidDirective\0" + "InvalidLabel\0" + "TooManyLabels\0" + "LabelAlreadyBound\0" + "LabelAlreadyDefined\0" + "LabelNameTooLong\0" + "InvalidLabelName\0" + "InvalidParentLabel\0" + "InvalidSection\0" + "TooManySections\0" + "InvalidSectionName\0" + "TooManyRelocations\0" + "InvalidRelocEntry\0" + "RelocOffsetOutOfRange\0" + "InvalidAssignment\0" + "InvalidInstruction\0" + "InvalidRegType\0" + "InvalidRegGroup\0" + "InvalidPhysId\0" + "InvalidVirtId\0" + "InvalidElementIndex\0" + "InvalidPrefixCombination\0" + "InvalidLockPrefix\0" + "InvalidXAcquirePrefix\0" + "InvalidXReleasePrefix\0" + "InvalidRepPrefix\0" + "InvalidRexPrefix\0" + "InvalidExtraReg\0" + "InvalidKMaskUse\0" + "InvalidKZeroUse\0" + "InvalidBroadcast\0" + "InvalidEROrSAE\0" + "InvalidAddress\0" + "InvalidAddressIndex\0" + "InvalidAddressScale\0" + "InvalidAddress64Bit\0" + "InvalidAddress64BitZeroExtension\0" + "InvalidDisplacement\0" + "InvalidSegment\0" + "InvalidImmediate\0" + "InvalidOperandSize\0" + "AmbiguousOperandSize\0" + "OperandSizeMismatch\0" + "InvalidOption\0" + "OptionAlreadyDefined\0" + "InvalidTypeId\0" + "InvalidUseOfGpbHi\0" + "InvalidUseOfGpq\0" + "InvalidUseOfF80\0" + "NotConsecutiveRegs\0" + "ConsecutiveRegsAllocation\0" + "IllegalVirtReg\0" + "TooManyVirtRegs\0" + "NoMorePhysRegs\0" + "OverlappedRegs\0" + "OverlappingStackRegWithRegArg\0" + "ExpressionLabelNotBound\0" + "ExpressionOverflow\0" + "FailedToOpenAnonymousMemory\0" + "\0"; + + static const uint16_t sErrorIndex[] = { + 0, 3, 15, 31, 44, 56, 71, 90, 108, 123, 132, 148, 165, 178, 192, 210, 230, + 247, 264, 283, 298, 314, 333, 352, 370, 392, 410, 429, 444, 460, 474, 488, + 508, 533, 551, 573, 595, 612, 629, 645, 661, 677, 694, 709, 724, 744, 764, + 784, 817, 837, 852, 869, 888, 909, 929, 943, 964, 978, 996, 1012, 1028, 1047, + 1073, 1088, 1104, 1119, 1134, 1164, 1188, 1207, 1235 + }; + // @EnumStringEnd@ + + return sErrorString + sErrorIndex[Support::min(err, kErrorCount)]; +#else + DebugUtils::unused(err); + static const char noMessage[] = ""; + return noMessage; +#endif +} + +// DebugUtils - Debug Output +// ========================= + +ASMJIT_FAVOR_SIZE void DebugUtils::debugOutput(const char* str) noexcept { +#if defined(_WIN32) + ::OutputDebugStringA(str); +#else + ::fputs(str, stderr); +#endif +} + +// DebugUtils - Fatal Errors +// ========================= + +ASMJIT_FAVOR_SIZE void DebugUtils::assertionFailed(const char* file, int line, const char* msg) noexcept { + char str[1024]; + + snprintf(str, 1024, + "[asmjit] Assertion failed at %s (line %d):\n" + "[asmjit] %s\n", file, line, msg); + + debugOutput(str); + ::abort(); +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/globals.h b/src/asmjit/core/globals.h new file mode 100644 index 0000000..f2d3c6e --- /dev/null +++ b/src/asmjit/core/globals.h @@ -0,0 +1,393 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_GLOBALS_H_INCLUDED +#define ASMJIT_CORE_GLOBALS_H_INCLUDED + +#include "../core/api-config.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_utilities +//! \{ +namespace Support { + //! Cast designed to cast between function and void* pointers. + template + static inline Dst ptr_cast_impl(Src p) noexcept { return (Dst)p; } +} // {Support} + +#if defined(ASMJIT_NO_STDCXX) +namespace Support { + ASMJIT_FORCE_INLINE void* operatorNew(size_t n) noexcept { return malloc(n); } + ASMJIT_FORCE_INLINE void operatorDelete(void* p) noexcept { if (p) free(p); } +} // {Support} + +#define ASMJIT_BASE_CLASS(TYPE) \ + ASMJIT_FORCE_INLINE void* operator new(size_t n) noexcept { \ + return Support::operatorNew(n); \ + } \ + \ + ASMJIT_FORCE_INLINE void operator delete(void* p) noexcept { \ + Support::operatorDelete(p); \ + } \ + \ + ASMJIT_FORCE_INLINE void* operator new(size_t, void* p) noexcept { return p; } \ + ASMJIT_FORCE_INLINE void operator delete(void*, void*) noexcept {} +#else +#define ASMJIT_BASE_CLASS(TYPE) +#endif + +//! \} +//! \endcond + +//! \addtogroup asmjit_core +//! \{ + +//! Byte order. +enum class ByteOrder { + //! Little endian. + kLE = 0, + //! Big endian. + kBE = 1, + //! Native byte order of the target architecture. + kNative = ASMJIT_ARCH_LE ? kLE : kBE, + //! Swapped byte order of the target architecture. + kSwapped = ASMJIT_ARCH_LE ? kBE : kLE +}; + +//! A policy that can be used with some `reset()` member functions. +enum class ResetPolicy : uint32_t { + //! Soft reset, doesn't deallocate memory (default). + kSoft = 0, + //! Hard reset, releases all memory used, if any. + kHard = 1 +}; + +//! Contains typedefs, constants, and variables used globally by AsmJit. +namespace Globals { + +//! Host memory allocator overhead. +static constexpr uint32_t kAllocOverhead = uint32_t(sizeof(intptr_t) * 4); + +//! Host memory allocator alignment. +static constexpr uint32_t kAllocAlignment = 8; + +//! Aggressive growing strategy threshold. +static constexpr uint32_t kGrowThreshold = 1024 * 1024 * 16; + +//! Maximum depth of RB-Tree is: +//! +//! `2 * log2(n + 1)` +//! +//! Size of RB node is at least two pointers (without data), so a theoretical architecture limit would be: +//! +//! `2 * log2(addressableMemorySize / sizeof(Node) + 1)` +//! +//! Which yields 30 on 32-bit arch and 61 on 64-bit arch. The final value was adjusted by +1 for safety reasons. +static constexpr uint32_t kMaxTreeHeight = (ASMJIT_ARCH_BITS == 32 ? 30 : 61) + 1; + +//! Maximum number of operands per a single instruction. +static constexpr uint32_t kMaxOpCount = 6; + +//! Maximum arguments of a function supported by the Compiler / Function API. +static constexpr uint32_t kMaxFuncArgs = 16; + +//! The number of values that can be assigned to a single function argument or +//! return value. +static constexpr uint32_t kMaxValuePack = 4; + +//! Maximum number of physical registers AsmJit can use per register group. +static constexpr uint32_t kMaxPhysRegs = 32; + +//! Maximum alignment. +static constexpr uint32_t kMaxAlignment = 64; + +//! Maximum label or symbol size in bytes. +static constexpr uint32_t kMaxLabelNameSize = 2048; + +//! Maximum section name size. +static constexpr uint32_t kMaxSectionNameSize = 35; + +//! Maximum size of comment. +static constexpr uint32_t kMaxCommentSize = 1024; + +//! Invalid identifier. +static constexpr uint32_t kInvalidId = 0xFFFFFFFFu; + +//! Returned by `indexOf()` and similar when working with containers that use 32-bit index/size. +static constexpr uint32_t kNotFound = 0xFFFFFFFFu; + +//! Invalid base address. +static constexpr uint64_t kNoBaseAddress = ~uint64_t(0); + +//! Number of virtual register groups. +static constexpr uint32_t kNumVirtGroups = 4; + +struct Init_ {}; +struct NoInit_ {}; + +static const constexpr Init_ Init {}; +static const constexpr NoInit_ NoInit {}; + +} // {Globals} + +template +static inline Func ptr_as_func(void* func) noexcept { return Support::ptr_cast_impl(func); } + +template +static inline void* func_as_ptr(Func func) noexcept { return Support::ptr_cast_impl(func); } + +//! \} + +//! \addtogroup asmjit_error_handling +//! \{ + +//! AsmJit error type (uint32_t). +typedef uint32_t Error; + +//! AsmJit error codes. +enum ErrorCode : uint32_t { + // @EnumValuesBegin{"enum": "ErrorCode"}@ + + //! No error (success). + kErrorOk = 0, + + //! Out of memory. + kErrorOutOfMemory, + + //! Invalid argument. + kErrorInvalidArgument, + + //! Invalid state. + //! + //! If this error is returned it means that either you are doing something wrong or AsmJit caught itself by + //! doing something wrong. This error should never be ignored. + kErrorInvalidState, + + //! Invalid or incompatible architecture. + kErrorInvalidArch, + + //! The object is not initialized. + kErrorNotInitialized, + //! The object is already initialized. + kErrorAlreadyInitialized, + + //! Built-in feature was disabled at compile time and it's not available. + kErrorFeatureNotEnabled, + + //! Too many handles (Windows) or file descriptors (Unix/Posix). + kErrorTooManyHandles, + //! Code generated is larger than allowed. + kErrorTooLarge, + + //! No code generated. + //! + //! Returned by runtime if the \ref CodeHolder contains no code. + kErrorNoCodeGenerated, + + //! Invalid directive. + kErrorInvalidDirective, + //! Attempt to use uninitialized label. + kErrorInvalidLabel, + //! Label index overflow - a single \ref BaseAssembler instance can hold almost 2^32 (4 billion) labels. If + //! there is an attempt to create more labels then this error is returned. + kErrorTooManyLabels, + //! Label is already bound. + kErrorLabelAlreadyBound, + //! Label is already defined (named labels). + kErrorLabelAlreadyDefined, + //! Label name is too long. + kErrorLabelNameTooLong, + //! Label must always be local if it's anonymous (without a name). + kErrorInvalidLabelName, + //! Parent id passed to \ref CodeHolder::newNamedLabelEntry() was either invalid or parent is not supported + //! by the requested `LabelType`. + kErrorInvalidParentLabel, + + //! Invalid section. + kErrorInvalidSection, + //! Too many sections (section index overflow). + kErrorTooManySections, + //! Invalid section name (most probably too long). + kErrorInvalidSectionName, + + //! Relocation index overflow (too many relocations). + kErrorTooManyRelocations, + //! Invalid relocation entry. + kErrorInvalidRelocEntry, + //! Reloc entry contains address that is out of range (unencodable). + kErrorRelocOffsetOutOfRange, + + //! Invalid assignment to a register, function argument, or function return value. + kErrorInvalidAssignment, + //! Invalid instruction. + kErrorInvalidInstruction, + //! Invalid register type. + kErrorInvalidRegType, + //! Invalid register group. + kErrorInvalidRegGroup, + //! Invalid physical register id. + kErrorInvalidPhysId, + //! Invalid virtual register id. + kErrorInvalidVirtId, + //! Invalid element index (ARM). + kErrorInvalidElementIndex, + //! Invalid prefix combination (X86|X64). + kErrorInvalidPrefixCombination, + //! Invalid LOCK prefix (X86|X64). + kErrorInvalidLockPrefix, + //! Invalid XACQUIRE prefix (X86|X64). + kErrorInvalidXAcquirePrefix, + //! Invalid XRELEASE prefix (X86|X64). + kErrorInvalidXReleasePrefix, + //! Invalid REP prefix (X86|X64). + kErrorInvalidRepPrefix, + //! Invalid REX prefix (X86|X64). + kErrorInvalidRexPrefix, + //! Invalid {...} register (X86|X64). + kErrorInvalidExtraReg, + //! Invalid {k} use (not supported by the instruction) (X86|X64). + kErrorInvalidKMaskUse, + //! Invalid {k}{z} use (not supported by the instruction) (X86|X64). + kErrorInvalidKZeroUse, + //! Invalid broadcast - Currently only related to invalid use of AVX-512 {1tox} (X86|X64). + kErrorInvalidBroadcast, + //! Invalid 'embedded-rounding' {er} or 'suppress-all-exceptions' {sae} (AVX-512) (X86|X64). + kErrorInvalidEROrSAE, + //! Invalid address used (not encodable). + kErrorInvalidAddress, + //! Invalid index register used in memory address (not encodable). + kErrorInvalidAddressIndex, + //! Invalid address scale (not encodable). + kErrorInvalidAddressScale, + //! Invalid use of 64-bit address. + kErrorInvalidAddress64Bit, + //! Invalid use of 64-bit address that require 32-bit zero-extension (X64). + kErrorInvalidAddress64BitZeroExtension, + //! Invalid displacement (not encodable). + kErrorInvalidDisplacement, + //! Invalid segment (X86). + kErrorInvalidSegment, + + //! Invalid immediate (out of bounds on X86 and invalid pattern on ARM). + kErrorInvalidImmediate, + + //! Invalid operand size. + kErrorInvalidOperandSize, + //! Ambiguous operand size (memory has zero size while it's required to determine the operation type. + kErrorAmbiguousOperandSize, + //! Mismatching operand size (size of multiple operands doesn't match the operation size). + kErrorOperandSizeMismatch, + + //! Invalid option. + kErrorInvalidOption, + //! Option already defined. + kErrorOptionAlreadyDefined, + + //! Invalid TypeId. + kErrorInvalidTypeId, + //! Invalid use of a 8-bit GPB-HIGH register. + kErrorInvalidUseOfGpbHi, + //! Invalid use of a 64-bit GPQ register in 32-bit mode. + kErrorInvalidUseOfGpq, + //! Invalid use of an 80-bit float (\ref TypeId::kFloat80). + kErrorInvalidUseOfF80, + //! Instruction requires the use of consecutive registers, but registers in operands weren't (AVX512, ASIMD load/store, etc...). + kErrorNotConsecutiveRegs, + //! Failed to allocate consecutive registers - allocable registers either too restricted or a bug in RW info. + kErrorConsecutiveRegsAllocation, + + //! Illegal virtual register - reported by instruction validation. + kErrorIllegalVirtReg, + //! AsmJit cannot create more virtual registers. + kErrorTooManyVirtRegs, + + //! AsmJit requires a physical register, but no one is available. + kErrorNoMorePhysRegs, + //! A variable has been assigned more than once to a function argument (BaseCompiler). + kErrorOverlappedRegs, + //! Invalid register to hold stack arguments offset. + kErrorOverlappingStackRegWithRegArg, + + //! Unbound label cannot be evaluated by expression. + kErrorExpressionLabelNotBound, + //! Arithmetic overflow during expression evaluation. + kErrorExpressionOverflow, + + //! Failed to open anonymous memory handle or file descriptor. + kErrorFailedToOpenAnonymousMemory, + + // @EnumValuesEnd@ + + //! Count of AsmJit error codes. + kErrorCount +}; + +//! Debugging utilities. +namespace DebugUtils { + +//! \cond INTERNAL +//! Used to silence warnings about unused arguments or variables. +template +static inline void unused(Args&&...) noexcept {} +//! \endcond + +//! Returns the error `err` passed. +//! +//! Provided for debugging purposes. Putting a breakpoint inside `errored` can help with tracing the origin of any +//! error reported / returned by AsmJit. +static constexpr Error errored(Error err) noexcept { return err; } + +//! Returns a printable version of `asmjit::Error` code. +ASMJIT_API const char* errorAsString(Error err) noexcept; + +//! Called to output debugging message(s). +ASMJIT_API void debugOutput(const char* str) noexcept; + +//! Called on assertion failure. +//! +//! \param file Source file name where it happened. +//! \param line Line in the source file. +//! \param msg Message to display. +//! +//! If you have problems with assertion failures a breakpoint can be put at \ref assertionFailed() function +//! (asmjit/core/globals.cpp). A call stack will be available when such assertion failure is triggered. AsmJit +//! always returns errors on failures, assertions are a last resort and usually mean unrecoverable state due to out +//! of range array access or totally invalid arguments like nullptr where a valid pointer should be provided, etc... +ASMJIT_API void ASMJIT_NORETURN assertionFailed(const char* file, int line, const char* msg) noexcept; + +} // {DebugUtils} + +//! \def ASMJIT_ASSERT(...) +//! +//! AsmJit's own assert macro used in AsmJit code-base. +#if defined(ASMJIT_BUILD_DEBUG) +#define ASMJIT_ASSERT(...) \ + do { \ + if (ASMJIT_LIKELY(__VA_ARGS__)) \ + break; \ + ::asmjit::DebugUtils::assertionFailed(__FILE__, __LINE__, #__VA_ARGS__); \ + } while (0) +#else +#define ASMJIT_ASSERT(...) ((void)0) +#endif + +//! \def ASMJIT_PROPAGATE(...) +//! +//! Propagates a possible `Error` produced by `...` to the caller by returning the error immediately. Used by AsmJit +//! internally, but kept public for users that want to use the same technique to propagate errors to the caller. +#define ASMJIT_PROPAGATE(...) \ + do { \ + ::asmjit::Error _err = __VA_ARGS__; \ + if (ASMJIT_UNLIKELY(_err)) \ + return _err; \ + } while (0) + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_GLOBALS_H_INCLUDED diff --git a/src/asmjit/core/inst.cpp b/src/asmjit/core/inst.cpp new file mode 100644 index 0000000..8f29d8b --- /dev/null +++ b/src/asmjit/core/inst.cpp @@ -0,0 +1,113 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/archtraits.h" +#include "../core/inst.h" + +#if !defined(ASMJIT_NO_X86) + #include "../x86/x86instapi_p.h" +#endif + +#if !defined(ASMJIT_NO_AARCH64) + #include "../arm/a64instapi_p.h" +#endif + +ASMJIT_BEGIN_NAMESPACE + +// InstAPI - InstId <-> String +// =========================== + +#ifndef ASMJIT_NO_TEXT +Error InstAPI::instIdToString(Arch arch, InstId instId, String& output) noexcept { +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::InstInternal::instIdToString(arch, instId, output); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::InstInternal::instIdToString(arch, instId, output); +#endif + + return DebugUtils::errored(kErrorInvalidArch); +} + +InstId InstAPI::stringToInstId(Arch arch, const char* s, size_t len) noexcept { +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::InstInternal::stringToInstId(arch, s, len); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::InstInternal::stringToInstId(arch, s, len); +#endif + + return 0; +} +#endif // !ASMJIT_NO_TEXT + +// InstAPI - Validate +// ================== + +#ifndef ASMJIT_NO_VALIDATION +Error InstAPI::validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags) noexcept { +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::InstInternal::validate(arch, inst, operands, opCount, validationFlags); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::InstInternal::validate(arch, inst, operands, opCount, validationFlags); +#endif + + return DebugUtils::errored(kErrorInvalidArch); +} +#endif // !ASMJIT_NO_VALIDATION + +// InstAPI - QueryRWInfo +// ===================== + +#ifndef ASMJIT_NO_INTROSPECTION +Error InstAPI::queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept { + if (ASMJIT_UNLIKELY(opCount > Globals::kMaxOpCount)) + return DebugUtils::errored(kErrorInvalidArgument); + +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::InstInternal::queryRWInfo(arch, inst, operands, opCount, out); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::InstInternal::queryRWInfo(arch, inst, operands, opCount, out); +#endif + + return DebugUtils::errored(kErrorInvalidArch); +} +#endif // !ASMJIT_NO_INTROSPECTION + +// InstAPI - QueryFeatures +// ======================= + +#ifndef ASMJIT_NO_INTROSPECTION +Error InstAPI::queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept { +#if !defined(ASMJIT_NO_X86) + if (Environment::isFamilyX86(arch)) + return x86::InstInternal::queryFeatures(arch, inst, operands, opCount, out); +#endif + +#if !defined(ASMJIT_NO_AARCH64) + if (Environment::isFamilyAArch64(arch)) + return a64::InstInternal::queryFeatures(arch, inst, operands, opCount, out); +#endif + + return DebugUtils::errored(kErrorInvalidArch); +} +#endif // !ASMJIT_NO_INTROSPECTION + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/inst.h b/src/asmjit/core/inst.h new file mode 100644 index 0000000..6436789 --- /dev/null +++ b/src/asmjit/core/inst.h @@ -0,0 +1,772 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_INST_H_INCLUDED +#define ASMJIT_CORE_INST_H_INCLUDED + +#include "../core/cpuinfo.h" +#include "../core/operand.h" +#include "../core/string.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_instruction_db +//! \{ + +//! Describes an instruction id and modifiers used together with the id. +//! +//! Each architecture has a set of valid instructions indexed from 0. Instruction with 0 id is, however, a special +//! instruction that describes a "no instruction" or "invalid instruction". Different architectures can assign a. +//! different instruction to the same id, each architecture typicall has its own instructions indexed from 1. +//! +//! Instruction identifiers listed by architecture: +//! +//! - \ref x86::Inst (X86 and X86_64) +//! - \ref a64::Inst (AArch64) +typedef uint32_t InstId; + +//! Instruction id parts. +//! +//! A mask that specifies a bit-layout of \ref InstId. +enum class InstIdParts : uint32_t { + // Common Masks + // ------------ + + //! Real id without any modifiers (always 16 least significant bits). + kRealId = 0x0000FFFFu, + //! Instruction is abstract (or virtual, IR, etc...). + kAbstract = 0x80000000u, + + // ARM Specific + // ------------ + + //! AArch32 first data type, used by ASIMD instructions (`inst.dt.dt2`). + kA32_DT = 0x000F0000u, + //! AArch32 second data type, used by ASIMD instructions (`inst.dt.dt2`). + kA32_DT2 = 0x00F00000u, + //! AArch32/AArch64 condition code. + kARM_Cond = 0x78000000u +}; + +//! Instruction options. +//! +//! Instruction options complement instruction identifier and attributes. +enum class InstOptions : uint32_t { + //! No options. + kNone = 0, + + //! Used internally by emitters for handling errors and rare cases. + kReserved = 0x00000001u, + + //! Prevents following a jump during compilation (Compiler). + kUnfollow = 0x00000002u, + + //! Overwrite the destination operand(s) (Compiler). + //! + //! Hint that is important for register liveness analysis. It tells the compiler that the destination operand will + //! be overwritten now or by adjacent instructions. Compiler knows when a register is completely overwritten by a + //! single instruction, for example you don't have to mark "movaps" or "pxor x, x", however, if a pair of + //! instructions is used and the first of them doesn't completely overwrite the content of the destination, + //! Compiler fails to mark that register as dead. + //! + //! X86 Specific + //! ------------ + //! + //! - All instructions that always overwrite at least the size of the register the virtual-register uses, for + //! example "mov", "movq", "movaps" don't need the overwrite option to be used - conversion, shuffle, and + //! other miscellaneous instructions included. + //! + //! - All instructions that clear the destination register if all operands are the same, for example "xor x, x", + //! "pcmpeqb x x", etc... + //! + //! - Consecutive instructions that partially overwrite the variable until there is no old content require + //! `BaseCompiler::overwrite()` to be used. Some examples (not always the best use cases thought): + //! + //! - `movlps xmm0, ?` followed by `movhps xmm0, ?` and vice versa + //! - `movlpd xmm0, ?` followed by `movhpd xmm0, ?` and vice versa + //! - `mov al, ?` followed by `and ax, 0xFF` + //! - `mov al, ?` followed by `mov ah, al` + //! - `pinsrq xmm0, ?, 0` followed by `pinsrq xmm0, ?, 1` + //! + //! - If the allocated virtual register is used temporarily for scalar operations. For example if you allocate a + //! full vector like `x86::Compiler::newXmm()` and then use that vector for scalar operations you should use + //! `overwrite()` directive: + //! + //! - `sqrtss x, y` - only LO element of `x` is changed, if you don't + //! use HI elements, use `compiler.overwrite().sqrtss(x, y)`. + kOverwrite = 0x00000004u, + + //! Emit short-form of the instruction. + kShortForm = 0x00000010u, + //! Emit long-form of the instruction. + kLongForm = 0x00000020u, + + //! Conditional jump is likely to be taken. + kTaken = 0x00000040u, + //! Conditional jump is unlikely to be taken. + kNotTaken = 0x00000080u, + + // X86 & X64 Options + // ----------------- + + //! Use ModMR instead of ModRM if applicable. + kX86_ModMR = 0x00000100u, + //! Use ModRM instead of ModMR if applicable. + kX86_ModRM = 0x00000200u, + //! Use 3-byte VEX prefix if possible (AVX) (must be 0x00000400). + kX86_Vex3 = 0x00000400u, + //! Use VEX prefix when both VEX|EVEX prefixes are available (HINT: AVX_VNNI). + kX86_Vex = 0x00000800u, + //! Use 4-byte EVEX prefix if possible (AVX-512) (must be 0x00001000). + kX86_Evex = 0x00001000u, + + //! LOCK prefix (lock-enabled instructions only). + kX86_Lock = 0x00002000u, + //! REP prefix (string instructions only). + kX86_Rep = 0x00004000u, + //! REPNE prefix (string instructions only). + kX86_Repne = 0x00008000u, + + //! XACQUIRE prefix (only allowed instructions). + kX86_XAcquire = 0x00010000u, + //! XRELEASE prefix (only allowed instructions). + kX86_XRelease = 0x00020000u, + + //! AVX-512: embedded-rounding {er} and implicit {sae}. + kX86_ER = 0x00040000u, + //! AVX-512: suppress-all-exceptions {sae}. + kX86_SAE = 0x00080000u, + //! AVX-512: round-to-nearest (even) {rn-sae} (bits 00). + kX86_RN_SAE = 0x00000000u, + //! AVX-512: round-down (toward -inf) {rd-sae} (bits 01). + kX86_RD_SAE = 0x00200000u, + //! AVX-512: round-up (toward +inf) {ru-sae} (bits 10). + kX86_RU_SAE = 0x00400000u, + //! AVX-512: round-toward-zero (truncate) {rz-sae} (bits 11). + kX86_RZ_SAE = 0x00600000u, + //! AVX-512: Use zeroing {k}{z} instead of merging {k}. + kX86_ZMask = 0x00800000u, + + //! AVX-512: Mask to get embedded rounding bits (2 bits). + kX86_ERMask = kX86_RZ_SAE, + //! AVX-512: Mask of all possible AVX-512 options except EVEX prefix flag. + kX86_AVX512Mask = 0x00FC0000u, + + //! Force REX.B and/or VEX.B field (X64 only). + kX86_OpCodeB = 0x01000000u, + //! Force REX.X and/or VEX.X field (X64 only). + kX86_OpCodeX = 0x02000000u, + //! Force REX.R and/or VEX.R field (X64 only). + kX86_OpCodeR = 0x04000000u, + //! Force REX.W and/or VEX.W field (X64 only). + kX86_OpCodeW = 0x08000000u, + //! Force REX prefix (X64 only). + kX86_Rex = 0x40000000u, + //! Invalid REX prefix (set by X86 or when AH|BH|CH|DH regs are used on X64). + kX86_InvalidRex = 0x80000000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(InstOptions) + +//! Instruction control flow. +enum class InstControlFlow : uint32_t { + //! Regular instruction. + kRegular = 0u, + //! Unconditional jump. + kJump = 1u, + //! Conditional jump (branch). + kBranch = 2u, + //! Function call. + kCall = 3u, + //! Function return. + kReturn = 4u, + + //! Maximum value of `InstType`. + kMaxValue = kReturn +}; + +//! Hint that is used when both input operands to the instruction are the same. +//! +//! Provides hints to the instrution RW query regarding special cases in which two or more operands are the same +//! registers. This is required by instructions such as XOR, AND, OR, SUB, etc... These hints will influence the +//! RW operations query. +enum class InstSameRegHint : uint8_t { + //! No special handling. + kNone = 0, + //! Operands become read-only, the operation doesn't change the content - `X & X` and similar. + kRO = 1, + //! Operands become write-only, the content of the input(s) don't matter - `X ^ X`, `X - X`, and similar. + kWO = 2 +}; + +//! Instruction id, options, and extraReg in a single structure. This structure exists mainly to simplify analysis +//! and validation API that requires `BaseInst` and `Operand[]` array. +class BaseInst { +public: + //! \name Members + //! \{ + + //! Instruction id with modifiers. + InstId _id; + //! Instruction options. + InstOptions _options; + //! Extra register used by the instruction (either REP register or AVX-512 selector). + RegOnly _extraReg; + + enum Id : uint32_t { + //! Invalid or uninitialized instruction id. + kIdNone = 0x00000000u, + //! Abstract instruction (BaseBuilder and BaseCompiler). + kIdAbstract = 0x80000000u + }; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a new BaseInst instance with `id` and `options` set. + //! + //! Default values of `id` and `options` are zero, which means 'none' instruction. Such instruction is guaranteed + //! to never exist for any architecture supported by AsmJit. + inline explicit BaseInst(InstId instId = 0, InstOptions options = InstOptions::kNone) noexcept + : _id(instId), + _options(options), + _extraReg() {} + + inline BaseInst(InstId instId, InstOptions options, const RegOnly& extraReg) noexcept + : _id(instId), + _options(options), + _extraReg(extraReg) {} + + inline BaseInst(InstId instId, InstOptions options, const BaseReg& extraReg) noexcept + : _id(instId), + _options(options), + _extraReg { extraReg.signature(), extraReg.id() } {} + + //! \} + + //! \name Instruction id and modifiers + //! \{ + + //! Returns the instruction id with modifiers. + inline InstId id() const noexcept { return _id; } + //! Sets the instruction id and modiiers from `id`. + inline void setId(InstId id) noexcept { _id = id; } + //! Resets the instruction id and modifiers to zero, see \ref kIdNone. + inline void resetId() noexcept { _id = 0; } + + //! Returns a real instruction id that doesn't contain any modifiers. + inline InstId realId() const noexcept { return _id & uint32_t(InstIdParts::kRealId); } + + template + inline uint32_t getInstIdPart() const noexcept { + return (uint32_t(_id) & uint32_t(kPart)) >> Support::ConstCTZ::value; + } + + template + inline void setInstIdPart(uint32_t value) noexcept { + _id = (_id & ~uint32_t(kPart)) | (value << Support::ConstCTZ::value); + } + + //! \} + + //! \name Instruction Options + //! \{ + + inline InstOptions options() const noexcept { return _options; } + inline bool hasOption(InstOptions option) const noexcept { return Support::test(_options, option); } + inline void setOptions(InstOptions options) noexcept { _options = options; } + inline void addOptions(InstOptions options) noexcept { _options |= options; } + inline void clearOptions(InstOptions options) noexcept { _options &= ~options; } + inline void resetOptions() noexcept { _options = InstOptions::kNone; } + + //! \} + + //! \name Extra Register + //! \{ + + inline bool hasExtraReg() const noexcept { return _extraReg.isReg(); } + inline RegOnly& extraReg() noexcept { return _extraReg; } + inline const RegOnly& extraReg() const noexcept { return _extraReg; } + inline void setExtraReg(const BaseReg& reg) noexcept { _extraReg.init(reg); } + inline void setExtraReg(const RegOnly& reg) noexcept { _extraReg.init(reg); } + inline void resetExtraReg() noexcept { _extraReg.reset(); } + + //! \} + + //! \name ARM Specific + //! \{ + + inline arm::CondCode armCondCode() const noexcept { return (arm::CondCode)getInstIdPart(); } + inline void setArmCondCode(arm::CondCode cc) noexcept { setInstIdPart(uint32_t(cc)); } + + //! \} + + //! \name Statics + //! \{ + + static inline constexpr InstId composeARMInstId(uint32_t id, arm::CondCode cc) noexcept { + return id | (uint32_t(cc) << Support::ConstCTZ::value); + } + + static inline constexpr InstId extractRealId(uint32_t id) noexcept { + return id & uint32_t(InstIdParts::kRealId); + } + + static inline constexpr arm::CondCode extractARMCondCode(uint32_t id) noexcept { + return (arm::CondCode)((uint32_t(id) & uint32_t(InstIdParts::kARM_Cond)) >> Support::ConstCTZ::value); + } + + //! \} +}; + +//! CPU read/write flags used by \ref InstRWInfo. +//! +//! These flags can be used to get a basic overview about CPU specifics flags used by instructions. +enum class CpuRWFlags : uint32_t { + //! No flags. + kNone = 0x00000000u, + + // Common RW Flags (0x000000FF) + // ---------------------------- + + //! Carry flag. + kCF = 0x00000001u, + //! Signed overflow flag. + kOF = 0x00000002u, + //! Sign flag (negative/sign, if set). + kSF = 0x00000004u, + //! Zero and/or equality flag (1 if zero/equal). + kZF = 0x00000008u, + + // X86 Specific RW Flags (0xFFFFFF00) + // ---------------------------------- + + //! Carry flag (X86, X86_64). + kX86_CF = kCF, + //! Overflow flag (X86, X86_64). + kX86_OF = kOF, + //! Sign flag (X86, X86_64). + kX86_SF = kSF, + //! Zero flag (X86, X86_64). + kX86_ZF = kZF, + + //! Adjust flag (X86, X86_64). + kX86_AF = 0x00000100u, + //! Parity flag (X86, X86_64). + kX86_PF = 0x00000200u, + //! Direction flag (X86, X86_64). + kX86_DF = 0x00000400u, + //! Interrupt enable flag (X86, X86_64). + kX86_IF = 0x00000800u, + + //! Alignment check flag (X86, X86_64). + kX86_AC = 0x00001000u, + + //! FPU C0 status flag (X86, X86_64). + kX86_C0 = 0x00010000u, + //! FPU C1 status flag (X86, X86_64). + kX86_C1 = 0x00020000u, + //! FPU C2 status flag (X86, X86_64). + kX86_C2 = 0x00040000u, + //! FPU C3 status flag (X86, X86_64). + kX86_C3 = 0x00080000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(CpuRWFlags) + +//! Operand read/write flags describe how the operand is accessed and some additional features. +enum class OpRWFlags { + //! No flags. + kNone = 0, + + //! Operand is read. + kRead = 0x00000001u, + + //! Operand is written. + kWrite = 0x00000002u, + + //! Operand is both read and written. + kRW = 0x00000003u, + + //! Register operand can be replaced by a memory operand. + kRegMem = 0x00000004u, + + //! The register must be allocated to the index of the previous register + 1. + //! + //! This flag is used by all architectures to describe instructions that use consecutive registers, where only the + //! first one is encoded in the instruction, and the others are just a sequence that starts with the first one. On + //! X86/X86_64 architecture this is used by instructions such as V4FMADDPS, V4FMADDSS, V4FNMADDPS, V4FNMADDSS, + //! VP4DPWSSD, VP4DPWSSDS, VP2INTERSECTD, and VP2INTERSECTQ. On ARM/AArch64 this is used by vector load and store + //! instructions that can load or store multiple registers at once. + kConsecutive = 0x00000008u, + + //! The `extendByteMask()` represents a zero extension. + kZExt = 0x00000010u, + + //! Register operand must use \ref OpRWInfo::physId(). + kRegPhysId = 0x00000100u, + //! Base register of a memory operand must use \ref OpRWInfo::physId(). + kMemPhysId = 0x00000200u, + + //! This memory operand is only used to encode registers and doesn't access memory. + //! + //! X86 Specific + //! ------------ + //! + //! Instructions that use such feature include BNDLDX, BNDSTX, and LEA. + kMemFake = 0x000000400u, + + //! Base register of the memory operand will be read. + kMemBaseRead = 0x00001000u, + //! Base register of the memory operand will be written. + kMemBaseWrite = 0x00002000u, + //! Base register of the memory operand will be read & written. + kMemBaseRW = 0x00003000u, + + //! Index register of the memory operand will be read. + kMemIndexRead = 0x00004000u, + //! Index register of the memory operand will be written. + kMemIndexWrite = 0x00008000u, + //! Index register of the memory operand will be read & written. + kMemIndexRW = 0x0000C000u, + + //! Base register of the memory operand will be modified before the operation. + kMemBasePreModify = 0x00010000u, + //! Base register of the memory operand will be modified after the operation. + kMemBasePostModify = 0x00020000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(OpRWFlags) + +// Don't remove these asserts. Read/Write flags are used extensively +// by Compiler and they must always be compatible with constants below. +static_assert(uint32_t(OpRWFlags::kRead) == 0x1, "OpRWFlags::kRead flag must be 0x1"); +static_assert(uint32_t(OpRWFlags::kWrite) == 0x2, "OpRWFlags::kWrite flag must be 0x2"); +static_assert(uint32_t(OpRWFlags::kRegMem) == 0x4, "OpRWFlags::kRegMem flag must be 0x4"); + +//! Read/Write information related to a single operand, used by \ref InstRWInfo. +struct OpRWInfo { + //! \name Members + //! \{ + + //! Read/Write flags. + OpRWFlags _opFlags; + //! Physical register index, if required. + uint8_t _physId; + //! Size of a possible memory operand that can replace a register operand. + uint8_t _rmSize; + //! If non-zero, then this is a consecutive lead register, and the value describes how many registers follow. + uint8_t _consecutiveLeadCount; + //! Reserved for future use. + uint8_t _reserved[1]; + //! Read bit-mask where each bit represents one byte read from Reg/Mem. + uint64_t _readByteMask; + //! Write bit-mask where each bit represents one byte written to Reg/Mem. + uint64_t _writeByteMask; + //! Zero/Sign extend bit-mask where each bit represents one byte written to Reg/Mem. + uint64_t _extendByteMask; + + //! \} + + //! \name Reset + //! \{ + + //! Resets this operand information to all zeros. + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! Resets this operand info (resets all members) and set common information + //! to the given `opFlags`, `regSize`, and possibly `physId`. + inline void reset(OpRWFlags opFlags, uint32_t regSize, uint32_t physId = BaseReg::kIdBad) noexcept { + _opFlags = opFlags; + _physId = uint8_t(physId); + _rmSize = Support::test(opFlags, OpRWFlags::kRegMem) ? uint8_t(regSize) : uint8_t(0); + _consecutiveLeadCount = 0; + _resetReserved(); + + uint64_t mask = Support::lsbMask(regSize); + _readByteMask = Support::test(opFlags, OpRWFlags::kRead) ? mask : uint64_t(0); + _writeByteMask = Support::test(opFlags, OpRWFlags::kWrite) ? mask : uint64_t(0); + _extendByteMask = 0; + } + + inline void _resetReserved() noexcept { + _reserved[0] = 0; + } + + //! \} + + //! \name Operand Flags + //! \{ + + //! Returns operand flags. + inline OpRWFlags opFlags() const noexcept { return _opFlags; } + //! Tests whether operand flags contain the given `flag`. + inline bool hasOpFlag(OpRWFlags flag) const noexcept { return Support::test(_opFlags, flag); } + + //! Adds the given `flags` to operand flags. + inline void addOpFlags(OpRWFlags flags) noexcept { _opFlags |= flags; } + //! Removes the given `flags` from operand flags. + inline void clearOpFlags(OpRWFlags flags) noexcept { _opFlags &= ~flags; } + + //! Tests whether this operand is read from. + inline bool isRead() const noexcept { return hasOpFlag(OpRWFlags::kRead); } + //! Tests whether this operand is written to. + inline bool isWrite() const noexcept { return hasOpFlag(OpRWFlags::kWrite); } + //! Tests whether this operand is both read and write. + inline bool isReadWrite() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kRW; } + //! Tests whether this operand is read only. + inline bool isReadOnly() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kRead; } + //! Tests whether this operand is write only. + inline bool isWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kRW) == OpRWFlags::kWrite; } + + //! Returns the type of a lead register, which is followed by consecutive registers. + inline uint32_t consecutiveLeadCount() const noexcept { return _consecutiveLeadCount; } + + //! Tests whether this operand is Reg/Mem + //! + //! Reg/Mem operands can use either register or memory. + inline bool isRm() const noexcept { return hasOpFlag(OpRWFlags::kRegMem); } + + //! Tests whether the operand will be zero extended. + inline bool isZExt() const noexcept { return hasOpFlag(OpRWFlags::kZExt); } + + //! \} + + //! \name Memory Flags + //! \{ + + //! Tests whether this is a fake memory operand, which is only used, because of encoding. Fake memory operands do + //! not access any memory, they are only used to encode registers. + inline bool isMemFake() const noexcept { return hasOpFlag(OpRWFlags::kMemFake); } + + //! Tests whether the instruction's memory BASE register is used. + inline bool isMemBaseUsed() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseRW); } + //! Tests whether the instruction reads from its BASE registers. + inline bool isMemBaseRead() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseRead); } + //! Tests whether the instruction writes to its BASE registers. + inline bool isMemBaseWrite() const noexcept { return hasOpFlag(OpRWFlags::kMemBaseWrite); } + //! Tests whether the instruction reads and writes from/to its BASE registers. + inline bool isMemBaseReadWrite() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseRW; } + //! Tests whether the instruction only reads from its BASE registers. + inline bool isMemBaseReadOnly() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseRead; } + //! Tests whether the instruction only writes to its BASE registers. + inline bool isMemBaseWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kMemBaseRW) == OpRWFlags::kMemBaseWrite; } + + //! Tests whether the instruction modifies the BASE register before it uses it to calculate the target address. + inline bool isMemBasePreModify() const noexcept { return hasOpFlag(OpRWFlags::kMemBasePreModify); } + //! Tests whether the instruction modifies the BASE register after it uses it to calculate the target address. + inline bool isMemBasePostModify() const noexcept { return hasOpFlag(OpRWFlags::kMemBasePostModify); } + + //! Tests whether the instruction's memory INDEX register is used. + inline bool isMemIndexUsed() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexRW); } + //! Tests whether the instruction reads the INDEX registers. + inline bool isMemIndexRead() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexRead); } + //! Tests whether the instruction writes to its INDEX registers. + inline bool isMemIndexWrite() const noexcept { return hasOpFlag(OpRWFlags::kMemIndexWrite); } + //! Tests whether the instruction reads and writes from/to its INDEX registers. + inline bool isMemIndexReadWrite() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexRW; } + //! Tests whether the instruction only reads from its INDEX registers. + inline bool isMemIndexReadOnly() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexRead; } + //! Tests whether the instruction only writes to its INDEX registers. + inline bool isMemIndexWriteOnly() const noexcept { return (_opFlags & OpRWFlags::kMemIndexRW) == OpRWFlags::kMemIndexWrite; } + + //! \} + + //! \name Physical Register ID + //! \{ + + //! Returns a physical id of the register that is fixed for this operand. + //! + //! Returns \ref BaseReg::kIdBad if any register can be used. + inline uint32_t physId() const noexcept { return _physId; } + //! Tests whether \ref physId() would return a valid physical register id. + inline bool hasPhysId() const noexcept { return _physId != BaseReg::kIdBad; } + //! Sets physical register id, which would be fixed for this operand. + inline void setPhysId(uint32_t physId) noexcept { _physId = uint8_t(physId); } + + //! \} + + //! \name Reg/Mem Information + //! \{ + + //! Returns Reg/Mem size of the operand. + inline uint32_t rmSize() const noexcept { return _rmSize; } + //! Sets Reg/Mem size of the operand. + inline void setRmSize(uint32_t rmSize) noexcept { _rmSize = uint8_t(rmSize); } + + //! \} + + //! \name Read & Write Masks + //! \{ + + //! Returns read mask. + inline uint64_t readByteMask() const noexcept { return _readByteMask; } + //! Returns write mask. + inline uint64_t writeByteMask() const noexcept { return _writeByteMask; } + //! Returns extend mask. + inline uint64_t extendByteMask() const noexcept { return _extendByteMask; } + + //! Sets read mask. + inline void setReadByteMask(uint64_t mask) noexcept { _readByteMask = mask; } + //! Sets write mask. + inline void setWriteByteMask(uint64_t mask) noexcept { _writeByteMask = mask; } + //! Sets externd mask. + inline void setExtendByteMask(uint64_t mask) noexcept { _extendByteMask = mask; } + + //! \} +}; + +//! Flags used by \ref InstRWInfo. +enum class InstRWFlags : uint32_t { + //! No flags. + kNone = 0x00000000u, + + //! Describes a move operation. + //! + //! This flag is used by RA to eliminate moves that are guaranteed to be moves only. + kMovOp = 0x00000001u +}; +ASMJIT_DEFINE_ENUM_FLAGS(InstRWFlags) + +//! Read/Write information of an instruction. +struct InstRWInfo { + //! \name Members + //! \{ + + //! Instruction flags (there are no flags at the moment, this field is reserved). + InstRWFlags _instFlags; + //! CPU flags read. + CpuRWFlags _readFlags; + //! CPU flags written. + CpuRWFlags _writeFlags; + //! Count of operands. + uint8_t _opCount; + //! CPU feature required for replacing register operand with memory operand. + uint8_t _rmFeature; + //! Reserved for future use. + uint8_t _reserved[18]; + //! Read/Write onfo of extra register (rep{} or kz{}). + OpRWInfo _extraReg; + //! Read/Write info of instruction operands. + OpRWInfo _operands[Globals::kMaxOpCount]; + + //! \} + + //! \name Commons + //! \{ + + //! Resets this RW information to all zeros. + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + + //! \} + + //! \name Instruction Flags + //! \{ + + //! Returns flags associated with the instruction, see \ref InstRWFlags. + inline InstRWFlags instFlags() const noexcept { return _instFlags; } + + //! Tests whether the instruction flags contain `flag`. + inline bool hasInstFlag(InstRWFlags flag) const noexcept { return Support::test(_instFlags, flag); } + + //! Tests whether the instruction flags contain \ref InstRWFlags::kMovOp. + inline bool isMovOp() const noexcept { return hasInstFlag(InstRWFlags::kMovOp); } + + //! \} + + //! \name CPU Flags Information + //! \{ + + //! Returns a mask of CPU flags read. + inline CpuRWFlags readFlags() const noexcept { return _readFlags; } + //! Returns a mask of CPU flags written. + inline CpuRWFlags writeFlags() const noexcept { return _writeFlags; } + + //! \} + + //! \name Reg/Mem Information + //! \{ + + //! Returns the CPU feature required to replace a register operand with memory operand. If the returned feature is + //! zero (none) then this instruction either doesn't provide memory operand combination or there is no extra CPU + //! feature required. + //! + //! X86 Specific + //! ------------ + //! + //! Some AVX+ instructions may require extra features for replacing registers with memory operands, for example + //! VPSLLDQ instruction only supports `vpslldq reg, reg, imm` combination on AVX/AVX2 capable CPUs and requires + //! AVX-512 for `vpslldq reg, mem, imm` combination. + inline uint32_t rmFeature() const noexcept { return _rmFeature; } + + //! \} + + //! \name Operand Read/Write Information + //! \{ + + //! Returns RW information of extra register operand (extraReg). + inline const OpRWInfo& extraReg() const noexcept { return _extraReg; } + + //! Returns RW information of all instruction's operands. + inline const OpRWInfo* operands() const noexcept { return _operands; } + + //! Returns RW information of the operand at the given `index`. + inline const OpRWInfo& operand(size_t index) const noexcept { + ASMJIT_ASSERT(index < Globals::kMaxOpCount); + return _operands[index]; + } + + //! Returns the number of operands this instruction has. + inline uint32_t opCount() const noexcept { return _opCount; } + + //! \} +}; + +//! Validation flags that can be used with \ref InstAPI::validate(). +enum class ValidationFlags : uint32_t { + //! No flags. + kNone = 0, + //! Allow virtual registers in the instruction. + kEnableVirtRegs = 0x01u +}; +ASMJIT_DEFINE_ENUM_FLAGS(ValidationFlags) + +//! Instruction API. +namespace InstAPI { + +#ifndef ASMJIT_NO_TEXT +//! Appends the name of the instruction specified by `instId` and `instOptions` into the `output` string. +//! +//! \note Instruction options would only affect instruction prefix & suffix, other options would be ignored. +//! If `instOptions` is zero then only raw instruction name (without any additional text) will be appended. +ASMJIT_API Error instIdToString(Arch arch, InstId instId, String& output) noexcept; + +//! Parses an instruction name in the given string `s`. Length is specified by `len` argument, which can be +//! `SIZE_MAX` if `s` is known to be null terminated. +//! +//! Returns the parsed instruction id or \ref BaseInst::kIdNone if no such instruction exists. +ASMJIT_API InstId stringToInstId(Arch arch, const char* s, size_t len) noexcept; +#endif // !ASMJIT_NO_TEXT + +#ifndef ASMJIT_NO_VALIDATION +//! Validates the given instruction considering the given `validationFlags`. +ASMJIT_API Error validate(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, ValidationFlags validationFlags = ValidationFlags::kNone) noexcept; +#endif // !ASMJIT_NO_VALIDATION + +#ifndef ASMJIT_NO_INTROSPECTION +//! Gets Read/Write information of the given instruction. +ASMJIT_API Error queryRWInfo(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, InstRWInfo* out) noexcept; + +//! Gets CPU features required by the given instruction. +ASMJIT_API Error queryFeatures(Arch arch, const BaseInst& inst, const Operand_* operands, size_t opCount, CpuFeatures* out) noexcept; +#endif // !ASMJIT_NO_INTROSPECTION + +} // {InstAPI} + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_INST_H_INCLUDED diff --git a/src/asmjit/core/jitallocator.cpp b/src/asmjit/core/jitallocator.cpp new file mode 100644 index 0000000..16117d7 --- /dev/null +++ b/src/asmjit/core/jitallocator.cpp @@ -0,0 +1,1309 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_JIT + +#include "../core/archtraits.h" +#include "../core/jitallocator.h" +#include "../core/osutils_p.h" +#include "../core/support.h" +#include "../core/virtmem.h" +#include "../core/zone.h" +#include "../core/zonelist.h" +#include "../core/zonetree.h" + +ASMJIT_BEGIN_NAMESPACE + +// JitAllocator - Constants +// ======================== + +//! Number of pools to use when `JitAllocatorOptions::kUseMultiplePools` is set. +//! +//! Each pool increases granularity twice to make memory management more +//! efficient. Ideal number of pools appears to be 3 to 4 as it distributes +//! small and large functions properly. +static constexpr uint32_t kJitAllocatorMultiPoolCount = 3; + +//! Minimum granularity (and the default granularity for pool #0). +static constexpr uint32_t kJitAllocatorBaseGranularity = 64; + +//! Maximum block size (32MB). +static constexpr uint32_t kJitAllocatorMaxBlockSize = 1024 * 1024 * 32; + +// JitAllocator - Fill Pattern +// =========================== + +static inline uint32_t JitAllocator_defaultFillPattern() noexcept { + // X86 and X86_64 - 4x 'int3' instruction. + if (ASMJIT_ARCH_X86) + return 0xCCCCCCCCu; + + // Unknown... + return 0u; +} + +// JitAllocator - BitVectorRangeIterator +// ===================================== + +template +class BitVectorRangeIterator { +public: + const T* _ptr; + size_t _idx; + size_t _end; + T _bitWord; + + enum : uint32_t { kBitWordSize = Support::bitSizeOf() }; + enum : T { kXorMask = B == 0 ? Support::allOnes() : T(0) }; + + ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords) noexcept { + init(data, numBitWords); + } + + ASMJIT_FORCE_INLINE BitVectorRangeIterator(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { + init(data, numBitWords, start, end); + } + + ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords) noexcept { + init(data, numBitWords, 0, numBitWords * kBitWordSize); + } + + ASMJIT_FORCE_INLINE void init(const T* data, size_t numBitWords, size_t start, size_t end) noexcept { + ASMJIT_ASSERT(numBitWords >= (end + kBitWordSize - 1) / kBitWordSize); + DebugUtils::unused(numBitWords); + + size_t idx = Support::alignDown(start, kBitWordSize); + const T* ptr = data + (idx / kBitWordSize); + + T bitWord = 0; + if (idx < end) + bitWord = (*ptr ^ kXorMask) & (Support::allOnes() << (start % kBitWordSize)); + + _ptr = ptr; + _idx = idx; + _end = end; + _bitWord = bitWord; + } + + ASMJIT_FORCE_INLINE bool nextRange(size_t* rangeStart, size_t* rangeEnd, size_t rangeHint = std::numeric_limits::max()) noexcept { + // Skip all empty BitWords. + while (_bitWord == 0) { + _idx += kBitWordSize; + if (_idx >= _end) + return false; + _bitWord = (*++_ptr) ^ kXorMask; + } + + size_t i = Support::ctz(_bitWord); + + *rangeStart = _idx + i; + _bitWord = ~(_bitWord ^ ~(Support::allOnes() << i)); + + if (_bitWord == 0) { + *rangeEnd = Support::min(_idx + kBitWordSize, _end); + while (*rangeEnd - *rangeStart < rangeHint) { + _idx += kBitWordSize; + if (_idx >= _end) + break; + + _bitWord = (*++_ptr) ^ kXorMask; + if (_bitWord != Support::allOnes()) { + size_t j = Support::ctz(~_bitWord); + *rangeEnd = Support::min(_idx + j, _end); + _bitWord = _bitWord ^ ~(Support::allOnes() << j); + break; + } + + *rangeEnd = Support::min(_idx + kBitWordSize, _end); + _bitWord = 0; + continue; + } + + return true; + } + else { + size_t j = Support::ctz(_bitWord); + *rangeEnd = Support::min(_idx + j, _end); + + _bitWord = ~(_bitWord ^ ~(Support::allOnes() << j)); + return true; + } + } +}; + +// JitAllocator - Pool +// =================== + +class JitAllocatorBlock; + +class JitAllocatorPool { +public: + ASMJIT_NONCOPYABLE(JitAllocatorPool) + + //! Double linked list of blocks. + ZoneList blocks; + //! Where to start looking first. + JitAllocatorBlock* cursor; + + //! Count of blocks. + uint32_t blockCount; + //! Allocation granularity. + uint16_t granularity; + //! Log2(granularity). + uint8_t granularityLog2; + //! Count of empty blocks (either 0 or 1 as we won't keep more blocks empty). + uint8_t emptyBlockCount; + + //! Number of bits reserved across all blocks. + size_t totalAreaSize; + //! Number of bits used across all blocks. + size_t totalAreaUsed; + //! Overhead of all blocks (in bytes). + size_t totalOverheadBytes; + + inline JitAllocatorPool(uint32_t granularity) noexcept + : blocks(), + cursor(nullptr), + blockCount(0), + granularity(uint16_t(granularity)), + granularityLog2(uint8_t(Support::ctz(granularity))), + emptyBlockCount(0), + totalAreaSize(0), + totalAreaUsed(0), + totalOverheadBytes(0) {} + + inline void reset() noexcept { + blocks.reset(); + cursor = nullptr; + blockCount = 0; + totalAreaSize = 0; + totalAreaUsed = 0; + totalOverheadBytes = 0; + } + + inline size_t byteSizeFromAreaSize(uint32_t areaSize) const noexcept { return size_t(areaSize) * granularity; } + inline uint32_t areaSizeFromByteSize(size_t size) const noexcept { return uint32_t((size + granularity - 1) >> granularityLog2); } + + inline size_t bitWordCountFromAreaSize(uint32_t areaSize) const noexcept { + using namespace Support; + return alignUp(areaSize, kBitWordSizeInBits) / kBitWordSizeInBits; + } +}; + +// JitAllocator - Block +// ==================== + +class JitAllocatorBlock : public ZoneTreeNodeT, + public ZoneListNode { +public: + ASMJIT_NONCOPYABLE(JitAllocatorBlock) + + enum Flags : uint32_t { + //! Block is empty. + kFlagEmpty = 0x00000001u, + //! Block is dirty (largestUnusedArea, searchStart, searchEnd). + kFlagDirty = 0x00000002u, + //! Block is dual-mapped. + kFlagDualMapped = 0x00000004u + }; + + //! Link to the pool that owns this block. + JitAllocatorPool* _pool; + //! Virtual memory mapping - either single mapping (both pointers equal) or + //! dual mapping, where one pointer is Read+Execute and the second Read+Write. + VirtMem::DualMapping _mapping; + //! Virtual memory size (block size) [bytes]. + size_t _blockSize; + + //! Block flags. + uint32_t _flags; + //! Size of the whole block area (bit-vector size). + uint32_t _areaSize; + //! Used area (number of bits in bit-vector used). + uint32_t _areaUsed; + //! The largest unused continuous area in the bit-vector (or `areaSize` to initiate rescan). + uint32_t _largestUnusedArea; + //! Start of a search range (for unused bits). + uint32_t _searchStart; + //! End of a search range (for unused bits). + uint32_t _searchEnd; + + //! Used bit-vector (0 = unused, 1 = used). + Support::BitWord* _usedBitVector; + //! Stop bit-vector (0 = don't care, 1 = stop). + Support::BitWord* _stopBitVector; + + inline JitAllocatorBlock( + JitAllocatorPool* pool, + VirtMem::DualMapping mapping, + size_t blockSize, + uint32_t blockFlags, + Support::BitWord* usedBitVector, + Support::BitWord* stopBitVector, + uint32_t areaSize) noexcept + : ZoneTreeNodeT(), + _pool(pool), + _mapping(mapping), + _blockSize(blockSize), + _flags(blockFlags), + _areaSize(areaSize), + _areaUsed(0), + _largestUnusedArea(areaSize), + _searchStart(0), + _searchEnd(areaSize), + _usedBitVector(usedBitVector), + _stopBitVector(stopBitVector) {} + + inline JitAllocatorPool* pool() const noexcept { return _pool; } + + inline uint8_t* rxPtr() const noexcept { return static_cast(_mapping.rx); } + inline uint8_t* rwPtr() const noexcept { return static_cast(_mapping.rw); } + + inline bool hasFlag(uint32_t f) const noexcept { return (_flags & f) != 0; } + inline void addFlags(uint32_t f) noexcept { _flags |= f; } + inline void clearFlags(uint32_t f) noexcept { _flags &= ~f; } + + inline bool isDirty() const noexcept { return hasFlag(kFlagDirty); } + inline void makeDirty() noexcept { addFlags(kFlagDirty); } + + inline size_t blockSize() const noexcept { return _blockSize; } + + inline uint32_t areaSize() const noexcept { return _areaSize; } + inline uint32_t areaUsed() const noexcept { return _areaUsed; } + inline uint32_t areaAvailable() const noexcept { return _areaSize - _areaUsed; } + inline uint32_t largestUnusedArea() const noexcept { return _largestUnusedArea; } + + inline void decreaseUsedArea(uint32_t value) noexcept { + _areaUsed -= value; + _pool->totalAreaUsed -= value; + } + + inline void markAllocatedArea(uint32_t allocatedAreaStart, uint32_t allocatedAreaEnd) noexcept { + uint32_t allocatedAreaSize = allocatedAreaEnd - allocatedAreaStart; + + // Mark the newly allocated space as occupied and also the sentinel. + Support::bitVectorFill(_usedBitVector, allocatedAreaStart, allocatedAreaSize); + Support::bitVectorSetBit(_stopBitVector, allocatedAreaEnd - 1, true); + + // Update search region and statistics. + _pool->totalAreaUsed += allocatedAreaSize; + _areaUsed += allocatedAreaSize; + + if (areaAvailable() == 0) { + _searchStart = _areaSize; + _searchEnd = 0; + _largestUnusedArea = 0; + clearFlags(kFlagDirty); + } + else { + if (_searchStart == allocatedAreaStart) + _searchStart = allocatedAreaEnd; + if (_searchEnd == allocatedAreaEnd) + _searchEnd = allocatedAreaStart; + addFlags(kFlagDirty); + } + } + + inline void markReleasedArea(uint32_t releasedAreaStart, uint32_t releasedAreaEnd) noexcept { + uint32_t releasedAreaSize = releasedAreaEnd - releasedAreaStart; + + // Update the search region and statistics. + _pool->totalAreaUsed -= releasedAreaSize; + _areaUsed -= releasedAreaSize; + _searchStart = Support::min(_searchStart, releasedAreaStart); + _searchEnd = Support::max(_searchEnd, releasedAreaEnd); + + // Unmark occupied bits and also the sentinel. + Support::bitVectorClear(_usedBitVector, releasedAreaStart, releasedAreaSize); + Support::bitVectorSetBit(_stopBitVector, releasedAreaEnd - 1, false); + + if (areaUsed() == 0) { + _searchStart = 0; + _searchEnd = _areaSize; + _largestUnusedArea = _areaSize; + addFlags(kFlagEmpty); + clearFlags(kFlagDirty); + } + else { + addFlags(kFlagDirty); + } + } + + inline void markShrunkArea(uint32_t shrunkAreaStart, uint32_t shrunkAreaEnd) noexcept { + uint32_t shrunkAreaSize = shrunkAreaEnd - shrunkAreaStart; + + // Shrunk area cannot start at zero as it would mean that we have shrunk the first + // block to zero bytes, which is not allowed as such block must be released instead. + ASMJIT_ASSERT(shrunkAreaStart != 0); + ASMJIT_ASSERT(shrunkAreaSize != 0); + + // Update the search region and statistics. + _pool->totalAreaUsed -= shrunkAreaSize; + _areaUsed -= shrunkAreaSize; + _searchStart = Support::min(_searchStart, shrunkAreaStart); + _searchEnd = Support::max(_searchEnd, shrunkAreaEnd); + + // Unmark the released space and move the sentinel. + Support::bitVectorClear(_usedBitVector, shrunkAreaStart, shrunkAreaSize); + Support::bitVectorSetBit(_stopBitVector, shrunkAreaEnd - 1, false); + Support::bitVectorSetBit(_stopBitVector, shrunkAreaStart - 1, true); + + addFlags(kFlagDirty); + } + + // RBTree default CMP uses '<' and '>' operators. + inline bool operator<(const JitAllocatorBlock& other) const noexcept { return rxPtr() < other.rxPtr(); } + inline bool operator>(const JitAllocatorBlock& other) const noexcept { return rxPtr() > other.rxPtr(); } + + // Special implementation for querying blocks by `key`, which must be in `[BlockPtr, BlockPtr + BlockSize)` range. + inline bool operator<(const uint8_t* key) const noexcept { return rxPtr() + _blockSize <= key; } + inline bool operator>(const uint8_t* key) const noexcept { return rxPtr() > key; } +}; + +// JitAllocator - PrivateImpl +// ========================== + +class JitAllocatorPrivateImpl : public JitAllocator::Impl { +public: + //! Lock for thread safety. + mutable Lock lock; + //! System page size (also a minimum block size). + uint32_t pageSize; + //! Number of active allocations. + size_t allocationCount; + + //! Blocks from all pools in RBTree. + ZoneTree tree; + //! Allocator pools. + JitAllocatorPool* pools; + //! Number of allocator pools. + size_t poolCount; + + inline JitAllocatorPrivateImpl(JitAllocatorPool* pools, size_t poolCount) noexcept + : JitAllocator::Impl {}, + pageSize(0), + allocationCount(0), + pools(pools), + poolCount(poolCount) {} + inline ~JitAllocatorPrivateImpl() noexcept {} +}; + +static const JitAllocator::Impl JitAllocatorImpl_none {}; +static const JitAllocator::CreateParams JitAllocatorParams_none {}; + +// JitAllocator - Utilities +// ======================== + +static inline JitAllocatorPrivateImpl* JitAllocatorImpl_new(const JitAllocator::CreateParams* params) noexcept { + VirtMem::Info vmInfo = VirtMem::info(); + + if (!params) + params = &JitAllocatorParams_none; + + JitAllocatorOptions options = params->options; + uint32_t blockSize = params->blockSize; + uint32_t granularity = params->granularity; + uint32_t fillPattern = params->fillPattern; + + // Setup pool count to [1..3]. + size_t poolCount = 1; + if (Support::test(options, JitAllocatorOptions::kUseMultiplePools)) + poolCount = kJitAllocatorMultiPoolCount;; + + // Setup block size [64kB..256MB]. + if (blockSize < 64 * 1024 || blockSize > 256 * 1024 * 1024 || !Support::isPowerOf2(blockSize)) + blockSize = vmInfo.pageGranularity; + + // Setup granularity [64..256]. + if (granularity < 64 || granularity > 256 || !Support::isPowerOf2(granularity)) + granularity = kJitAllocatorBaseGranularity; + + // Setup fill-pattern. + if (uint32_t(options & JitAllocatorOptions::kCustomFillPattern) == 0) + fillPattern = JitAllocator_defaultFillPattern(); + + size_t size = sizeof(JitAllocatorPrivateImpl) + sizeof(JitAllocatorPool) * poolCount; + void* p = ::malloc(size); + if (ASMJIT_UNLIKELY(!p)) + return nullptr; + + JitAllocatorPool* pools = reinterpret_cast((uint8_t*)p + sizeof(JitAllocatorPrivateImpl)); + JitAllocatorPrivateImpl* impl = new(p) JitAllocatorPrivateImpl(pools, poolCount); + + impl->options = options; + impl->blockSize = blockSize; + impl->granularity = granularity; + impl->fillPattern = fillPattern; + impl->pageSize = vmInfo.pageSize; + + for (size_t poolId = 0; poolId < poolCount; poolId++) + new(&pools[poolId]) JitAllocatorPool(granularity << poolId); + + return impl; +} + +static inline void JitAllocatorImpl_destroy(JitAllocatorPrivateImpl* impl) noexcept { + impl->~JitAllocatorPrivateImpl(); + ::free(impl); +} + +static inline size_t JitAllocatorImpl_sizeToPoolId(const JitAllocatorPrivateImpl* impl, size_t size) noexcept { + size_t poolId = impl->poolCount - 1; + size_t granularity = size_t(impl->granularity) << poolId; + + while (poolId) { + if (Support::alignUp(size, granularity) == size) + break; + poolId--; + granularity >>= 1; + } + + return poolId; +} + +static inline size_t JitAllocatorImpl_bitVectorSizeToByteSize(uint32_t areaSize) noexcept { + using Support::kBitWordSizeInBits; + return ((areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits) * sizeof(Support::BitWord); +} + +static inline size_t JitAllocatorImpl_calculateIdealBlockSize(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t allocationSize) noexcept { + JitAllocatorBlock* last = pool->blocks.last(); + size_t blockSize = last ? last->blockSize() : size_t(impl->blockSize); + + if (blockSize < kJitAllocatorMaxBlockSize) + blockSize *= 2u; + + if (allocationSize > blockSize) { + blockSize = Support::alignUp(allocationSize, impl->blockSize); + if (ASMJIT_UNLIKELY(blockSize < allocationSize)) + return 0; // Overflown. + } + + return blockSize; +} + +ASMJIT_FAVOR_SPEED static void JitAllocatorImpl_fillPattern(void* mem, uint32_t pattern, size_t sizeInBytes) noexcept { + size_t n = sizeInBytes / 4u; + uint32_t* p = static_cast(mem); + + for (size_t i = 0; i < n; i++) + p[i] = pattern; +} + +// Allocate a new `JitAllocatorBlock` for the given `blockSize`. +// +// NOTE: The block doesn't have `kFlagEmpty` flag set, because the new block +// is only allocated when it's actually needed, so it would be cleared anyway. +static JitAllocatorBlock* JitAllocatorImpl_newBlock(JitAllocatorPrivateImpl* impl, JitAllocatorPool* pool, size_t blockSize) noexcept { + using Support::BitWord; + using Support::kBitWordSizeInBits; + + uint32_t areaSize = uint32_t((blockSize + pool->granularity - 1) >> pool->granularityLog2); + uint32_t numBitWords = (areaSize + kBitWordSizeInBits - 1u) / kBitWordSizeInBits; + + JitAllocatorBlock* block = static_cast(::malloc(sizeof(JitAllocatorBlock))); + BitWord* bitWords = nullptr; + VirtMem::DualMapping virtMem {}; + Error err = kErrorOutOfMemory; + + if (block != nullptr) + bitWords = static_cast(::malloc(size_t(numBitWords) * 2 * sizeof(BitWord))); + + uint32_t blockFlags = 0; + if (bitWords != nullptr) { + if (Support::test(impl->options, JitAllocatorOptions::kUseDualMapping)) { + err = VirtMem::allocDualMapping(&virtMem, blockSize, VirtMem::MemoryFlags::kAccessRWX); + blockFlags |= JitAllocatorBlock::kFlagDualMapped; + } + else { + err = VirtMem::alloc(&virtMem.rx, blockSize, VirtMem::MemoryFlags::kAccessRWX); + virtMem.rw = virtMem.rx; + } + } + + // Out of memory. + if (ASMJIT_UNLIKELY(!block || !bitWords || err != kErrorOk)) { + if (bitWords) + ::free(bitWords); + + if (block) + ::free(block); + + return nullptr; + } + + // Fill the memory if the secure mode is enabled. + if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) { + VirtMem::ProtectJitReadWriteScope scope(virtMem.rw, blockSize); + JitAllocatorImpl_fillPattern(virtMem.rw, impl->fillPattern, blockSize); + } + + memset(bitWords, 0, size_t(numBitWords) * 2 * sizeof(BitWord)); + return new(block) JitAllocatorBlock(pool, virtMem, blockSize, blockFlags, bitWords, bitWords + numBitWords, areaSize); +} + +static void JitAllocatorImpl_deleteBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { + DebugUtils::unused(impl); + + if (block->hasFlag(JitAllocatorBlock::kFlagDualMapped)) + VirtMem::releaseDualMapping(&block->_mapping, block->blockSize()); + else + VirtMem::release(block->rxPtr(), block->blockSize()); + + ::free(block->_usedBitVector); + ::free(block); +} + +static void JitAllocatorImpl_insertBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { + JitAllocatorPool* pool = block->pool(); + + if (!pool->cursor) + pool->cursor = block; + + // Add to RBTree and List. + impl->tree.insert(block); + pool->blocks.append(block); + + // Update statistics. + pool->blockCount++; + pool->totalAreaSize += block->areaSize(); + pool->totalOverheadBytes += sizeof(JitAllocatorBlock) + JitAllocatorImpl_bitVectorSizeToByteSize(block->areaSize()) * 2u; +} + +static void JitAllocatorImpl_removeBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { + JitAllocatorPool* pool = block->pool(); + + // Remove from RBTree and List. + if (pool->cursor == block) + pool->cursor = block->hasPrev() ? block->prev() : block->next(); + + impl->tree.remove(block); + pool->blocks.unlink(block); + + // Update statistics. + pool->blockCount--; + pool->totalAreaSize -= block->areaSize(); + pool->totalOverheadBytes -= sizeof(JitAllocatorBlock) + JitAllocatorImpl_bitVectorSizeToByteSize(block->areaSize()) * 2u; +} + +static void JitAllocatorImpl_wipeOutBlock(JitAllocatorPrivateImpl* impl, JitAllocatorBlock* block) noexcept { + if (block->hasFlag(JitAllocatorBlock::kFlagEmpty)) + return; + + JitAllocatorPool* pool = block->pool(); + uint32_t areaSize = block->areaSize(); + uint32_t granularity = pool->granularity; + size_t numBitWords = pool->bitWordCountFromAreaSize(areaSize); + + VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadWrite); + if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) { + uint8_t* rwPtr = block->rwPtr(); + BitVectorRangeIterator it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize())); + + size_t rangeStart; + size_t rangeEnd; + + while (it.nextRange(&rangeStart, &rangeEnd)) { + uint8_t* spanPtr = rwPtr + rangeStart * granularity; + size_t spanSize = (rangeEnd - rangeStart) * granularity; + + JitAllocatorImpl_fillPattern(spanPtr, impl->fillPattern, spanSize); + VirtMem::flushInstructionCache(spanPtr, spanSize); + } + } + VirtMem::protectJitMemory(VirtMem::ProtectJitAccess::kReadExecute); + + memset(block->_usedBitVector, 0, size_t(numBitWords) * sizeof(Support::BitWord)); + memset(block->_stopBitVector, 0, size_t(numBitWords) * sizeof(Support::BitWord)); + + block->_areaUsed = 0; + block->_largestUnusedArea = areaSize; + block->_searchStart = 0; + block->_searchEnd = areaSize; + block->addFlags(JitAllocatorBlock::kFlagEmpty); + block->clearFlags(JitAllocatorBlock::kFlagDirty); +} + +// JitAllocator - Construction & Destruction +// ========================================= + +JitAllocator::JitAllocator(const CreateParams* params) noexcept { + _impl = JitAllocatorImpl_new(params); + if (ASMJIT_UNLIKELY(!_impl)) + _impl = const_cast(&JitAllocatorImpl_none); +} + +JitAllocator::~JitAllocator() noexcept { + if (_impl == &JitAllocatorImpl_none) + return; + + reset(ResetPolicy::kHard); + JitAllocatorImpl_destroy(static_cast(_impl)); +} + +// JitAllocator - Reset +// ==================== + +void JitAllocator::reset(ResetPolicy resetPolicy) noexcept { + if (_impl == &JitAllocatorImpl_none) + return; + + JitAllocatorPrivateImpl* impl = static_cast(_impl); + impl->tree.reset(); + size_t poolCount = impl->poolCount; + + for (size_t poolId = 0; poolId < poolCount; poolId++) { + JitAllocatorPool& pool = impl->pools[poolId]; + JitAllocatorBlock* block = pool.blocks.first(); + + JitAllocatorBlock* blockToKeep = nullptr; + if (resetPolicy != ResetPolicy::kHard && uint32_t(impl->options & JitAllocatorOptions::kImmediateRelease) == 0) { + blockToKeep = block; + block = block->next(); + } + + while (block) { + JitAllocatorBlock* next = block->next(); + JitAllocatorImpl_deleteBlock(impl, block); + block = next; + } + + pool.reset(); + + if (blockToKeep) { + blockToKeep->_listNodes[0] = nullptr; + blockToKeep->_listNodes[1] = nullptr; + JitAllocatorImpl_wipeOutBlock(impl, blockToKeep); + JitAllocatorImpl_insertBlock(impl, blockToKeep); + pool.emptyBlockCount = 1; + } + } +} + +// JitAllocator - Statistics +// ========================= + +JitAllocator::Statistics JitAllocator::statistics() const noexcept { + Statistics statistics; + statistics.reset(); + + if (ASMJIT_LIKELY(_impl != &JitAllocatorImpl_none)) { + JitAllocatorPrivateImpl* impl = static_cast(_impl); + LockGuard guard(impl->lock); + + size_t poolCount = impl->poolCount; + for (size_t poolId = 0; poolId < poolCount; poolId++) { + const JitAllocatorPool& pool = impl->pools[poolId]; + statistics._blockCount += size_t(pool.blockCount); + statistics._reservedSize += size_t(pool.totalAreaSize) * pool.granularity; + statistics._usedSize += size_t(pool.totalAreaUsed) * pool.granularity; + statistics._overheadSize += size_t(pool.totalOverheadBytes); + } + + statistics._allocationCount = impl->allocationCount; + } + + return statistics; +} + +// JitAllocator - Alloc & Release +// ============================== + +Error JitAllocator::alloc(void** rxPtrOut, void** rwPtrOut, size_t size) noexcept { + if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) + return DebugUtils::errored(kErrorNotInitialized); + + JitAllocatorPrivateImpl* impl = static_cast(_impl); + constexpr uint32_t kNoIndex = std::numeric_limits::max(); + + *rxPtrOut = nullptr; + *rwPtrOut = nullptr; + + // Align to the minimum granularity by default. + size = Support::alignUp(size, impl->granularity); + if (ASMJIT_UNLIKELY(size == 0)) + return DebugUtils::errored(kErrorInvalidArgument); + + if (ASMJIT_UNLIKELY(size > std::numeric_limits::max() / 2)) + return DebugUtils::errored(kErrorTooLarge); + + LockGuard guard(impl->lock); + JitAllocatorPool* pool = &impl->pools[JitAllocatorImpl_sizeToPoolId(impl, size)]; + + uint32_t areaIndex = kNoIndex; + uint32_t areaSize = uint32_t(pool->areaSizeFromByteSize(size)); + + // Try to find the requested memory area in existing blocks. + JitAllocatorBlock* block = pool->blocks.first(); + if (block) { + JitAllocatorBlock* initial = block; + do { + JitAllocatorBlock* next = block->hasNext() ? block->next() : pool->blocks.first(); + if (block->areaAvailable() >= areaSize) { + if (block->isDirty() || block->largestUnusedArea() >= areaSize) { + BitVectorRangeIterator it(block->_usedBitVector, pool->bitWordCountFromAreaSize(block->areaSize()), block->_searchStart, block->_searchEnd); + + size_t rangeStart = 0; + size_t rangeEnd = block->areaSize(); + + size_t searchStart = SIZE_MAX; + size_t largestArea = 0; + + while (it.nextRange(&rangeStart, &rangeEnd, areaSize)) { + size_t rangeSize = rangeEnd - rangeStart; + if (rangeSize >= areaSize) { + areaIndex = uint32_t(rangeStart); + break; + } + + searchStart = Support::min(searchStart, rangeStart); + largestArea = Support::max(largestArea, rangeSize); + } + + if (areaIndex != kNoIndex) + break; + + if (searchStart != SIZE_MAX) { + // Because we have iterated over the entire block, we can now mark the + // largest unused area that can be used to cache the next traversal. + size_t searchEnd = rangeEnd; + + block->_searchStart = uint32_t(searchStart); + block->_searchEnd = uint32_t(searchEnd); + block->_largestUnusedArea = uint32_t(largestArea); + block->clearFlags(JitAllocatorBlock::kFlagDirty); + } + } + } + + block = next; + } while (block != initial); + } + + // Allocate a new block if there is no region of a required width. + if (areaIndex == kNoIndex) { + size_t blockSize = JitAllocatorImpl_calculateIdealBlockSize(impl, pool, size); + if (ASMJIT_UNLIKELY(!blockSize)) + return DebugUtils::errored(kErrorOutOfMemory); + + block = JitAllocatorImpl_newBlock(impl, pool, blockSize); + areaIndex = 0; + + if (ASMJIT_UNLIKELY(!block)) + return DebugUtils::errored(kErrorOutOfMemory); + + JitAllocatorImpl_insertBlock(impl, block); + block->_searchStart = areaSize; + block->_largestUnusedArea = block->areaSize() - areaSize; + } + else if (block->hasFlag(JitAllocatorBlock::kFlagEmpty)) { + pool->emptyBlockCount--; + block->clearFlags(JitAllocatorBlock::kFlagEmpty); + } + + // Update statistics. + impl->allocationCount++; + block->markAllocatedArea(areaIndex, areaIndex + areaSize); + + // Return a pointer to the allocated memory. + size_t offset = pool->byteSizeFromAreaSize(areaIndex); + ASMJIT_ASSERT(offset <= block->blockSize() - size); + + *rxPtrOut = block->rxPtr() + offset; + *rwPtrOut = block->rwPtr() + offset; + return kErrorOk; +} + +Error JitAllocator::release(void* rxPtr) noexcept { + if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) + return DebugUtils::errored(kErrorNotInitialized); + + if (ASMJIT_UNLIKELY(!rxPtr)) + return DebugUtils::errored(kErrorInvalidArgument); + + JitAllocatorPrivateImpl* impl = static_cast(_impl); + LockGuard guard(impl->lock); + + JitAllocatorBlock* block = impl->tree.get(static_cast(rxPtr)); + if (ASMJIT_UNLIKELY(!block)) + return DebugUtils::errored(kErrorInvalidState); + + // Offset relative to the start of the block. + JitAllocatorPool* pool = block->pool(); + size_t offset = (size_t)((uint8_t*)rxPtr - block->rxPtr()); + + // The first bit representing the allocated area and its size. + uint32_t areaIndex = uint32_t(offset >> pool->granularityLog2); + uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaIndex, true)) + 1; + uint32_t areaSize = areaEnd - areaIndex; + + impl->allocationCount--; + block->markReleasedArea(areaIndex, areaEnd); + + // Fill the released memory if the secure mode is enabled. + if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) { + uint8_t* spanPtr = block->rwPtr() + areaIndex * pool->granularity; + size_t spanSize = areaSize * pool->granularity; + + VirtMem::ProtectJitReadWriteScope scope(spanPtr, spanSize); + JitAllocatorImpl_fillPattern(spanPtr, impl->fillPattern, spanSize); + } + + // Release the whole block if it became empty. + if (block->areaUsed() == 0) { + if (pool->emptyBlockCount || Support::test(impl->options, JitAllocatorOptions::kImmediateRelease)) { + JitAllocatorImpl_removeBlock(impl, block); + JitAllocatorImpl_deleteBlock(impl, block); + } + else { + pool->emptyBlockCount++; + } + } + + return kErrorOk; +} + +Error JitAllocator::shrink(void* rxPtr, size_t newSize) noexcept { + if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) + return DebugUtils::errored(kErrorNotInitialized); + + if (ASMJIT_UNLIKELY(!rxPtr)) + return DebugUtils::errored(kErrorInvalidArgument); + + if (ASMJIT_UNLIKELY(newSize == 0)) + return release(rxPtr); + + JitAllocatorPrivateImpl* impl = static_cast(_impl); + LockGuard guard(impl->lock); + JitAllocatorBlock* block = impl->tree.get(static_cast(rxPtr)); + + if (ASMJIT_UNLIKELY(!block)) + return DebugUtils::errored(kErrorInvalidArgument); + + // Offset relative to the start of the block. + JitAllocatorPool* pool = block->pool(); + size_t offset = (size_t)((uint8_t*)rxPtr - block->rxPtr()); + + // The first bit representing the allocated area and its size. + uint32_t areaStart = uint32_t(offset >> pool->granularityLog2); + + bool isUsed = Support::bitVectorGetBit(block->_usedBitVector, areaStart); + if (ASMJIT_UNLIKELY(!isUsed)) + return DebugUtils::errored(kErrorInvalidArgument); + + uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaStart, true)) + 1; + uint32_t areaPrevSize = areaEnd - areaStart; + uint32_t areaShrunkSize = pool->areaSizeFromByteSize(newSize); + + if (ASMJIT_UNLIKELY(areaShrunkSize > areaPrevSize)) + return DebugUtils::errored(kErrorInvalidState); + + uint32_t areaDiff = areaPrevSize - areaShrunkSize; + if (areaDiff) { + block->markShrunkArea(areaStart + areaShrunkSize, areaEnd); + + // Fill released memory if the secure mode is enabled. + if (Support::test(impl->options, JitAllocatorOptions::kFillUnusedMemory)) + JitAllocatorImpl_fillPattern(block->rwPtr() + (areaStart + areaShrunkSize) * pool->granularity, fillPattern(), areaDiff * pool->granularity); + } + + return kErrorOk; +} + +Error JitAllocator::query(void* rxPtr, void** rxPtrOut, void** rwPtrOut, size_t* sizeOut) const noexcept { + *rxPtrOut = nullptr; + *rwPtrOut = nullptr; + *sizeOut = 0u; + + if (ASMJIT_UNLIKELY(_impl == &JitAllocatorImpl_none)) + return DebugUtils::errored(kErrorNotInitialized); + + JitAllocatorPrivateImpl* impl = static_cast(_impl); + LockGuard guard(impl->lock); + JitAllocatorBlock* block = impl->tree.get(static_cast(rxPtr)); + + if (ASMJIT_UNLIKELY(!block)) + return DebugUtils::errored(kErrorInvalidArgument); + + // Offset relative to the start of the block. + JitAllocatorPool* pool = block->pool(); + size_t offset = (size_t)((uint8_t*)rxPtr - block->rxPtr()); + + // The first bit representing the allocated area and its size. + uint32_t areaStart = uint32_t(offset >> pool->granularityLog2); + + bool isUsed = Support::bitVectorGetBit(block->_usedBitVector, areaStart); + if (ASMJIT_UNLIKELY(!isUsed)) + return DebugUtils::errored(kErrorInvalidArgument); + + uint32_t areaEnd = uint32_t(Support::bitVectorIndexOf(block->_stopBitVector, areaStart, true)) + 1; + size_t byteOffset = pool->byteSizeFromAreaSize(areaStart); + size_t byteSize = pool->byteSizeFromAreaSize(areaEnd - areaStart); + + *rxPtrOut = static_cast(block->_mapping.rx) + byteOffset; + *rwPtrOut = static_cast(block->_mapping.rw) + byteOffset; + *sizeOut = byteSize; + + return kErrorOk; +} + +// JitAllocator - Tests +// ==================== + +#if defined(ASMJIT_TEST) +// A pseudo random number generator based on a paper by Sebastiano Vigna: +// http://vigna.di.unimi.it/ftp/papers/xorshiftplus.pdf +class Random { +public: + // Constants suggested as `23/18/5`. + enum Steps : uint32_t { + kStep1_SHL = 23, + kStep2_SHR = 18, + kStep3_SHR = 5 + }; + + inline explicit Random(uint64_t seed = 0) noexcept { reset(seed); } + inline Random(const Random& other) noexcept = default; + + inline void reset(uint64_t seed = 0) noexcept { + // The number is arbitrary, it means nothing. + constexpr uint64_t kZeroSeed = 0x1F0A2BE71D163FA0u; + + // Generate the state data by using splitmix64. + for (uint32_t i = 0; i < 2; i++) { + seed += 0x9E3779B97F4A7C15u; + uint64_t x = seed; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9u; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBu; + x = (x ^ (x >> 31)); + _state[i] = x != 0 ? x : kZeroSeed; + } + } + + inline uint32_t nextUInt32() noexcept { + return uint32_t(nextUInt64() >> 32); + } + + inline uint64_t nextUInt64() noexcept { + uint64_t x = _state[0]; + uint64_t y = _state[1]; + + x ^= x << kStep1_SHL; + y ^= y >> kStep3_SHR; + x ^= x >> kStep2_SHR; + x ^= y; + + _state[0] = y; + _state[1] = x; + return x + y; + } + + uint64_t _state[2]; +}; + +// Helper class to verify that JitAllocator doesn't return addresses that overlap. +class JitAllocatorWrapper { +public: + // Address to a memory region of a given size. + class Range { + public: + inline Range(uint8_t* addr, size_t size) noexcept + : addr(addr), + size(size) {} + uint8_t* addr; + size_t size; + }; + + // Based on JitAllocator::Block, serves our purpose well... + class Record : public ZoneTreeNodeT, + public Range { + public: + inline Record(uint8_t* addr, size_t size) + : ZoneTreeNodeT(), + Range(addr, size) {} + + inline bool operator<(const Record& other) const noexcept { return addr < other.addr; } + inline bool operator>(const Record& other) const noexcept { return addr > other.addr; } + + inline bool operator<(const uint8_t* key) const noexcept { return addr + size <= key; } + inline bool operator>(const uint8_t* key) const noexcept { return addr > key; } + }; + + Zone _zone; + ZoneAllocator _heap; + ZoneTree _records; + JitAllocator _allocator; + + explicit JitAllocatorWrapper(const JitAllocator::CreateParams* params) noexcept + : _zone(1024 * 1024), + _heap(&_zone), + _allocator(params) {} + + void _insert(void* p_, size_t size) noexcept { + uint8_t* p = static_cast(p_); + uint8_t* pEnd = p + size - 1; + + Record* record; + + record = _records.get(p); + if (record) + EXPECT(record == nullptr, "Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size); + + record = _records.get(pEnd); + if (record) + EXPECT(record == nullptr, "Address [%p:%p] collides with a newly allocated [%p:%p]\n", record->addr, record->addr + record->size, p, p + size); + + record = _heap.newT(p, size); + EXPECT(record != nullptr, "Out of memory, cannot allocate 'Record'"); + + _records.insert(record); + } + + void _remove(void* p) noexcept { + Record* record = _records.get(static_cast(p)); + EXPECT(record != nullptr, "Address [%p] doesn't exist\n", p); + + _records.remove(record); + _heap.release(record, sizeof(Record)); + } + + void* alloc(size_t size) noexcept { + void* rxPtr; + void* rwPtr; + + Error err = _allocator.alloc(&rxPtr, &rwPtr, size); + EXPECT(err == kErrorOk, "JitAllocator failed to allocate %zu bytes\n", size); + + _insert(rxPtr, size); + return rxPtr; + } + + void release(void* p) noexcept { + _remove(p); + EXPECT(_allocator.release(p) == kErrorOk, "JitAllocator failed to release '%p'\n", p); + } + + void shrink(void* p, size_t newSize) noexcept { + Record* record = _records.get(static_cast(p)); + EXPECT(record != nullptr, "Address [%p] doesn't exist\n", p); + + if (!newSize) + return release(p); + + Error err = _allocator.shrink(p, newSize); + EXPECT(err == kErrorOk, "JitAllocator failed to shrink %p to %zu bytes\n", p, newSize); + + record->size = newSize; + } +}; + +static void JitAllocatorTest_shuffle(void** ptrArray, size_t count, Random& prng) noexcept { + for (size_t i = 0; i < count; ++i) + std::swap(ptrArray[i], ptrArray[size_t(prng.nextUInt32() % count)]); +} + +static void JitAllocatorTest_usage(JitAllocator& allocator) noexcept { + JitAllocator::Statistics stats = allocator.statistics(); + INFO(" Block Count : %9llu [Blocks]" , (unsigned long long)(stats.blockCount())); + INFO(" Reserved (VirtMem): %9llu [Bytes]" , (unsigned long long)(stats.reservedSize())); + INFO(" Used (VirtMem): %9llu [Bytes] (%.1f%%)", (unsigned long long)(stats.usedSize()), stats.usedSizeAsPercent()); + INFO(" Overhead (HeapMem): %9llu [Bytes] (%.1f%%)", (unsigned long long)(stats.overheadSize()), stats.overheadSizeAsPercent()); +} + +template +static void BitVectorRangeIterator_testRandom(Random& rnd, size_t count) noexcept { + for (size_t i = 0; i < count; i++) { + T in[kPatternSize]; + T out[kPatternSize]; + + for (size_t j = 0; j < kPatternSize; j++) { + in[j] = T(uint64_t(rnd.nextUInt32() & 0xFFu) * 0x0101010101010101); + out[j] = Bit == 0 ? Support::allOnes() : T(0); + } + + { + BitVectorRangeIterator it(in, kPatternSize); + size_t rangeStart, rangeEnd; + while (it.nextRange(&rangeStart, &rangeEnd)) { + if (Bit) + Support::bitVectorFill(out, rangeStart, rangeEnd - rangeStart); + else + Support::bitVectorClear(out, rangeStart, rangeEnd - rangeStart); + } + } + + for (size_t j = 0; j < kPatternSize; j++) { + EXPECT(in[j] == out[j], "Invalid pattern detected at [%zu] (%llX != %llX)", j, (unsigned long long)in[j], (unsigned long long)out[j]); + } + } +} + +static void test_jit_allocator_alloc_release() noexcept { + size_t kCount = BrokenAPI::hasArg("--quick") ? 1000 : 100000; + + struct TestParams { + const char* name; + JitAllocatorOptions options; + uint32_t blockSize; + uint32_t granularity; + }; + + static TestParams testParams[] = { + { "Default", JitAllocatorOptions::kNone, 0, 0 }, + { "16MB blocks", JitAllocatorOptions::kNone, 16 * 1024 * 1024, 0 }, + { "256B granularity", JitAllocatorOptions::kNone, 0, 256 }, + { "kUseDualMapping", JitAllocatorOptions::kUseDualMapping, 0, 0 }, + { "kUseMultiplePools", JitAllocatorOptions::kUseMultiplePools, 0, 0 }, + { "kFillUnusedMemory", JitAllocatorOptions::kFillUnusedMemory, 0, 0 }, + { "kImmediateRelease", JitAllocatorOptions::kImmediateRelease, 0, 0 }, + { "kUseDualMapping | kFillUnusedMemory", JitAllocatorOptions::kUseDualMapping | JitAllocatorOptions::kFillUnusedMemory, 0, 0 } + }; + + INFO("BitVectorRangeIterator"); + { + Random rnd; + BitVectorRangeIterator_testRandom(rnd, kCount); + } + + INFO("BitVectorRangeIterator"); + { + Random rnd; + BitVectorRangeIterator_testRandom(rnd, kCount); + } + + for (uint32_t testId = 0; testId < ASMJIT_ARRAY_SIZE(testParams); testId++) { + INFO("JitAllocator(%s)", testParams[testId].name); + + JitAllocator::CreateParams params {}; + params.options = testParams[testId].options; + params.blockSize = testParams[testId].blockSize; + params.granularity = testParams[testId].granularity; + + size_t fixedBlockSize = 256; + + JitAllocatorWrapper wrapper(¶ms); + Random prng(100); + + size_t i; + + INFO(" Memory alloc/release test - %d allocations", kCount); + + void** ptrArray = (void**)::malloc(sizeof(void*) * size_t(kCount)); + EXPECT(ptrArray != nullptr, + "Couldn't allocate '%u' bytes for pointer-array", unsigned(sizeof(void*) * size_t(kCount))); + + // Random blocks tests... + INFO(" Allocating random blocks..."); + for (i = 0; i < kCount; i++) + ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Releasing all allocated blocks from the beginning..."); + for (i = 0; i < kCount; i++) + wrapper.release(ptrArray[i]); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Allocating random blocks again...", kCount); + for (i = 0; i < kCount; i++) + ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Shuffling allocated blocks..."); + JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng); + + INFO(" Releasing 50%% of allocated blocks..."); + for (i = 0; i < kCount / 2; i++) + wrapper.release(ptrArray[i]); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Allocating 50%% more blocks again..."); + for (i = 0; i < kCount / 2; i++) + ptrArray[i] = wrapper.alloc((prng.nextUInt32() % 1024) + 8); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Releasing all allocated blocks from the end..."); + for (i = 0; i < kCount; i++) + wrapper.release(ptrArray[kCount - i - 1]); + JitAllocatorTest_usage(wrapper._allocator); + + // Fixed blocks tests... + INFO(" Allocating %zuB blocks...", fixedBlockSize); + for (i = 0; i < kCount / 2; i++) + ptrArray[i] = wrapper.alloc(fixedBlockSize); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Shrinking each %zuB block to 1 byte", fixedBlockSize); + for (i = 0; i < kCount / 2; i++) + wrapper.shrink(ptrArray[i], 1); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Allocating more 64B blocks...", 64); + for (i = kCount / 2; i < kCount; i++) + ptrArray[i] = wrapper.alloc(64); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Releasing all blocks from the beginning..."); + for (i = 0; i < kCount; i++) + wrapper.release(ptrArray[i]); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Allocating %zuB blocks...", fixedBlockSize); + for (i = 0; i < kCount; i++) + ptrArray[i] = wrapper.alloc(fixedBlockSize); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Shuffling allocated blocks..."); + JitAllocatorTest_shuffle(ptrArray, unsigned(kCount), prng); + + INFO(" Releasing 50%% of allocated blocks..."); + for (i = 0; i < kCount / 2; i++) + wrapper.release(ptrArray[i]); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Allocating 50%% more %zuB blocks again...", fixedBlockSize); + for (i = 0; i < kCount / 2; i++) + ptrArray[i] = wrapper.alloc(fixedBlockSize); + JitAllocatorTest_usage(wrapper._allocator); + + INFO(" Releasing all allocated blocks from the end..."); + for (i = 0; i < kCount; i++) + wrapper.release(ptrArray[kCount - i - 1]); + JitAllocatorTest_usage(wrapper._allocator); + + ::free(ptrArray); + } +} + +static void test_jit_allocator_query() noexcept { + JitAllocator allocator; + + void* rxPtr = nullptr; + void* rwPtr = nullptr; + size_t size = 100; + + EXPECT(allocator.alloc(&rxPtr, &rwPtr, size) == kErrorOk); + EXPECT(rxPtr != nullptr); + EXPECT(rwPtr != nullptr); + + void* rxPtrQueried = nullptr; + void* rwPtrQueried = nullptr; + size_t sizeQueried; + + EXPECT(allocator.query(rxPtr, &rxPtrQueried, &rwPtrQueried, &sizeQueried) == kErrorOk); + EXPECT(rxPtrQueried == rxPtr); + EXPECT(rwPtrQueried == rwPtr); + EXPECT(sizeQueried == Support::alignUp(size, allocator.granularity())); +} + +UNIT(jit_allocator) { + test_jit_allocator_alloc_release(); + test_jit_allocator_query(); +} +#endif + +ASMJIT_END_NAMESPACE + +#endif diff --git a/src/asmjit/core/jitallocator.h b/src/asmjit/core/jitallocator.h new file mode 100644 index 0000000..e8fe695 --- /dev/null +++ b/src/asmjit/core/jitallocator.h @@ -0,0 +1,261 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_JITALLOCATOR_H_INCLUDED +#define ASMJIT_CORE_JITALLOCATOR_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_JIT + +#include "../core/globals.h" +#include "../core/virtmem.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_virtual_memory +//! \{ + +//! Options used by \ref JitAllocator. +enum class JitAllocatorOptions : uint32_t { + //! No options. + kNone = 0, + + //! Enables the use of an anonymous memory-mapped memory that is mapped into two buffers having a different pointer. + //! The first buffer has read and execute permissions and the second buffer has read+write permissions. + //! + //! See \ref VirtMem::allocDualMapping() for more details about this feature. + kUseDualMapping = 0x00000001u, + + //! Enables the use of multiple pools with increasing granularity instead of a single pool. This flag would enable + //! 3 internal pools in total having 64, 128, and 256 bytes granularity. + //! + //! This feature is only recommended for users that generate a lot of code and would like to minimize the overhead + //! of `JitAllocator` itself by having blocks of different allocation granularities. Using this feature only for + //! few allocations won't pay off as the allocator may need to create more blocks initially before it can take the + //! advantage of variable block granularity. + kUseMultiplePools = 0x00000002u, + + //! Always fill reserved memory by a fill-pattern. + //! + //! Causes a new block to be cleared by the fill pattern and freshly released memory to be cleared before making + //! it ready for another use. + kFillUnusedMemory = 0x00000004u, + + //! When this flag is set the allocator would immediately release unused blocks during `release()` or `reset()`. + //! When this flag is not set the allocator would keep one empty block in each pool to prevent excessive virtual + //! memory allocations and deallocations in border cases, which involve constantly allocating and deallocating a + //! single block caused by repetitive calling `alloc()` and `release()` when the allocator has either no blocks + //! or have all blocks fully occupied. + kImmediateRelease = 0x00000008u, + + //! Use a custom fill pattern, must be combined with `kFlagFillUnusedMemory`. + kCustomFillPattern = 0x10000000u +}; +ASMJIT_DEFINE_ENUM_FLAGS(JitAllocatorOptions) + +//! A simple implementation of memory manager that uses `asmjit::VirtMem` +//! functions to manage virtual memory for JIT compiled code. +//! +//! Implementation notes: +//! +//! - Granularity of allocated blocks is different than granularity for a typical C malloc. In addition, the allocator +//! can use several memory pools having a different granularity to minimize the maintenance overhead. Multiple pools +//! feature requires `kFlagUseMultiplePools` flag to be set. +//! +//! - The allocator doesn't store any information in executable memory, instead, the implementation uses two +//! bit-vectors to manage allocated memory of each allocator-block. The first bit-vector called 'used' is used to +//! track used memory (where each bit represents memory size defined by granularity) and the second bit vector called +//! 'stop' is used as a sentinel to mark where the allocated area ends. +//! +//! - Internally, the allocator also uses RB tree to keep track of all blocks across all pools. Each inserted block is +//! added to the tree so it can be matched fast during `release()` and `shrink()`. +class JitAllocator { +public: + ASMJIT_NONCOPYABLE(JitAllocator) + + struct Impl { + //! Allocator options. + JitAllocatorOptions options; + //! Base block size (0 if the allocator is not initialized). + uint32_t blockSize; + //! Base granularity (0 if the allocator is not initialized). + uint32_t granularity; + //! A pattern that is used to fill unused memory if secure mode is enabled. + uint32_t fillPattern; + }; + + //! Allocator implementation (private). + Impl* _impl; + + //! \name Construction & Destruction + //! \{ + + //! Parameters that can be passed to `JitAllocator` constructor. + //! + //! Use it like this: + //! + //! ``` + //! // Zero initialize (zero means the default value) and change what you need. + //! JitAllocator::CreateParams params {}; + //! params.blockSize = 1024 * 1024; + //! + //! // Create the allocator. + //! JitAllocator allocator(¶ms); + //! ``` + struct CreateParams { + //! Allocator options. + //! + //! No options are used by default. + JitAllocatorOptions options = JitAllocatorOptions::kNone; + + //! Base size of a single block in bytes (default 64kB). + //! + //! \remarks Block size must be equal to or greater than page size and must be power of 2. If the input is not + //! valid then the default block size will be used instead. + uint32_t blockSize = 0; + + //! Base granularity (and also natural alignment) of allocations in bytes (default 64). + //! + //! Since the `JitAllocator` uses bit-arrays to mark used memory the granularity also specifies how many bytes + //! correspond to a single bit in such bit-array. Higher granularity means more waste of virtual memory (as it + //! increases the natural alignment), but smaller bit-arrays as less bits would be required per a single block. + uint32_t granularity = 0; + + //! Patter to use to fill unused memory. + //! + //! Only used if \ref JitAllocatorOptions::kCustomFillPattern is set. + uint32_t fillPattern = 0; + + // Reset the content of `CreateParams`. + inline void reset() noexcept { memset(this, 0, sizeof(*this)); } + }; + + //! Creates a `JitAllocator` instance. + ASMJIT_API explicit JitAllocator(const CreateParams* params = nullptr) noexcept; + //! Destroys the `JitAllocator` instance and release all blocks held. + ASMJIT_API ~JitAllocator() noexcept; + + inline bool isInitialized() const noexcept { return _impl->blockSize == 0; } + + //! Free all allocated memory - makes all pointers returned by `alloc()` invalid. + //! + //! \remarks This function is not thread-safe as it's designed to be used when nobody else is using allocator. + //! The reason is that there is no point of calling `reset()` when the allocator is still in use. + ASMJIT_API void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns allocator options, see `Flags`. + inline JitAllocatorOptions options() const noexcept { return _impl->options; } + //! Tests whether the allocator has the given `option` set. + inline bool hasOption(JitAllocatorOptions option) const noexcept { return uint32_t(_impl->options & option) != 0; } + + //! Returns a base block size (a minimum size of block that the allocator would allocate). + inline uint32_t blockSize() const noexcept { return _impl->blockSize; } + //! Returns granularity of the allocator. + inline uint32_t granularity() const noexcept { return _impl->granularity; } + //! Returns pattern that is used to fill unused memory if `kFlagUseFillPattern` is set. + inline uint32_t fillPattern() const noexcept { return _impl->fillPattern; } + + //! \} + + //! \name Alloc & Release + //! \{ + + //! Allocates a new memory block of the requested `size`. + //! + //! When the function is successful it stores two pointers in `rxPtrOut` and `rwPtrOut`. The pointers will be + //! different only if `kOptionUseDualMapping` was used to setup the allocator (in that case the `rxPtrOut` would + //! point to a Read+Execute region and `rwPtrOut` would point to a Read+Write region of the same memory-mapped block. + ASMJIT_API Error alloc(void** rxPtrOut, void** rwPtrOut, size_t size) noexcept; + + //! Releases a memory block returned by `alloc()`. + //! + //! \remarks This function is thread-safe. + ASMJIT_API Error release(void* rxPtr) noexcept; + + //! Frees extra memory allocated with `rxPtr` by shrinking it to the given `newSize`. + //! + //! \remarks This function is thread-safe. + ASMJIT_API Error shrink(void* rxPtr, size_t newSize) noexcept; + + //! Queries information about an allocated memory block that contains the given `rxPtr`. + //! + //! The function returns `kErrorOk` when `rxPtr` is matched and fills `rxPtrOut`, `rwPtrOut`, and `sizeOut` output + //! arguments. The returned `rxPtrOut` and `rwPtrOut` pointers point to the beginning of the block, and `sizeOut` + //! describes the total amount of bytes this allocation uses - `sizeOut` will always be aligned to the allocation + //! granularity, so for example if an allocation was 1 byte and the size granularity is 64, the returned `sizeOut` + //! will be 64 bytes, because that's what the allocator sees. + ASMJIT_API Error query(void* rxPtr, void** rxPtrOut, void** rwPtrOut, size_t* sizeOut) const noexcept; + + //! \} + + //! \name Statistics + //! \{ + + //! Statistics about `JitAllocator`. + struct Statistics { + //! Number of blocks `JitAllocator` maintains. + size_t _blockCount; + //! Number of active allocations. + size_t _allocationCount; + //! How many bytes are currently used / allocated. + size_t _usedSize; + //! How many bytes are currently reserved by the allocator. + size_t _reservedSize; + //! Allocation overhead (in bytes) required to maintain all blocks. + size_t _overheadSize; + + inline void reset() noexcept { + _blockCount = 0; + _usedSize = 0; + _reservedSize = 0; + _overheadSize = 0; + } + + //! Returns count of blocks managed by `JitAllocator` at the moment. + inline size_t blockCount() const noexcept { return _blockCount; } + //! Returns the number of active allocations. + inline size_t allocationCount() const noexcept { return _allocationCount; } + + //! Returns how many bytes are currently used. + inline size_t usedSize() const noexcept { return _usedSize; } + //! Returns the number of bytes unused by the allocator at the moment. + inline size_t unusedSize() const noexcept { return _reservedSize - _usedSize; } + //! Returns the total number of bytes bytes reserved by the allocator (sum of sizes of all blocks). + inline size_t reservedSize() const noexcept { return _reservedSize; } + //! Returns the number of bytes the allocator needs to manage the allocated memory. + inline size_t overheadSize() const noexcept { return _overheadSize; } + + inline double usedSizeAsPercent() const noexcept { + return (double(usedSize()) / (double(reservedSize()) + 1e-16)) * 100.0; + } + + inline double unusedSizeAsPercent() const noexcept { + return (double(unusedSize()) / (double(reservedSize()) + 1e-16)) * 100.0; + } + + inline double overheadSizeAsPercent() const noexcept { + return (double(overheadSize()) / (double(reservedSize()) + 1e-16)) * 100.0; + } + }; + + //! Returns JIT allocator statistics. + //! + //! \remarks This function is thread-safe. + ASMJIT_API Statistics statistics() const noexcept; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif +#endif diff --git a/src/asmjit/core/jitruntime.cpp b/src/asmjit/core/jitruntime.cpp new file mode 100644 index 0000000..491c204 --- /dev/null +++ b/src/asmjit/core/jitruntime.cpp @@ -0,0 +1,80 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_JIT + +#include "../core/cpuinfo.h" +#include "../core/jitruntime.h" + +ASMJIT_BEGIN_NAMESPACE + +JitRuntime::JitRuntime(const JitAllocator::CreateParams* params) noexcept + : _allocator(params) { + _environment = Environment::host(); + _environment.setObjectFormat(ObjectFormat::kJIT); +} + +JitRuntime::~JitRuntime() noexcept {} + +Error JitRuntime::_add(void** dst, CodeHolder* code) noexcept { + *dst = nullptr; + + ASMJIT_PROPAGATE(code->flatten()); + ASMJIT_PROPAGATE(code->resolveUnresolvedLinks()); + + size_t estimatedCodeSize = code->codeSize(); + if (ASMJIT_UNLIKELY(estimatedCodeSize == 0)) + return DebugUtils::errored(kErrorNoCodeGenerated); + + uint8_t* rx; + uint8_t* rw; + ASMJIT_PROPAGATE(_allocator.alloc((void**)&rx, (void**)&rw, estimatedCodeSize)); + + // Relocate the code. + Error err = code->relocateToBase(uintptr_t((void*)rx)); + if (ASMJIT_UNLIKELY(err)) { + _allocator.release(rx); + return err; + } + + // Recalculate the final code size and shrink the memory we allocated for it + // in case that some relocations didn't require records in an address table. + size_t codeSize = code->codeSize(); + if (codeSize < estimatedCodeSize) + _allocator.shrink(rx, codeSize); + + if (codeSize < estimatedCodeSize) + _allocator.shrink(rx, codeSize); + + { + VirtMem::ProtectJitReadWriteScope rwScope(rx, codeSize); + + for (Section* section : code->_sections) { + size_t offset = size_t(section->offset()); + size_t bufferSize = size_t(section->bufferSize()); + size_t virtualSize = size_t(section->virtualSize()); + + ASMJIT_ASSERT(offset + bufferSize <= codeSize); + memcpy(rw + offset, section->data(), bufferSize); + + if (virtualSize > bufferSize) { + ASMJIT_ASSERT(offset + virtualSize <= codeSize); + memset(rw + offset + bufferSize, 0, virtualSize - bufferSize); + } + } + } + + *dst = rx; + return kErrorOk; +} + +Error JitRuntime::_release(void* p) noexcept { + return _allocator.release(p); +} + +ASMJIT_END_NAMESPACE + +#endif diff --git a/src/asmjit/core/jitruntime.h b/src/asmjit/core/jitruntime.h new file mode 100644 index 0000000..6f35e21 --- /dev/null +++ b/src/asmjit/core/jitruntime.h @@ -0,0 +1,89 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_JITRUNTIME_H_INCLUDED +#define ASMJIT_CORE_JITRUNTIME_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_JIT + +#include "../core/codeholder.h" +#include "../core/jitallocator.h" +#include "../core/target.h" + +ASMJIT_BEGIN_NAMESPACE + +class CodeHolder; + +//! \addtogroup asmjit_virtual_memory +//! \{ + +//! JIT execution runtime is a special `Target` that is designed to store and +//! execute the generated code. +class ASMJIT_VIRTAPI JitRuntime : public Target { +public: + ASMJIT_NONCOPYABLE(JitRuntime) + + //! Virtual memory allocator. + JitAllocator _allocator; + + //! \name Construction & Destruction + //! \{ + + //! Creates a `JitRuntime` instance. + ASMJIT_API explicit JitRuntime(const JitAllocator::CreateParams* params = nullptr) noexcept; + //! Destroys the `JitRuntime` instance. + ASMJIT_API virtual ~JitRuntime() noexcept; + + inline void reset(ResetPolicy resetPolicy = ResetPolicy::kSoft) noexcept { + _allocator.reset(resetPolicy); + } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the associated `JitAllocator`. + inline JitAllocator* allocator() const noexcept { return const_cast(&_allocator); } + + //! \} + + //! \name Utilities + //! \{ + + // NOTE: To allow passing function pointers to `add()` and `release()` the + // virtual methods are prefixed with `_` and called from templates instead. + + //! Allocates memory needed for a code stored in the `CodeHolder` and relocates the code to the pointer allocated. + //! + //! The beginning of the memory allocated for the function is returned in `dst`. If failed `Error` code is returned + //! and `dst` is explicitly set to `nullptr` (this means that you don't have to set it to null before calling `add()`). + template + inline Error add(Func* dst, CodeHolder* code) noexcept { + return _add(Support::ptr_cast_impl(dst), code); + } + + //! Releases `p` which was obtained by calling `add()`. + template + inline Error release(Func p) noexcept { + return _release(Support::ptr_cast_impl(p)); + } + + //! Type-unsafe version of `add()`. + ASMJIT_API virtual Error _add(void** dst, CodeHolder* code) noexcept; + + //! Type-unsafe version of `release()`. + ASMJIT_API virtual Error _release(void* p) noexcept; + + //! \} +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif +#endif diff --git a/src/asmjit/core/logger.cpp b/src/asmjit/core/logger.cpp new file mode 100644 index 0000000..4567b3c --- /dev/null +++ b/src/asmjit/core/logger.cpp @@ -0,0 +1,69 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#ifndef ASMJIT_NO_LOGGING + +#include "../core/logger.h" +#include "../core/string.h" +#include "../core/support.h" + +ASMJIT_BEGIN_NAMESPACE + +// Logger - Implementation +// ======================= + +Logger::Logger() noexcept + : _options() {} +Logger::~Logger() noexcept {} + +Error Logger::logf(const char* fmt, ...) noexcept { + Error err; + va_list ap; + + va_start(ap, fmt); + err = logv(fmt, ap); + va_end(ap); + + return err; +} + +Error Logger::logv(const char* fmt, va_list ap) noexcept { + StringTmp<2048> sb; + ASMJIT_PROPAGATE(sb.appendVFormat(fmt, ap)); + return log(sb); +} + +// FileLogger - Implementation +// =========================== + +FileLogger::FileLogger(FILE* file) noexcept + : _file(file) {} +FileLogger::~FileLogger() noexcept {} + +Error FileLogger::_log(const char* data, size_t size) noexcept { + if (!_file) + return kErrorOk; + + if (size == SIZE_MAX) + size = strlen(data); + + fwrite(data, 1, size, _file); + return kErrorOk; +} + +// StringLogger - Implementation +// ============================= + +StringLogger::StringLogger() noexcept {} +StringLogger::~StringLogger() noexcept {} + +Error StringLogger::_log(const char* data, size_t size) noexcept { + return _content.append(data, size); +} + +ASMJIT_END_NAMESPACE + +#endif diff --git a/src/asmjit/core/logger.h b/src/asmjit/core/logger.h new file mode 100644 index 0000000..d416a50 --- /dev/null +++ b/src/asmjit/core/logger.h @@ -0,0 +1,198 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_LOGGING_H_INCLUDED +#define ASMJIT_CORE_LOGGING_H_INCLUDED + +#include "../core/inst.h" +#include "../core/string.h" +#include "../core/formatter.h" + +#ifndef ASMJIT_NO_LOGGING + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_logging +//! \{ + +//! Logging interface. +//! +//! This class can be inherited and reimplemented to fit into your own logging needs. When reimplementing a logger +//! use \ref Logger::_log() method to log customize the output. +//! +//! There are two `Logger` implementations offered by AsmJit: +//! - \ref FileLogger - logs into a `FILE*`. +//! - \ref StringLogger - concatenates all logs into a \ref String. +class ASMJIT_VIRTAPI Logger { +public: + ASMJIT_BASE_CLASS(Logger) + ASMJIT_NONCOPYABLE(Logger) + + //! Format options. + FormatOptions _options; + + //! \name Construction & Destruction + //! \{ + + //! Creates a `Logger` instance. + ASMJIT_API Logger() noexcept; + //! Destroys the `Logger` instance. + ASMJIT_API virtual ~Logger() noexcept; + + //! \} + + //! \name Format Options + //! \{ + + //! Returns \ref FormatOptions of this logger. + inline FormatOptions& options() noexcept { return _options; } + //! \overload + inline const FormatOptions& options() const noexcept { return _options; } + //! Sets formatting options of this Logger to `options`. + inline void setOptions(const FormatOptions& options) noexcept { _options = options; } + //! Resets formatting options of this Logger to defaults. + inline void resetOptions() noexcept { _options.reset(); } + + //! Returns formatting flags. + inline FormatFlags flags() const noexcept { return _options.flags(); } + //! Tests whether the logger has the given `flag` enabled. + inline bool hasFlag(FormatFlags flag) const noexcept { return _options.hasFlag(flag); } + //! Sets formatting flags to `flags`. + inline void setFlags(FormatFlags flags) noexcept { _options.setFlags(flags); } + //! Enables the given formatting `flags`. + inline void addFlags(FormatFlags flags) noexcept { _options.addFlags(flags); } + //! Disables the given formatting `flags`. + inline void clearFlags(FormatFlags flags) noexcept { _options.clearFlags(flags); } + + //! Returns indentation of a given indentation `group`. + inline uint32_t indentation(FormatIndentationGroup type) const noexcept { return _options.indentation(type); } + //! Sets indentation of the given indentation `group` to `n` spaces. + inline void setIndentation(FormatIndentationGroup type, uint32_t n) noexcept { _options.setIndentation(type, n); } + //! Resets indentation of the given indentation `group` to 0 spaces. + inline void resetIndentation(FormatIndentationGroup type) noexcept { _options.resetIndentation(type); } + + //! Returns padding of a given padding `group`. + inline size_t padding(FormatPaddingGroup type) const noexcept { return _options.padding(type); } + //! Sets padding of a given padding `group` to `n`. + inline void setPadding(FormatPaddingGroup type, uint32_t n) noexcept { _options.setPadding(type, n); } + //! Resets padding of a given padding `group` to 0, which means that a default will be used. + inline void resetPadding(FormatPaddingGroup type) noexcept { _options.resetPadding(type); } + + //! \} + + //! \name Logging Interface + //! \{ + + //! Logs `str` - must be reimplemented. + //! + //! The function can accept either a null terminated string if `size` is `SIZE_MAX` or a non-null terminated + //! string of the given `size`. The function cannot assume that the data is null terminated and must handle + //! non-null terminated inputs. + virtual Error _log(const char* data, size_t size) noexcept = 0; + + //! Logs string `str`, which is either null terminated or having size `size`. + inline Error log(const char* data, size_t size = SIZE_MAX) noexcept { return _log(data, size); } + //! Logs content of a string `str`. + inline Error log(const String& str) noexcept { return _log(str.data(), str.size()); } + + //! Formats the message by using `snprintf()` and then passes the formatted string to \ref _log(). + ASMJIT_API Error logf(const char* fmt, ...) noexcept; + + //! Formats the message by using `vsnprintf()` and then passes the formatted string to \ref _log(). + ASMJIT_API Error logv(const char* fmt, va_list ap) noexcept; + + //! \} +}; + +//! Logger that can log to a `FILE*`. +class ASMJIT_VIRTAPI FileLogger : public Logger { +public: + ASMJIT_NONCOPYABLE(FileLogger) + + FILE* _file; + + //! \name Construction & Destruction + //! \{ + + //! Creates a new `FileLogger` that logs to `FILE*`. + ASMJIT_API FileLogger(FILE* file = nullptr) noexcept; + //! Destroys the `FileLogger`. + ASMJIT_API virtual ~FileLogger() noexcept; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns the logging output stream or null if the logger has no output stream. + inline FILE* file() const noexcept { return _file; } + + //! Sets the logging output stream to `stream` or null. + //! + //! \note If the `file` is null the logging will be disabled. When a logger is attached to `CodeHolder` or any + //! emitter the logging API will always be called regardless of the output file. This means that if you really + //! want to disable logging at emitter level you must not attach a logger to it. + inline void setFile(FILE* file) noexcept { _file = file; } + + //! \} + + ASMJIT_API Error _log(const char* data, size_t size = SIZE_MAX) noexcept override; +}; + +//! Logger that stores everything in an internal string buffer. +class ASMJIT_VIRTAPI StringLogger : public Logger { +public: + ASMJIT_NONCOPYABLE(StringLogger) + + //! Logger data as string. + String _content; + + //! \name Construction & Destruction + //! \{ + + //! Create new `StringLogger`. + ASMJIT_API StringLogger() noexcept; + //! Destroys the `StringLogger`. + ASMJIT_API virtual ~StringLogger() noexcept; + + //! \} + + //! \name Logger Data Accessors + //! \{ + + //! Returns the content of the logger as \ref String. + //! + //! It can be moved, if desired. + inline String& content() noexcept { return _content; } + //! \overload + inline const String& content() const noexcept { return _content; } + + //! Returns aggregated logger data as `char*` pointer. + //! + //! The pointer is owned by `StringLogger`, it can't be modified or freed. + inline const char* data() const noexcept { return _content.data(); } + //! Returns size of the data returned by `data()`. + inline size_t dataSize() const noexcept { return _content.size(); } + + //! \} + + //! \name Logger Data Manipulation + //! \{ + + //! Clears the accumulated logger data. + inline void clear() noexcept { _content.clear(); } + + //! \} + + ASMJIT_API Error _log(const char* data, size_t size = SIZE_MAX) noexcept override; +}; + +//! \} + +ASMJIT_END_NAMESPACE + +#endif + +#endif // ASMJIT_CORE_LOGGER_H_INCLUDED diff --git a/src/asmjit/core/misc_p.h b/src/asmjit/core/misc_p.h new file mode 100644 index 0000000..5cd934e --- /dev/null +++ b/src/asmjit/core/misc_p.h @@ -0,0 +1,33 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_MISC_P_H_INCLUDED +#define ASMJIT_CORE_MISC_P_H_INCLUDED + +#include "../core/api-config.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_utilities +//! \{ + +#define ASMJIT_LOOKUP_TABLE_4(T, I) T((I)), T((I+1)), T((I+2)), T((I+3)) +#define ASMJIT_LOOKUP_TABLE_8(T, I) ASMJIT_LOOKUP_TABLE_4(T, I), ASMJIT_LOOKUP_TABLE_4(T, I + 4) +#define ASMJIT_LOOKUP_TABLE_16(T, I) ASMJIT_LOOKUP_TABLE_8(T, I), ASMJIT_LOOKUP_TABLE_8(T, I + 8) +#define ASMJIT_LOOKUP_TABLE_32(T, I) ASMJIT_LOOKUP_TABLE_16(T, I), ASMJIT_LOOKUP_TABLE_16(T, I + 16) +#define ASMJIT_LOOKUP_TABLE_40(T, I) ASMJIT_LOOKUP_TABLE_16(T, I), ASMJIT_LOOKUP_TABLE_16(T, I + 16), ASMJIT_LOOKUP_TABLE_8(T, I + 32) +#define ASMJIT_LOOKUP_TABLE_64(T, I) ASMJIT_LOOKUP_TABLE_32(T, I), ASMJIT_LOOKUP_TABLE_32(T, I + 32) +#define ASMJIT_LOOKUP_TABLE_128(T, I) ASMJIT_LOOKUP_TABLE_64(T, I), ASMJIT_LOOKUP_TABLE_64(T, I + 64) +#define ASMJIT_LOOKUP_TABLE_256(T, I) ASMJIT_LOOKUP_TABLE_128(T, I), ASMJIT_LOOKUP_TABLE_128(T, I + 128) +#define ASMJIT_LOOKUP_TABLE_512(T, I) ASMJIT_LOOKUP_TABLE_256(T, I), ASMJIT_LOOKUP_TABLE_256(T, I + 256) +#define ASMJIT_LOOKUP_TABLE_1024(T, I) ASMJIT_LOOKUP_TABLE_512(T, I), ASMJIT_LOOKUP_TABLE_512(T, I + 512) + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_MISC_P_H_INCLUDED diff --git a/src/asmjit/core/operand.cpp b/src/asmjit/core/operand.cpp new file mode 100644 index 0000000..ee02681 --- /dev/null +++ b/src/asmjit/core/operand.cpp @@ -0,0 +1,132 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/operand.h" + +ASMJIT_BEGIN_NAMESPACE + +// Operand - Tests +// =============== + +#if defined(ASMJIT_TEST) +enum class StrongEnumForImmTests : uint32_t { + kValue0, + kValue0xFFFFFFFF = 0xFFFFFFFFu +}; + +UNIT(operand) { + INFO("Checking operand sizes"); + EXPECT(sizeof(Operand) == 16); + EXPECT(sizeof(BaseReg) == 16); + EXPECT(sizeof(BaseMem) == 16); + EXPECT(sizeof(Imm) == 16); + EXPECT(sizeof(Label) == 16); + + INFO("Checking basic functionality of Operand"); + Operand a, b; + Operand dummy; + + EXPECT(a.isNone() == true); + EXPECT(a.isReg() == false); + EXPECT(a.isMem() == false); + EXPECT(a.isImm() == false); + EXPECT(a.isLabel() == false); + EXPECT(a == b); + EXPECT(a._data[0] == 0); + EXPECT(a._data[1] == 0); + + INFO("Checking basic functionality of Label"); + Label label; + EXPECT(label.isValid() == false); + EXPECT(label.id() == Globals::kInvalidId); + + INFO("Checking basic functionality of BaseReg"); + EXPECT(BaseReg().isReg() == true); + EXPECT(BaseReg().isValid() == false); + EXPECT(BaseReg()._data[0] == 0); + EXPECT(BaseReg()._data[1] == 0); + EXPECT(dummy.as().isValid() == false); + + // Create some register (not specific to any architecture). + OperandSignature rSig = OperandSignature::fromOpType(OperandType::kReg) | + OperandSignature::fromRegType(RegType::kVec128) | + OperandSignature::fromRegGroup(RegGroup::kVec) | + OperandSignature::fromSize(8); + BaseReg r1(rSig, 5); + + EXPECT(r1.isValid() == true); + EXPECT(r1.isReg() == true); + EXPECT(r1.isReg(RegType::kVec128) == true); + EXPECT(r1.isPhysReg() == true); + EXPECT(r1.isVirtReg() == false); + EXPECT(r1.signature() == rSig); + EXPECT(r1.type() == RegType::kVec128); + EXPECT(r1.group() == RegGroup::kVec); + EXPECT(r1.size() == 8); + EXPECT(r1.id() == 5); + EXPECT(r1.isReg(RegType::kVec128, 5) == true); // RegType and Id. + EXPECT(r1._data[0] == 0); + EXPECT(r1._data[1] == 0); + + // The same type of register having different id. + BaseReg r2(r1, 6); + EXPECT(r2.isValid() == true); + EXPECT(r2.isReg() == true); + EXPECT(r2.isReg(RegType::kVec128) == true); + EXPECT(r2.isPhysReg() == true); + EXPECT(r2.isVirtReg() == false); + EXPECT(r2.signature() == rSig); + EXPECT(r2.type() == r1.type()); + EXPECT(r2.group() == r1.group()); + EXPECT(r2.size() == r1.size()); + EXPECT(r2.id() == 6); + EXPECT(r2.isReg(RegType::kVec128, 6) == true); + + r1.reset(); + EXPECT(!r1.isReg()); + EXPECT(!r1.isValid()); + + INFO("Checking basic functionality of BaseMem"); + BaseMem m; + EXPECT(m.isMem()); + EXPECT(m == BaseMem()); + EXPECT(m.hasBase() == false); + EXPECT(m.hasIndex() == false); + EXPECT(m.hasOffset() == false); + EXPECT(m.isOffset64Bit() == true); + EXPECT(m.offset() == 0); + + m.setOffset(-1); + EXPECT(m.offsetLo32() == -1); + EXPECT(m.offset() == -1); + + int64_t x = int64_t(0xFF00FF0000000001u); + int32_t xHi = int32_t(0xFF00FF00u); + + m.setOffset(x); + EXPECT(m.offset() == x); + EXPECT(m.offsetLo32() == 1); + EXPECT(m.offsetHi32() == xHi); + + INFO("Checking basic functionality of Imm"); + Imm immValue(-42); + EXPECT(immValue.type() == ImmType::kInt); + EXPECT(Imm(-1).value() == -1); + EXPECT(imm(-1).value() == -1); + EXPECT(immValue.value() == -42); + EXPECT(imm(0xFFFFFFFF).value() == int64_t(0xFFFFFFFF)); + + Imm immDouble(0.4); + EXPECT(immDouble.type() == ImmType::kDouble); + EXPECT(immDouble.valueAs() == 0.4); + EXPECT(immDouble == imm(0.4)); + + EXPECT(Imm(StrongEnumForImmTests::kValue0).value() == 0); + EXPECT(Imm(StrongEnumForImmTests::kValue0xFFFFFFFF).value() == 0xFFFFFFFFu); +} +#endif + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/operand.h b/src/asmjit/core/operand.h new file mode 100644 index 0000000..02a4093 --- /dev/null +++ b/src/asmjit/core/operand.h @@ -0,0 +1,1611 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_OPERAND_H_INCLUDED +#define ASMJIT_CORE_OPERAND_H_INCLUDED + +#include "../core/archcommons.h" +#include "../core/support.h" +#include "../core/type.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_assembler +//! \{ + +//! Operand type used by \ref Operand_. +enum class OperandType : uint32_t { + //! Not an operand or not initialized. + kNone = 0, + //! Operand is a register. + kReg = 1, + //! Operand is a memory. + kMem = 2, + //! Operand is an immediate value. + kImm = 3, + //! Operand is a label. + kLabel = 4, + + //! Maximum value of `OperandType`. + kMaxValue = kLabel +}; + +static_assert(uint32_t(OperandType::kMem) == uint32_t(OperandType::kReg) + 1, + "AsmJit requires that `OperandType::kMem` equals to `OperandType::kReg + 1`"); + +//! Register mask is a convenience typedef that describes a mask where each bit describes a physical register id +//! in the same \ref RegGroup. At the moment 32 bits are enough as AsmJit doesn't support any architecture that +//! would provide more than 32 registers for a register group. +typedef uint32_t RegMask; + +//! Register type. +//! +//! Provides a unique type that can be used to identify a register or its view. +enum class RegType : uint8_t { + //! No register - unused, invalid, multiple meanings. + kNone = 0, + + //! This is not a register type. This value is reserved for a \ref Label that used in \ref BaseMem as a base. + //! + //! Label tag is used as a sub-type, forming a unique signature across all operand types as 0x1 is never associated + //! with any register type. This means that a memory operand's BASE register can be constructed from virtually any + //! operand (register vs. label) by just assigning its type (register type or label-tag) and operand id. + kLabelTag = 1, + + //! Universal type describing program counter (PC) or instruction pointer (IP) register, if the target architecture + //! actually exposes it as a separate register type, which most modern targets do. + kPC = 2, + + //! 8-bit low general purpose register (X86). + kGp8Lo = 3, + //! 8-bit high general purpose register (X86). + kGp8Hi = 4, + //! 16-bit general purpose register (X86). + kGp16 = 5, + //! 32-bit general purpose register (X86|ARM). + kGp32 = 6, + //! 64-bit general purpose register (X86|ARM). + kGp64 = 7, + //! 8-bit view of a vector register (ARM). + kVec8 = 8, + //! 16-bit view of a vector register (ARM). + kVec16 = 9, + //! 32-bit view of a vector register (ARM). + kVec32 = 10, + //! 64-bit view of a vector register (ARM). + //! + //! \note This is never used for MMX registers on X86, MMX registers have its own category. + kVec64 = 11, + //! 128-bit view of a vector register (X86|ARM). + kVec128 = 12, + //! 256-bit view of a vector register (X86). + kVec256 = 13, + //! 512-bit view of a vector register (X86). + kVec512 = 14, + //! 1024-bit view of a vector register (future). + kVec1024 = 15, + //! View of a vector register, which width is implementation specific (AArch64). + kVecNLen = 16, + + //! Mask register (X86). + kMask = 17, + + //! Start of architecture dependent register types. + kExtra = 18, + + // X86 Specific Register Types + // --------------------------- + + // X86 Specific Register Types + // =========================== + + //! Instruction pointer (RIP), only addressable in \ref x86::Mem in 64-bit targets. + kX86_Rip = kPC, + //! Low GPB register (AL, BL, CL, DL, ...). + kX86_GpbLo = kGp8Lo, + //! High GPB register (AH, BH, CH, DH only). + kX86_GpbHi = kGp8Hi, + //! GPW register. + kX86_Gpw = kGp16, + //! GPD register. + kX86_Gpd = kGp32, + //! GPQ register (64-bit). + kX86_Gpq = kGp64, + //! XMM register (SSE+). + kX86_Xmm = kVec128, + //! YMM register (AVX+). + kX86_Ymm = kVec256, + //! ZMM register (AVX512+). + kX86_Zmm = kVec512, + //! K register (AVX512+). + kX86_KReg = kMask, + //! MMX register. + kX86_Mm = kExtra + 0, + //! Segment register (None, ES, CS, SS, DS, FS, GS). + kX86_SReg = kExtra + 1, + //! Control register (CR). + kX86_CReg = kExtra + 2, + //! Debug register (DR). + kX86_DReg = kExtra + 3, + //! FPU (x87) register. + kX86_St = kExtra + 4, + //! Bound register (BND). + kX86_Bnd = kExtra + 5, + //! TMM register (AMX_TILE) + kX86_Tmm = kExtra + 6, + + // ARM Specific Register Types + // =========================== + + //! Program pointer (PC) register (AArch64). + kARM_PC = kPC, + //! 32-bit general purpose register (R or W). + kARM_GpW = kGp32, + //! 64-bit general purpose register (X). + kARM_GpX = kGp64, + //! 8-bit view of VFP/ASIMD register (B). + kARM_VecB = kVec8, + //! 16-bit view of VFP/ASIMD register (H). + kARM_VecH = kVec16, + //! 32-bit view of VFP/ASIMD register (S). + kARM_VecS = kVec32, + //! 64-bit view of VFP/ASIMD register (D). + kARM_VecD = kVec64, + //! 128-bit view of VFP/ASIMD register (Q|V). + kARM_VecV = kVec128, + + //! Maximum value of `RegType`. + kMaxValue = 31 +}; +ASMJIT_DEFINE_ENUM_COMPARE(RegType) + +//! Register group. +//! +//! Provides a unique value that identifies groups of registers and their views. +enum class RegGroup : uint8_t { + //! General purpose register group compatible with all backends. + kGp = 0, + //! Vector register group compatible with all backends. + //! + //! Describes X86 XMM|YMM|ZMM registers ARM/AArch64 V registers. + kVec = 1, + + //! Extra virtual group #2 that can be used by Compiler for register allocation. + kExtraVirt2 = 2, + //! Extra virtual group #3 that can be used by Compiler for register allocation. + kExtraVirt3 = 3, + + //! Program counter group. + kPC = 4, + + //! Extra non-virtual group that can be used by registers not managed by Compiler. + kExtraNonVirt = 5, + + // X86 Specific Register Groups + // ---------------------------- + + //! K register group (KReg) - maps to \ref RegGroup::kExtraVirt2 (X86, X86_64). + kX86_K = kExtraVirt2, + //! MMX register group (MM) - maps to \ref RegGroup::kExtraVirt3 (X86, X86_64). + kX86_MM = kExtraVirt3, + + //! Instruction pointer (X86, X86_64). + kX86_Rip = kPC, + //! Segment register group (X86, X86_64). + kX86_SReg = kExtraNonVirt + 0, + //! CR register group (X86, X86_64). + kX86_CReg = kExtraNonVirt + 1, + //! DR register group (X86, X86_64). + kX86_DReg = kExtraNonVirt + 2, + //! FPU register group (X86, X86_64). + kX86_St = kExtraNonVirt + 3, + //! BND register group (X86, X86_64). + kX86_Bnd = kExtraNonVirt + 4, + //! TMM register group (X86, X86_64). + kX86_Tmm = kExtraNonVirt + 5, + + //! First group - only used in loops. + k0 = 0, + //! Last value of a virtual register that is managed by \ref BaseCompiler. + kMaxVirt = Globals::kNumVirtGroups - 1, + //! Maximum value of `RegGroup`. + kMaxValue = 15 +}; +ASMJIT_DEFINE_ENUM_COMPARE(RegGroup) + +typedef Support::EnumValues RegGroupVirtValues; + +//! Operand signature is a 32-bit number describing \ref Operand and some of its payload. +//! +//! In AsmJit operand signature is used to store additional payload of register, memory, and immediate operands. +//! In practice the biggest pressure on OperandSignature is from \ref BaseMem and architecture specific memory +//! operands that need to store additional payload that cannot be stored elsewhere as values of all other members +//! are fully specified by \ref BaseMem. +struct OperandSignature { + //! \name Constants + //! \{ + + enum : uint32_t { + // Operand type (3 least significant bits). + // |........|........|........|.....XXX| + kOpTypeShift = 0, + kOpTypeMask = 0x07u << kOpTypeShift, + + // Register type (5 bits). + // |........|........|........|XXXXX...| + kRegTypeShift = 3, + kRegTypeMask = 0x1Fu << kRegTypeShift, + + // Register group (4 bits). + // |........|........|....XXXX|........| + kRegGroupShift = 8, + kRegGroupMask = 0x0Fu << kRegGroupShift, + + // Memory base type (5 bits). + // |........|........|........|XXXXX...| + kMemBaseTypeShift = 3, + kMemBaseTypeMask = 0x1Fu << kMemBaseTypeShift, + + // Memory index type (5 bits). + // |........|........|...XXXXX|........| + kMemIndexTypeShift = 8, + kMemIndexTypeMask = 0x1Fu << kMemIndexTypeShift, + + // Memory base+index combined (10 bits). + // |........|........|...XXXXX|XXXXX...| + kMemBaseIndexShift = 3, + kMemBaseIndexMask = 0x3FFu << kMemBaseIndexShift, + + // This memory operand represents a home-slot or stack (Compiler) (1 bit). + // |........|........|..X.....|........| + kMemRegHomeShift = 13, + kMemRegHomeFlag = 0x01u << kMemRegHomeShift, + + // Immediate type (1 bit). + // |........|........|........|....X...| + kImmTypeShift = 3, + kImmTypeMask = 0x01u << kImmTypeShift, + + // Predicate used by either registers or immediate values (4 bits). + // |........|XXXX....|........|........| + kPredicateShift = 20, + kPredicateMask = 0x0Fu << kPredicateShift, + + // Operand size (8 most significant bits). + // |XXXXXXXX|........|........|........| + kSizeShift = 24, + kSizeMask = 0xFFu << kSizeShift + }; + + //! \} + + //! \name Members + //! \{ + + uint32_t _bits; + + //! \} + + //! \name Overloaded Operators + //! + //! Overloaded operators make `OperandSignature` behave like regular integer. + //! + //! \{ + + inline constexpr bool operator!() const noexcept { return _bits != 0; } + inline constexpr explicit operator bool() const noexcept { return _bits != 0; } + + inline OperandSignature& operator|=(uint32_t x) noexcept { _bits |= x; return *this; } + inline OperandSignature& operator&=(uint32_t x) noexcept { _bits &= x; return *this; } + inline OperandSignature& operator^=(uint32_t x) noexcept { _bits ^= x; return *this; } + + inline OperandSignature& operator|=(const OperandSignature& other) noexcept { return operator|=(other._bits); } + inline OperandSignature& operator&=(const OperandSignature& other) noexcept { return operator&=(other._bits); } + inline OperandSignature& operator^=(const OperandSignature& other) noexcept { return operator^=(other._bits); } + + inline constexpr OperandSignature operator~() const noexcept { return OperandSignature{~_bits}; } + + inline constexpr OperandSignature operator|(uint32_t x) const noexcept { return OperandSignature{_bits | x}; } + inline constexpr OperandSignature operator&(uint32_t x) const noexcept { return OperandSignature{_bits & x}; } + inline constexpr OperandSignature operator^(uint32_t x) const noexcept { return OperandSignature{_bits ^ x}; } + + inline constexpr OperandSignature operator|(const OperandSignature& other) const noexcept { return OperandSignature{_bits | other._bits}; } + inline constexpr OperandSignature operator&(const OperandSignature& other) const noexcept { return OperandSignature{_bits & other._bits}; } + inline constexpr OperandSignature operator^(const OperandSignature& other) const noexcept { return OperandSignature{_bits ^ other._bits}; } + + inline constexpr bool operator==(uint32_t x) const noexcept { return _bits == x; } + inline constexpr bool operator!=(uint32_t x) const noexcept { return _bits != x; } + + inline constexpr bool operator==(const OperandSignature& other) const noexcept { return _bits == other._bits; } + inline constexpr bool operator!=(const OperandSignature& other) const noexcept { return _bits != other._bits; } + + //! \} + + //! \name Accessors + //! \{ + + inline void reset() noexcept { _bits = 0; } + + inline constexpr uint32_t bits() const noexcept { return _bits; } + inline void setBits(uint32_t bits) noexcept { _bits = bits; } + + template::value> + inline constexpr bool hasField() const noexcept { + return (_bits & kFieldMask) != 0; + } + + template::value> + inline constexpr bool hasField(uint32_t value) const noexcept { + return (_bits & kFieldMask) != value << kFieldShift; + } + + template::value> + inline constexpr uint32_t getField() const noexcept { + return (_bits >> kFieldShift) & (kFieldMask >> kFieldShift); + } + + template::value> + inline void setField(uint32_t value) noexcept { + ASMJIT_ASSERT((value & ~(kFieldMask >> kFieldShift)) == 0); + _bits = (_bits & ~kFieldMask) | (value << kFieldShift); + } + + inline constexpr OperandSignature subset(uint32_t mask) const noexcept { return OperandSignature{_bits & mask}; } + + template + inline constexpr bool matchesSignature(const OperandSignature& signature) const noexcept { + return (_bits & kFieldMask) == signature._bits; + } + + template + inline constexpr bool matchesFields(uint32_t bits) const noexcept { + return (_bits & kFieldMask) == bits; + } + + template + inline constexpr bool matchesFields(const OperandSignature& fields) const noexcept { + return (_bits & kFieldMask) == fields._bits; + } + + inline constexpr bool isValid() const noexcept { return _bits != 0; } + + inline constexpr OperandType opType() const noexcept { return (OperandType)getField(); } + + inline constexpr RegType regType() const noexcept { return (RegType)getField(); } + inline constexpr RegGroup regGroup() const noexcept { return (RegGroup)getField(); } + + inline constexpr RegType memBaseType() const noexcept { return (RegType)getField(); } + inline constexpr RegType memIndexType() const noexcept { return (RegType)getField(); } + + inline constexpr uint32_t predicate() const noexcept { return getField(); } + inline constexpr uint32_t size() const noexcept { return getField(); } + + inline void setOpType(OperandType opType) noexcept { setField(uint32_t(opType)); } + inline void setRegType(RegType regType) noexcept { setField(uint32_t(regType)); } + inline void setRegGroup(RegGroup regGroup) noexcept { setField(uint32_t(regGroup)); } + + inline void setMemBaseType(RegGroup baseType) noexcept { setField(uint32_t(baseType)); } + inline void setMemIndexType(RegGroup indexType) noexcept { setField(uint32_t(indexType)); } + + inline void setPredicate(uint32_t predicate) noexcept { setField(predicate); } + inline void setSize(uint32_t size) noexcept { setField(size); } + + //! \} + + //! \name Static Constructors + //! \{ + + static inline constexpr OperandSignature fromBits(uint32_t bits) noexcept { + return OperandSignature{bits}; + } + + template + static inline constexpr OperandSignature fromValue(const T& value) noexcept { + return OperandSignature{uint32_t(value) << Support::ConstCTZ::value}; + } + + static inline constexpr OperandSignature fromOpType(OperandType opType) noexcept { + return OperandSignature{uint32_t(opType) << kOpTypeShift}; + } + + static inline constexpr OperandSignature fromRegType(RegType regType) noexcept { + return OperandSignature{uint32_t(regType) << kRegTypeShift}; + } + + static inline constexpr OperandSignature fromRegGroup(RegGroup regGroup) noexcept { + return OperandSignature{uint32_t(regGroup) << kRegGroupShift}; + } + + static inline constexpr OperandSignature fromRegTypeAndGroup(RegType regType, RegGroup regGroup) noexcept { + return fromRegType(regType) | fromRegGroup(regGroup); + } + + static inline constexpr OperandSignature fromMemBaseType(RegType baseType) noexcept { + return OperandSignature{uint32_t(baseType) << kMemBaseTypeShift}; + } + + static inline constexpr OperandSignature fromMemIndexType(RegType indexType) noexcept { + return OperandSignature{uint32_t(indexType) << kMemIndexTypeShift}; + } + + static inline constexpr OperandSignature fromPredicate(uint32_t predicate) noexcept { + return OperandSignature{predicate << kPredicateShift}; + } + + static inline constexpr OperandSignature fromSize(uint32_t size) noexcept { + return OperandSignature{size << kSizeShift}; + } + + //! \} +}; + +//! Base class representing an operand in AsmJit (non-default constructed version). +//! +//! Contains no initialization code and can be used safely to define an array of operands that won't be initialized. +//! This is a \ref Operand base structure designed to be statically initialized, static const, or to be used by user +//! code to define an array of operands without having them default initialized at construction time. +//! +//! The key difference between \ref Operand and \ref Operand_ is: +//! +//! ``` +//! Operand_ xArray[10]; // Not initialized, contains garbage. +//! Operand_ yArray[10] {}; // All operands initialized to none explicitly (zero initialized). +//! Operand yArray[10]; // All operands initialized to none implicitly (zero initialized). +//! ``` +struct Operand_ { + //! \name Types + //! \{ + + typedef OperandSignature Signature; + + //! \} + + //! \name Constants + //! \{ + + // Indexes to `_data` array. + enum DataIndex : uint32_t { + kDataMemIndexId = 0, + kDataMemOffsetLo = 1, + + kDataImmValueLo = ASMJIT_ARCH_LE ? 0 : 1, + kDataImmValueHi = ASMJIT_ARCH_LE ? 1 : 0 + }; + + //! Constants useful for VirtId <-> Index translation. + enum VirtIdConstants : uint32_t { + //! Minimum valid packed-id. + kVirtIdMin = 256, + //! Maximum valid packed-id, excludes Globals::kInvalidId. + kVirtIdMax = Globals::kInvalidId - 1, + //! Count of valid packed-ids. + kVirtIdCount = uint32_t(kVirtIdMax - kVirtIdMin + 1) + }; + + //! \} + + //! \name Members + //! \{ + + //! Provides operand type and additional payload. + Signature _signature; + //! Either base id as used by memory operand or any id as used by others. + uint32_t _baseId; + + //! Data specific to the operand type. + //! + //! The reason we don't use union is that we have `constexpr` constructors that construct operands and other + //!`constexpr` functions that return whether another Operand or something else. These cannot generally work with + //! unions so we also cannot use `union` if we want to be standard compliant. + uint32_t _data[2]; + + //! \} + + //! Tests whether the given `id` is a valid virtual register id. Since AsmJit supports both physical and virtual + //! registers it must be able to distinguish between these two. The idea is that physical registers are always + //! limited in size, so virtual identifiers start from `kVirtIdMin` and end at `kVirtIdMax`. + static inline bool isVirtId(uint32_t id) noexcept { return id - kVirtIdMin < uint32_t(kVirtIdCount); } + //! Converts a real-id into a packed-id that can be stored in Operand. + static inline uint32_t indexToVirtId(uint32_t id) noexcept { return id + kVirtIdMin; } + //! Converts a packed-id back to real-id. + static inline uint32_t virtIdToIndex(uint32_t id) noexcept { return id - kVirtIdMin; } + + //! \name Construction & Destruction + //! \{ + + //! \cond INTERNAL + //! Initializes a `BaseReg` operand from `signature` and register `id`. + inline void _initReg(const Signature& signature, uint32_t id) noexcept { + _signature = signature; + _baseId = id; + _data[0] = 0; + _data[1] = 0; + } + //! \endcond + + //! Initializes the operand from `other` operand (used by operator overloads). + inline void copyFrom(const Operand_& other) noexcept { memcpy(this, &other, sizeof(Operand_)); } + + //! Resets the `Operand` to none. + //! + //! None operand is defined the following way: + //! - Its signature is zero (OperandType::kNone, and the rest zero as well). + //! - Its id is `0`. + //! - The reserved8_4 field is set to `0`. + //! - The reserved12_4 field is set to zero. + //! + //! In other words, reset operands have all members set to zero. Reset operand must match the Operand state + //! right after its construction. Alternatively, if you have an array of operands, you can simply use `memset()`. + //! + //! ``` + //! using namespace asmjit; + //! + //! Operand a; + //! Operand b; + //! assert(a == b); + //! + //! b = x86::eax; + //! assert(a != b); + //! + //! b.reset(); + //! assert(a == b); + //! + //! memset(&b, 0, sizeof(Operand)); + //! assert(a == b); + //! ``` + inline void reset() noexcept { + _signature.reset(); + _baseId = 0; + _data[0] = 0; + _data[1] = 0; + } + + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Tests whether this operand is the same as `other`. + inline constexpr bool operator==(const Operand_& other) const noexcept { return equals(other); } + //! Tests whether this operand is not the same as `other`. + inline constexpr bool operator!=(const Operand_& other) const noexcept { return !equals(other); } + + //! \} + + //! \name Cast + //! \{ + + //! Casts this operand to `T` type. + template + inline T& as() noexcept { return static_cast(*this); } + + //! Casts this operand to `T` type (const). + template + inline const T& as() const noexcept { return static_cast(*this); } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the operand's signature matches the signature of the `other` operand. + inline constexpr bool hasSignature(const Operand_& other) const noexcept { return _signature == other._signature; } + //! Tests whether the operand's signature matches the given signature `sign`. + inline constexpr bool hasSignature(const Signature& other) const noexcept { return _signature == other; } + + //! Returns operand signature as unsigned 32-bit integer. + //! + //! Signature is first 4 bytes of the operand data. It's used mostly for operand checking as it's + //! much faster to check packed 4 bytes at once than having to check these bytes individually. + inline constexpr Signature signature() const noexcept { return _signature; } + + //! Sets the operand signature, see `signature()`. + //! + //! \note Improper use of `setSignature()` can lead to hard-to-debug errors. + inline void setSignature(const Signature& signature) noexcept { _signature = signature; } + + //! Returns the type of the operand, see `OpType`. + inline constexpr OperandType opType() const noexcept { return _signature.opType(); } + //! Tests whether the operand is none (`OperandType::kNone`). + inline constexpr bool isNone() const noexcept { return _signature == Signature::fromBits(0); } + //! Tests whether the operand is a register (`OperandType::kReg`). + inline constexpr bool isReg() const noexcept { return opType() == OperandType::kReg; } + //! Tests whether the operand is a memory location (`OperandType::kMem`). + inline constexpr bool isMem() const noexcept { return opType() == OperandType::kMem; } + //! Tests whether the operand is an immediate (`OperandType::kImm`). + inline constexpr bool isImm() const noexcept { return opType() == OperandType::kImm; } + //! Tests whether the operand is a label (`OperandType::kLabel`). + inline constexpr bool isLabel() const noexcept { return opType() == OperandType::kLabel; } + + //! Tests whether the operand is a physical register. + inline constexpr bool isPhysReg() const noexcept { return isReg() && _baseId < 0xFFu; } + //! Tests whether the operand is a virtual register. + inline constexpr bool isVirtReg() const noexcept { return isReg() && _baseId > 0xFFu; } + + //! Tests whether the operand specifies a size (i.e. the size is not zero). + inline constexpr bool hasSize() const noexcept { return _signature.hasField(); } + //! Tests whether the size of the operand matches `size`. + inline constexpr bool hasSize(uint32_t s) const noexcept { return size() == s; } + + //! Returns the size of the operand in bytes. + //! + //! The value returned depends on the operand type: + //! * None - Should always return zero size. + //! * Reg - Should always return the size of the register. If the register size depends on architecture + //! (like `x86::CReg` and `x86::DReg`) the size returned should be the greatest possible (so it + //! should return 64-bit size in such case). + //! * Mem - Size is optional and will be in most cases zero. + //! * Imm - Should always return zero size. + //! * Label - Should always return zero size. + inline constexpr uint32_t size() const noexcept { return _signature.getField(); } + + //! Returns the operand id. + //! + //! The value returned should be interpreted accordingly to the operand type: + //! * None - Should be `0`. + //! * Reg - Physical or virtual register id. + //! * Mem - Multiple meanings - BASE address (register or label id), or high value of a 64-bit absolute address. + //! * Imm - Should be `0`. + //! * Label - Label id if it was created by using `newLabel()` or `Globals::kInvalidId` if the label is invalid or + //! not initialized. + inline constexpr uint32_t id() const noexcept { return _baseId; } + + //! Tests whether the operand is 100% equal to `other` operand. + //! + //! \note This basically performs a binary comparison, if aby bit is + //! different the operands are not equal. + inline constexpr bool equals(const Operand_& other) const noexcept { + return (_signature == other._signature) & + (_baseId == other._baseId ) & + (_data[0] == other._data[0] ) & + (_data[1] == other._data[1] ) ; + } + + //! Tests whether the operand is a register matching the given register `type`. + inline constexpr bool isReg(RegType type) const noexcept { + return _signature.subset(Signature::kOpTypeMask | Signature::kRegTypeMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegType(type)); + } + + //! Tests whether the operand is register and of register `type` and `id`. + inline constexpr bool isReg(RegType type, uint32_t id) const noexcept { + return isReg(type) && this->id() == id; + } + + //! Tests whether the operand is a register or memory. + inline constexpr bool isRegOrMem() const noexcept { + return Support::isBetween(uint32_t(opType()), uint32_t(OperandType::kReg), uint32_t(OperandType::kMem)); + } + + //! \} +}; + +//! Base class representing an operand in AsmJit (default constructed version). +class Operand : public Operand_ { +public: + //! \name Construction & Destruction + //! \{ + + //! Creates `kOpNone` operand having all members initialized to zero. + inline constexpr Operand() noexcept + : Operand_{ Signature::fromOpType(OperandType::kNone), 0u, { 0u, 0u }} {} + + //! Creates a cloned `other` operand. + inline constexpr Operand(const Operand& other) noexcept = default; + + //! Creates a cloned `other` operand. + inline constexpr explicit Operand(const Operand_& other) + : Operand_(other) {} + + //! Creates an operand initialized to raw `[u0, u1, u2, u3]` values. + inline constexpr Operand(Globals::Init_, const Signature& u0, uint32_t u1, uint32_t u2, uint32_t u3) noexcept + : Operand_{ u0, u1, { u2, u3 }} {} + + //! Creates an uninitialized operand (dangerous). + inline explicit Operand(Globals::NoInit_) noexcept {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline Operand& operator=(const Operand& other) noexcept = default; + inline Operand& operator=(const Operand_& other) noexcept { return operator=(static_cast(other)); } + + //! \} + + //! \name Clone + //! \{ + + //! Clones this operand and returns its copy. + inline constexpr Operand clone() const noexcept { return Operand(*this); } + + //! \} +}; + +static_assert(sizeof(Operand) == 16, "asmjit::Operand must be exactly 16 bytes long"); + +//! Label (jump target or data location). +//! +//! Label represents a location in code typically used as a jump target, but may be also a reference to some data or +//! a static variable. Label has to be explicitly created by BaseEmitter. +//! +//! Example of using labels: +//! +//! ``` +//! // Create some emitter (for example x86::Assembler). +//! x86::Assembler a; +//! +//! // Create Label instance. +//! Label L1 = a.newLabel(); +//! +//! // ... your code ... +//! +//! // Using label. +//! a.jump(L1); +//! +//! // ... your code ... +//! +//! // Bind label to the current position, see `BaseEmitter::bind()`. +//! a.bind(L1); +//! ``` +class Label : public Operand { +public: + //! \name Construction & Destruction + //! \{ + + //! Creates a label operand without ID (you must set the ID to make it valid). + inline constexpr Label() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kLabel), Globals::kInvalidId, 0, 0) {} + + //! Creates a cloned label operand of `other`. + inline constexpr Label(const Label& other) noexcept + : Operand(other) {} + + //! Creates a label operand of the given `id`. + inline constexpr explicit Label(uint32_t id) noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kLabel), id, 0, 0) {} + + inline explicit Label(Globals::NoInit_) noexcept + : Operand(Globals::NoInit) {} + + //! Resets the label, will reset all properties and set its ID to `Globals::kInvalidId`. + inline void reset() noexcept { + _signature = Signature::fromOpType(OperandType::kLabel); + _baseId = Globals::kInvalidId; + _data[0] = 0; + _data[1] = 0; + } + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline Label& operator=(const Label& other) noexcept = default; + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether the label was created by CodeHolder and/or an attached emitter. + inline constexpr bool isValid() const noexcept { return _baseId != Globals::kInvalidId; } + //! Sets the label `id`. + inline void setId(uint32_t id) noexcept { _baseId = id; } + + //! \} +}; + +//! \cond INTERNAL +//! Default register traits. +struct BaseRegTraits { + enum : uint32_t { + //! \ref TypeId representing this register type, could be \ref TypeId::kVoid if such type doesn't exist. + kTypeId = uint32_t(TypeId::kVoid), + //! RegType is not valid by default. + kValid = 0, + //! Count of registers (0 if none). + kCount = 0, + + //! Zero type by default (defeaults to None). + kType = uint32_t(RegType::kNone), + //! Zero group by default (defaults to GP). + kGroup = uint32_t(RegGroup::kGp), + //! No size by default. + kSize = 0, + + //! Empty signature by default (not even having operand type set to register). + kSignature = 0 + }; +}; +//! \endcond + +//! Physical or virtual register operand. +class BaseReg : public Operand { +public: + //! \name Constants + //! \{ + + enum : uint32_t { + //! None or any register (mostly internal). + kIdBad = 0xFFu, + + kBaseSignatureMask = + Signature::kOpTypeMask | + Signature::kRegTypeMask | + Signature::kRegGroupMask | + Signature::kSizeMask, + + kTypeNone = uint32_t(RegType::kNone), + kSignature = Signature::fromOpType(OperandType::kReg).bits() + }; + + //! \} + + //! \name Construction & Destruction + //! \{ + + //! Creates a dummy register operand. + inline constexpr BaseReg() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kReg), kIdBad, 0, 0) {} + + //! Creates a new register operand which is the same as `other` . + inline constexpr BaseReg(const BaseReg& other) noexcept + : Operand(other) {} + + //! Creates a new register operand compatible with `other`, but with a different `id`. + inline constexpr BaseReg(const BaseReg& other, uint32_t id) noexcept + : Operand(Globals::Init, other._signature, id, 0, 0) {} + + //! Creates a register initialized to the given `signature` and `id`. + inline constexpr BaseReg(const Signature& signature, uint32_t id) noexcept + : Operand(Globals::Init, signature, id, 0, 0) {} + + inline explicit BaseReg(Globals::NoInit_) noexcept + : Operand(Globals::NoInit) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline BaseReg& operator=(const BaseReg& other) noexcept = default; + + //! \} + + //! \name Accessors + //! \{ + + //! Returns base signature of the register associated with each register type. + //! + //! Base signature only contains the operand type, register type, register group, and register size. It doesn't + //! contain element type, predicate, or other architecture-specific data. Base signature is a signature that is + //! provided by architecture-specific `RegTraits`, like \ref x86::RegTraits. + inline constexpr OperandSignature baseSignature() const noexcept { return _signature & kBaseSignatureMask; } + + //! Tests whether the operand's base signature matches the given signature `sign`. + inline constexpr bool hasBaseSignature(uint32_t signature) const noexcept { return baseSignature() == signature; } + //! Tests whether the operand's base signature matches the given signature `sign`. + inline constexpr bool hasBaseSignature(const OperandSignature& signature) const noexcept { return baseSignature() == signature; } + //! Tests whether the operand's base signature matches the base signature of the `other` operand. + inline constexpr bool hasBaseSignature(const BaseReg& other) const noexcept { return baseSignature() == other.baseSignature(); } + + //! Tests whether this register is the same as `other`. + //! + //! This is just an optimization. Registers by default only use the first 8 bytes of Operand data, so this method + //! takes advantage of this knowledge and only compares these 8 bytes. If both operands were created correctly + //! both \ref equals() and \ref isSame() should give the same answer, however, if any of these two contains garbage + //! or other metadata in the upper 8 bytes then \ref isSame() may return `true` in cases in which \ref equals() + //! returns false. + inline constexpr bool isSame(const BaseReg& other) const noexcept { + return (_signature == other._signature) & (_baseId == other._baseId); + } + + //! Tests whether the register is valid (either virtual or physical). + inline constexpr bool isValid() const noexcept { return (_signature != 0) & (_baseId != kIdBad); } + + //! Tests whether this is a physical register. + inline constexpr bool isPhysReg() const noexcept { return _baseId < kIdBad; } + //! Tests whether this is a virtual register. + inline constexpr bool isVirtReg() const noexcept { return _baseId > kIdBad; } + + //! Tests whether the register type matches `type` - same as `isReg(type)`, provided for convenience. + inline constexpr bool isType(RegType type) const noexcept { return _signature.subset(Signature::kRegTypeMask) == Signature::fromRegType(type); } + //! Tests whether the register group matches `group`. + inline constexpr bool isGroup(RegGroup group) const noexcept { return _signature.subset(Signature::kRegGroupMask) == Signature::fromRegGroup(group); } + + //! Tests whether the register is a general purpose register (any size). + inline constexpr bool isGp() const noexcept { return isGroup(RegGroup::kGp); } + //! Tests whether the register is a vector register. + inline constexpr bool isVec() const noexcept { return isGroup(RegGroup::kVec); } + + using Operand_::isReg; + + //! Same as `isType()`, provided for convenience. + inline constexpr bool isReg(RegType rType) const noexcept { return isType(rType); } + //! Tests whether the register type matches `type` and register id matches `id`. + inline constexpr bool isReg(RegType rType, uint32_t id) const noexcept { return isType(rType) && this->id() == id; } + + //! Returns the register type. + inline constexpr RegType type() const noexcept { return _signature.regType(); } + //! Returns the register group. + inline constexpr RegGroup group() const noexcept { return _signature.regGroup(); } + + //! Returns operation predicate of the register (ARM/AArch64). + //! + //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp + //! of the register. + inline constexpr uint32_t predicate() const noexcept { return _signature.getField(); } + + //! Sets operation predicate of the register to `predicate` (ARM/AArch64). + //! + //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp + //! of the register. + inline void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } + + //! Resets shift operation type of the register to the default value (ARM/AArch64). + inline void resetPredicate() noexcept { _signature.setField(0); } + + //! Clones the register operand. + inline constexpr BaseReg clone() const noexcept { return BaseReg(*this); } + + //! Casts this register to `RegT` by also changing its signature. + //! + //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. + template + inline constexpr RegT cloneAs() const noexcept { return RegT(Signature(RegT::kSignature), id()); } + + //! Casts this register to `other` by also changing its signature. + //! + //! \note Improper use of `cloneAs()` can lead to hard-to-debug errors. + template + inline constexpr RegT cloneAs(const RegT& other) const noexcept { return RegT(other.signature(), id()); } + + //! Sets the register id to `id`. + inline void setId(uint32_t id) noexcept { _baseId = id; } + + //! Sets a 32-bit operand signature based on traits of `RegT`. + template + inline void setSignatureT() noexcept { _signature = RegT::kSignature; } + + //! Sets the register `signature` and `id`. + inline void setSignatureAndId(const OperandSignature& signature, uint32_t id) noexcept { + _signature = signature; + _baseId = id; + } + + //! \} + + //! \name Static Functions + //! \{ + + //! Tests whether the `op` operand is a general purpose register. + static inline bool isGp(const Operand_& op) noexcept { + // Check operand type and register group. Not interested in register type and size. + return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kGp)); + } + + //! Tests whether the `op` operand is a vector register. + static inline bool isVec(const Operand_& op) noexcept { + // Check operand type and register group. Not interested in register type and size. + return op.signature().subset(Signature::kOpTypeMask | Signature::kRegGroupMask) == (Signature::fromOpType(OperandType::kReg) | Signature::fromRegGroup(RegGroup::kVec)); + } + + //! Tests whether the `op` is a general purpose register of the given `id`. + static inline bool isGp(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isGp(op)) & unsigned(op.id() == id)); } + //! Tests whether the `op` is a vector register of the given `id`. + static inline bool isVec(const Operand_& op, uint32_t id) noexcept { return bool(unsigned(isVec(op)) & unsigned(op.id() == id)); } + + //! \} +}; + +//! RegOnly is 8-byte version of `BaseReg` that allows to store either register or nothing. +//! +//! It's designed to decrease the space consumed by an extra "operand" in \ref BaseEmitter and \ref InstNode. +struct RegOnly { + //! \name Types + //! \{ + + typedef OperandSignature Signature; + + //! \} + + //! Operand signature - only \ref OperandType::kNone and \ref OperandType::kReg are supported. + Signature _signature; + //! Physical or virtual register id. + uint32_t _id; + + //! \name Construction & Destruction + //! \{ + + //! Initializes the `RegOnly` instance to hold register `signature` and `id`. + inline void init(const OperandSignature& signature, uint32_t id) noexcept { + _signature = signature; + _id = id; + } + + inline void init(const BaseReg& reg) noexcept { init(reg.signature(), reg.id()); } + inline void init(const RegOnly& reg) noexcept { init(reg.signature(), reg.id()); } + + //! Resets the `RegOnly` members to zeros (none). + inline void reset() noexcept { init(Signature::fromBits(0), 0); } + + //! \} + + //! \name Accessors + //! \{ + + //! Tests whether this ExtraReg is none (same as calling `Operand_::isNone()`). + inline constexpr bool isNone() const noexcept { return _signature == 0; } + //! Tests whether the register is valid (either virtual or physical). + inline constexpr bool isReg() const noexcept { return _signature != 0; } + + //! Tests whether this is a physical register. + inline constexpr bool isPhysReg() const noexcept { return _id < BaseReg::kIdBad; } + //! Tests whether this is a virtual register (used by `BaseCompiler`). + inline constexpr bool isVirtReg() const noexcept { return _id > BaseReg::kIdBad; } + + //! Returns the register signature or 0 if no register is assigned. + inline constexpr OperandSignature signature() const noexcept { return _signature; } + //! Returns the register id. + //! + //! \note Always check whether the register is assigned before using the returned identifier as + //! non-assigned `RegOnly` instance would return zero id, which is still a valid register id. + inline constexpr uint32_t id() const noexcept { return _id; } + + //! Sets the register id. + inline void setId(uint32_t id) noexcept { _id = id; } + + //! Returns the register type. + inline constexpr RegType type() const noexcept { return _signature.regType(); } + //! Returns the register group. + inline constexpr RegGroup group() const noexcept { return _signature.regGroup(); } + + //! \} + + //! \name Utilities + //! \{ + + //! Converts this ExtraReg to a real `RegT` operand. + template + inline constexpr RegT toReg() const noexcept { return RegT(_signature, _id); } + + //! \} +}; + +//! \cond INTERNAL +//! Adds a template specialization for `REG_TYPE` into the local `RegTraits`. +#define ASMJIT_DEFINE_REG_TRAITS(REG, REG_TYPE, GROUP, SIZE, COUNT, TYPE_ID) \ +template<> \ +struct RegTraits { \ + typedef REG RegT; \ + \ + enum : uint32_t { \ + kValid = uint32_t(true), \ + kCount = uint32_t(COUNT), \ + kType = uint32_t(REG_TYPE), \ + kGroup = uint32_t(GROUP), \ + kSize = uint32_t(SIZE), \ + kTypeId = uint32_t(TYPE_ID), \ + \ + kSignature = (OperandSignature::fromOpType(OperandType::kReg) | \ + OperandSignature::fromRegType(REG_TYPE) | \ + OperandSignature::fromRegGroup(GROUP) | \ + OperandSignature::fromSize(kSize)).bits(), \ + }; \ +} + +//! Adds constructors and member functions to a class that implements abstract register. Abstract register is register +//! that doesn't have type or signature yet, it's a base class like `x86::Reg` or `arm::Reg`. +#define ASMJIT_DEFINE_ABSTRACT_REG(REG, BASE) \ +public: \ + /*! Default constructor that only setups basics. */ \ + inline constexpr REG() noexcept \ + : BASE(Signature{kSignature}, kIdBad) {} \ + \ + /*! Makes a copy of the `other` register operand. */ \ + inline constexpr REG(const REG& other) noexcept \ + : BASE(other) {} \ + \ + /*! Makes a copy of the `other` register having id set to `id` */ \ + inline constexpr REG(const BaseReg& other, uint32_t id) noexcept \ + : BASE(other, id) {} \ + \ + /*! Creates a register based on `signature` and `id`. */ \ + inline constexpr REG(const OperandSignature& sgn, uint32_t id) noexcept \ + : BASE(sgn, id) {} \ + \ + /*! Creates a completely uninitialized REG register operand (garbage). */ \ + inline explicit REG(Globals::NoInit_) noexcept \ + : BASE(Globals::NoInit) {} \ + \ + /*! Creates a new register from register type and id. */ \ + static inline REG fromTypeAndId(RegType type, uint32_t id) noexcept { \ + return REG(signatureOf(type), id); \ + } \ + \ + /*! Clones the register operand. */ \ + inline constexpr REG clone() const noexcept { return REG(*this); } \ + \ + inline REG& operator=(const REG& other) noexcept = default; + +//! Adds constructors and member functions to a class that implements final register. Final registers MUST HAVE a valid +//! signature. +#define ASMJIT_DEFINE_FINAL_REG(REG, BASE, TRAITS) \ +public: \ + enum : uint32_t { \ + kThisType = TRAITS::kType, \ + kThisGroup = TRAITS::kGroup, \ + kThisSize = TRAITS::kSize, \ + kSignature = TRAITS::kSignature \ + }; \ + \ + ASMJIT_DEFINE_ABSTRACT_REG(REG, BASE) \ + \ + /*! Creates a register operand having its id set to `id`. */ \ + inline constexpr explicit REG(uint32_t id) noexcept \ + : BASE(Signature{kSignature}, id) {} +//! \endcond + +//! Base class for all memory operands. +//! +//! The data is split into the following parts: +//! +//! - BASE - Base register or label - requires 36 bits total. 4 bits are used to encode the type of the BASE operand +//! (label vs. register type) and the remaining 32 bits define the BASE id, which can be a physical or virtual +//! register index. If BASE type is zero, which is never used as a register type and label doesn't use it as well +//! then BASE field contains a high DWORD of a possible 64-bit absolute address, which is possible on X64. +//! +//! - INDEX - Index register (or theoretically Label, which doesn't make sense). Encoding is similar to BASE - it +//! also requires 36 bits and splits the encoding to INDEX type (4 bits defining the register type) and 32-bit id. +//! +//! - OFFSET - A relative offset of the address. Basically if BASE is specified the relative displacement adjusts +//! BASE and an optional INDEX. if BASE is not specified then the OFFSET should be considered as ABSOLUTE address +//! (at least on X86). In that case its low 32 bits are stored in DISPLACEMENT field and the remaining high 32 +//! bits are stored in BASE. +//! +//! - OTHER - There is rest 8 bits that can be used for whatever purpose. For example \ref x86::Mem operand uses +//! these bits to store segment override prefix and index shift (or scale). +class BaseMem : public Operand { +public: + //! \name Construction & Destruction + //! \{ + + //! Creates a default `BaseMem` operand, that points to [0]. + inline constexpr BaseMem() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kMem), 0, 0, 0) {} + + //! Creates a `BaseMem` operand that is a clone of `other`. + inline constexpr BaseMem(const BaseMem& other) noexcept + : Operand(other) {} + + //! Creates a `BaseMem` operand from `baseReg` and `offset`. + //! + //! \note This is an architecture independent constructor that can be used to create an architecture + //! independent memory operand to be used in portable code that can handle multiple architectures. + inline constexpr explicit BaseMem(const BaseReg& baseReg, int32_t offset = 0) noexcept + : Operand(Globals::Init, + Signature::fromOpType(OperandType::kMem) | Signature::fromMemBaseType(baseReg.type()), + baseReg.id(), + 0, + uint32_t(offset)) {} + + //! \cond INTERNAL + //! Creates a `BaseMem` operand from 4 integers as used by `Operand_` struct. + inline constexpr BaseMem(const OperandSignature& u0, uint32_t baseId, uint32_t indexId, int32_t offset) noexcept + : Operand(Globals::Init, u0, baseId, indexId, uint32_t(offset)) {} + //! \endcond + + //! Creates a completely uninitialized `BaseMem` operand. + inline explicit BaseMem(Globals::NoInit_) noexcept + : Operand(Globals::NoInit) {} + + //! Resets the memory operand - after the reset the memory points to [0]. + inline void reset() noexcept { + _signature = Signature::fromOpType(OperandType::kMem); + _baseId = 0; + _data[0] = 0; + _data[1] = 0; + } + + //! \} + + //! \name Overloaded Operators + //! \{ + + inline BaseMem& operator=(const BaseMem& other) noexcept { copyFrom(other); return *this; } + + //! \} + + //! \name Accessors + //! \{ + + //! Clones the memory operand. + inline constexpr BaseMem clone() const noexcept { return BaseMem(*this); } + + //! Creates a new copy of this memory operand adjusted by `off`. + inline BaseMem cloneAdjusted(int64_t off) const noexcept { + BaseMem result(*this); + result.addOffset(off); + return result; + } + + //! Tests whether this memory operand is a register home (only used by \ref asmjit_compiler) + inline constexpr bool isRegHome() const noexcept { return _signature.hasField(); } + //! Mark this memory operand as register home (only used by \ref asmjit_compiler). + inline void setRegHome() noexcept { _signature |= Signature::kMemRegHomeFlag; } + //! Marks this operand to not be a register home (only used by \ref asmjit_compiler). + inline void clearRegHome() noexcept { _signature &= ~Signature::kMemRegHomeFlag; } + + //! Tests whether the memory operand has a BASE register or label specified. + inline constexpr bool hasBase() const noexcept { + return (_signature & Signature::kMemBaseTypeMask) != 0; + } + + //! Tests whether the memory operand has an INDEX register specified. + inline constexpr bool hasIndex() const noexcept { + return (_signature & Signature::kMemIndexTypeMask) != 0; + } + + //! Tests whether the memory operand has BASE or INDEX register. + inline constexpr bool hasBaseOrIndex() const noexcept { + return (_signature & Signature::kMemBaseIndexMask) != 0; + } + + //! Tests whether the memory operand has BASE and INDEX register. + inline constexpr bool hasBaseAndIndex() const noexcept { + return (_signature & Signature::kMemBaseTypeMask) != 0 && (_signature & Signature::kMemIndexTypeMask) != 0; + } + + //! Tests whether the BASE operand is a label. + inline constexpr bool hasBaseLabel() const noexcept { + return _signature.subset(Signature::kMemBaseTypeMask) == Signature::fromMemBaseType(RegType::kLabelTag); + } + + //! Tests whether the BASE operand is a register (registers start after `RegType::kLabelTag`). + inline constexpr bool hasBaseReg() const noexcept { + return _signature.subset(Signature::kMemBaseTypeMask).bits() > Signature::fromMemBaseType(RegType::kLabelTag).bits(); + } + + //! Tests whether the INDEX operand is a register (registers start after `RegType::kLabelTag`). + inline constexpr bool hasIndexReg() const noexcept { + return _signature.subset(Signature::kMemIndexTypeMask).bits() > Signature::fromMemIndexType(RegType::kLabelTag).bits(); + } + + //! Returns the type of the BASE register (0 if this memory operand doesn't use the BASE register). + //! + //! \note If the returned type is one (a value never associated to a register type) the BASE is not register, but it + //! is a label. One equals to `kLabelTag`. You should always check `hasBaseLabel()` before using `baseId()` result. + inline constexpr RegType baseType() const noexcept { return _signature.memBaseType(); } + + //! Returns the type of an INDEX register (0 if this memory operand doesn't + //! use the INDEX register). + inline constexpr RegType indexType() const noexcept { return _signature.memIndexType(); } + + //! This is used internally for BASE+INDEX validation. + inline constexpr uint32_t baseAndIndexTypes() const noexcept { return _signature.getField(); } + + //! Returns both BASE (4:0 bits) and INDEX (9:5 bits) types combined into a single value. + //! + //! \remarks Returns id of the BASE register or label (if the BASE was specified as label). + inline constexpr uint32_t baseId() const noexcept { return _baseId; } + + //! Returns the id of the INDEX register. + inline constexpr uint32_t indexId() const noexcept { return _data[kDataMemIndexId]; } + + //! Sets the id of the BASE register (without modifying its type). + inline void setBaseId(uint32_t id) noexcept { _baseId = id; } + //! Sets the id of the INDEX register (without modifying its type). + inline void setIndexId(uint32_t id) noexcept { _data[kDataMemIndexId] = id; } + + //! Sets the base register to type and id of the given `base` operand. + inline void setBase(const BaseReg& base) noexcept { return _setBase(base.type(), base.id()); } + //! Sets the index register to type and id of the given `index` operand. + inline void setIndex(const BaseReg& index) noexcept { return _setIndex(index.type(), index.id()); } + + //! \cond INTERNAL + inline void _setBase(RegType type, uint32_t id) noexcept { + _signature.setField(uint32_t(type)); + _baseId = id; + } + + inline void _setIndex(RegType type, uint32_t id) noexcept { + _signature.setField(uint32_t(type)); + _data[kDataMemIndexId] = id; + } + //! \endcond + + //! Resets the memory operand's BASE register or label. + inline void resetBase() noexcept { _setBase(RegType::kNone, 0); } + //! Resets the memory operand's INDEX register. + inline void resetIndex() noexcept { _setIndex(RegType::kNone, 0); } + + //! Sets the memory operand size (in bytes). + inline void setSize(uint32_t size) noexcept { _signature.setField(size); } + + //! Tests whether the memory operand has a 64-bit offset or absolute address. + //! + //! If this is true then `hasBase()` must always report false. + inline constexpr bool isOffset64Bit() const noexcept { return baseType() == RegType::kNone; } + + //! Tests whether the memory operand has a non-zero offset or absolute address. + inline constexpr bool hasOffset() const noexcept { + return (_data[kDataMemOffsetLo] | uint32_t(_baseId & Support::bitMaskFromBool(isOffset64Bit()))) != 0; + } + + //! Returns either relative offset or absolute address as 64-bit integer. + inline constexpr int64_t offset() const noexcept { + return isOffset64Bit() ? int64_t(uint64_t(_data[kDataMemOffsetLo]) | (uint64_t(_baseId) << 32)) + : int64_t(int32_t(_data[kDataMemOffsetLo])); // Sign extend 32-bit offset. + } + + //! Returns a 32-bit low part of a 64-bit offset or absolute address. + inline constexpr int32_t offsetLo32() const noexcept { return int32_t(_data[kDataMemOffsetLo]); } + //! Returns a 32-but high part of a 64-bit offset or absolute address. + //! + //! \note This function is UNSAFE and returns garbage if `isOffset64Bit()` + //! returns false. Never use it blindly without checking it first. + inline constexpr int32_t offsetHi32() const noexcept { return int32_t(_baseId); } + + //! Sets a 64-bit offset or an absolute address to `offset`. + //! + //! \note This functions attempts to set both high and low parts of a 64-bit offset, however, if the operand has + //! a BASE register it will store only the low 32 bits of the offset / address as there is no way to store both + //! BASE and 64-bit offset, and there is currently no architecture that has such capability targeted by AsmJit. + inline void setOffset(int64_t offset) noexcept { + uint32_t lo = uint32_t(uint64_t(offset) & 0xFFFFFFFFu); + uint32_t hi = uint32_t(uint64_t(offset) >> 32); + uint32_t hiMsk = Support::bitMaskFromBool(isOffset64Bit()); + + _data[kDataMemOffsetLo] = lo; + _baseId = (hi & hiMsk) | (_baseId & ~hiMsk); + } + //! Sets a low 32-bit offset to `offset` (don't use without knowing how BaseMem works). + inline void setOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] = uint32_t(offset); } + + //! Adjusts the offset by `offset`. + //! + //! \note This is a fast function that doesn't use the HI 32-bits of a 64-bit offset. Use it only if you know that + //! there is a BASE register and the offset is only 32 bits anyway. + + //! Adjusts the memory operand offset by a `offset`. + inline void addOffset(int64_t offset) noexcept { + if (isOffset64Bit()) { + int64_t result = offset + int64_t(uint64_t(_data[kDataMemOffsetLo]) | (uint64_t(_baseId) << 32)); + _data[kDataMemOffsetLo] = uint32_t(uint64_t(result) & 0xFFFFFFFFu); + _baseId = uint32_t(uint64_t(result) >> 32); + } + else { + _data[kDataMemOffsetLo] += uint32_t(uint64_t(offset) & 0xFFFFFFFFu); + } + } + + //! Adds `offset` to a low 32-bit offset part (don't use without knowing how BaseMem works). + inline void addOffsetLo32(int32_t offset) noexcept { _data[kDataMemOffsetLo] += uint32_t(offset); } + + //! Resets the memory offset to zero. + inline void resetOffset() noexcept { setOffset(0); } + + //! Resets the lo part of the memory offset to zero (don't use without knowing how BaseMem works). + inline void resetOffsetLo32() noexcept { setOffsetLo32(0); } + + //! \} +}; + +//! Type of the an immediate value. +enum class ImmType : uint32_t { + //! Immediate is integer. + kInt = 0, + //! Immediate is a floating point stored as double-precision. + kDouble = 1 +}; + +//! Immediate operands are encoded with instruction data. +class Imm : public Operand { +public: + //! \cond INTERNAL + template + struct IsConstexprConstructibleAsImmType + : public std::integral_constant::value || + std::is_pointer::value || + std::is_integral::value || + std::is_function::value> {}; + + template + struct IsConvertibleToImmType + : public std::integral_constant::value || + std::is_floating_point::value> {}; + //! \endcond + + //! \name Construction & Destruction + //! \{ + + //! Creates a new immediate value (initial value is 0). + inline constexpr Imm() noexcept + : Operand(Globals::Init, Signature::fromOpType(OperandType::kImm), 0, 0, 0) {} + + //! Creates a new immediate value from `other`. + inline constexpr Imm(const Imm& other) noexcept + : Operand(other) {} + + //! Creates a new immediate value from ARM/AArch64 specific `shift`. + inline constexpr Imm(const arm::Shift& shift) noexcept + : Operand(Globals::Init, + Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(uint32_t(shift.op())), + 0, + Support::unpackU32At0(shift.value()), + Support::unpackU32At1(shift.value())) {} + + //! Creates a new signed immediate value, assigning the value to `val` and an architecture-specific predicate + //! to `predicate`. + //! + //! \note Predicate is currently only used by ARM architectures. + template::type>::value>::type> + inline constexpr Imm(const T& val, const uint32_t predicate = 0) noexcept + : Operand(Globals::Init, + Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(predicate), + 0, + Support::unpackU32At0(int64_t(val)), + Support::unpackU32At1(int64_t(val))) {} + + inline Imm(const float& val, const uint32_t predicate = 0) noexcept + : Operand(Globals::Init, + Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(predicate), + 0, + 0, + 0) { setValue(val); } + + inline Imm(const double& val, const uint32_t predicate = 0) noexcept + : Operand(Globals::Init, + Signature::fromOpType(OperandType::kImm) | Signature::fromPredicate(predicate), + 0, + 0, + 0) { setValue(val); } + + inline explicit Imm(Globals::NoInit_) noexcept + : Operand(Globals::NoInit) {} + + //! \} + + //! \name Overloaded Operators + //! \{ + + //! Assigns the value of the `other` operand to this immediate. + inline Imm& operator=(const Imm& other) noexcept { copyFrom(other); return *this; } + + //! \} + + //! \name Accessors + //! \{ + + //! Returns immediate type. + inline constexpr ImmType type() const noexcept { return (ImmType)_signature.getField(); } + //! Sets the immediate type to `type`. + inline void setType(ImmType type) noexcept { _signature.setField(uint32_t(type)); } + //! Resets immediate type to \ref ImmType::kInt. + inline void resetType() noexcept { setType(ImmType::kInt); } + + //! Returns operation predicate of the immediate. + //! + //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp + //! of the immediate. + inline constexpr uint32_t predicate() const noexcept { return _signature.getField(); } + + //! Sets operation predicate of the immediate to `predicate`. + //! + //! The meaning depends on architecture, for example on ARM hardware this describes \ref arm::ShiftOp + //! of the immediate. + inline void setPredicate(uint32_t predicate) noexcept { _signature.setField(predicate); } + + //! Resets the shift operation type of the immediate to the default value (no operation). + inline void resetPredicate() noexcept { _signature.setField(0); } + + //! Returns the immediate value as `int64_t`, which is the internal format Imm uses. + inline constexpr int64_t value() const noexcept { + return int64_t((uint64_t(_data[kDataImmValueHi]) << 32) | _data[kDataImmValueLo]); + } + + //! Tests whether this immediate value is integer of any size. + inline constexpr uint32_t isInt() const noexcept { return type() == ImmType::kInt; } + //! Tests whether this immediate value is a double precision floating point value. + inline constexpr uint32_t isDouble() const noexcept { return type() == ImmType::kDouble; } + + //! Tests whether the immediate can be casted to 8-bit signed integer. + inline constexpr bool isInt8() const noexcept { return type() == ImmType::kInt && Support::isInt8(value()); } + //! Tests whether the immediate can be casted to 8-bit unsigned integer. + inline constexpr bool isUInt8() const noexcept { return type() == ImmType::kInt && Support::isUInt8(value()); } + //! Tests whether the immediate can be casted to 16-bit signed integer. + inline constexpr bool isInt16() const noexcept { return type() == ImmType::kInt && Support::isInt16(value()); } + //! Tests whether the immediate can be casted to 16-bit unsigned integer. + inline constexpr bool isUInt16() const noexcept { return type() == ImmType::kInt && Support::isUInt16(value()); } + //! Tests whether the immediate can be casted to 32-bit signed integer. + inline constexpr bool isInt32() const noexcept { return type() == ImmType::kInt && Support::isInt32(value()); } + //! Tests whether the immediate can be casted to 32-bit unsigned integer. + inline constexpr bool isUInt32() const noexcept { return type() == ImmType::kInt && _data[kDataImmValueHi] == 0; } + + //! Returns the immediate value casted to `T`. + //! + //! The value is masked before it's casted to `T` so the returned value is simply the representation of `T` + //! considering the original value's lowest bits. + template + inline T valueAs() const noexcept { return Support::immediateToT(value()); } + + //! Returns low 32-bit signed integer. + inline constexpr int32_t int32Lo() const noexcept { return int32_t(_data[kDataImmValueLo]); } + //! Returns high 32-bit signed integer. + inline constexpr int32_t int32Hi() const noexcept { return int32_t(_data[kDataImmValueHi]); } + //! Returns low 32-bit signed integer. + inline constexpr uint32_t uint32Lo() const noexcept { return _data[kDataImmValueLo]; } + //! Returns high 32-bit signed integer. + inline constexpr uint32_t uint32Hi() const noexcept { return _data[kDataImmValueHi]; } + + //! Sets immediate value to `val`, the value is casted to a signed 64-bit integer. + template + inline void setValue(const T& val) noexcept { + _setValueInternal(Support::immediateFromT(val), std::is_floating_point::value ? ImmType::kDouble : ImmType::kInt); + } + + inline void _setValueInternal(int64_t val, ImmType type) noexcept { + setType(type); + _data[kDataImmValueHi] = uint32_t(uint64_t(val) >> 32); + _data[kDataImmValueLo] = uint32_t(uint64_t(val) & 0xFFFFFFFFu); + } + + //! \} + + //! \name Utilities + //! \{ + + //! Clones the immediate operand. + inline constexpr Imm clone() const noexcept { return Imm(*this); } + + inline void signExtend8Bits() noexcept { setValue(int64_t(valueAs())); } + inline void signExtend16Bits() noexcept { setValue(int64_t(valueAs())); } + inline void signExtend32Bits() noexcept { setValue(int64_t(valueAs())); } + + inline void zeroExtend8Bits() noexcept { setValue(valueAs()); } + inline void zeroExtend16Bits() noexcept { setValue(valueAs()); } + inline void zeroExtend32Bits() noexcept { _data[kDataImmValueHi] = 0u; } + + //! \} +}; + +//! Creates a new immediate operand. +template +static inline constexpr Imm imm(const T& val) noexcept { return Imm(val); } + +//! \} + +namespace Globals { + //! \ingroup asmjit_assembler + //! + //! A default-constructed operand of `Operand_::kOpNone` type. + static constexpr const Operand none; +} + +//! \cond INTERNAL +namespace Support { + +template +struct ForwardOpImpl { + static inline const T& forward(const T& value) noexcept { return value; } +}; + +template +struct ForwardOpImpl { + static inline Imm forward(const T& value) noexcept { return Imm(value); } +}; + +//! Either forwards operand T or returns a new operand that wraps it if T is a type convertible to operand. +//! At the moment this is only used to convert integers, floats, and enumarations to \ref Imm operands. +template +struct ForwardOp : public ForwardOpImpl::type>::value> {}; + +} // {Support} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_OPERAND_H_INCLUDED diff --git a/src/asmjit/core/osutils.cpp b/src/asmjit/core/osutils.cpp new file mode 100644 index 0000000..fa900bf --- /dev/null +++ b/src/asmjit/core/osutils.cpp @@ -0,0 +1,84 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#include "../core/api-build_p.h" +#include "../core/osutils.h" +#include "../core/support.h" + +#if defined(_WIN32) + #include +#elif defined(__APPLE__) + #include +#else + #include + #include +#endif + +ASMJIT_BEGIN_NAMESPACE + +uint32_t OSUtils::getTickCount() noexcept { +#if defined(_WIN32) + enum HiResStatus : uint32_t { + kHiResUnknown = 0, + kHiResAvailable = 1, + kHiResNotAvailable = 2 + }; + + static std::atomic _hiResStatus(kHiResUnknown); + static volatile double _hiResFreq(0); + + uint32_t status = _hiResStatus.load(); + LARGE_INTEGER now, qpf; + + if (status != kHiResNotAvailable && ::QueryPerformanceCounter(&now)) { + double freq = _hiResFreq; + if (status == kHiResUnknown) { + // Detects the availability of high resolution counter. + if (::QueryPerformanceFrequency(&qpf)) { + freq = double(qpf.QuadPart) / 1000.0; + _hiResFreq = freq; + _hiResStatus.compare_exchange_strong(status, kHiResAvailable); + status = kHiResAvailable; + } + else { + // High resolution not available. + _hiResStatus.compare_exchange_strong(status, kHiResNotAvailable); + } + } + + if (status == kHiResAvailable) + return uint32_t(uint64_t(int64_t(double(now.QuadPart) / freq)) & 0xFFFFFFFFu); + } + + // Bail to `GetTickCount()` if we cannot use high resolution. + return ::GetTickCount(); +#elif defined(__APPLE__) + // See Apple's QA1398. + static mach_timebase_info_data_t _machTime; + + uint32_t denom = _machTime.denom; + if (ASMJIT_UNLIKELY(!denom)) { + if (mach_timebase_info(&_machTime) != KERN_SUCCESS || !(denom = _machTime.denom)) + return 0; + } + + // `mach_absolute_time()` returns nanoseconds, we want milliseconds. + uint64_t t = mach_absolute_time() / 1000000u; + t = (t * _machTime.numer) / _machTime.denom; + return uint32_t(t & 0xFFFFFFFFu); +#elif defined(_POSIX_MONOTONIC_CLOCK) && _POSIX_MONOTONIC_CLOCK >= 0 + struct timespec ts; + if (ASMJIT_UNLIKELY(clock_gettime(CLOCK_MONOTONIC, &ts) != 0)) + return 0; + + uint64_t t = (uint64_t(ts.tv_sec ) * 1000u) + (uint64_t(ts.tv_nsec) / 1000000u); + return uint32_t(t & 0xFFFFFFFFu); +#else + #pragma message("asmjit::OSUtils::getTickCount() doesn't have implementation for the target OS.") + return 0; +#endif +} + +ASMJIT_END_NAMESPACE diff --git a/src/asmjit/core/osutils.h b/src/asmjit/core/osutils.h new file mode 100644 index 0000000..3c5c3d9 --- /dev/null +++ b/src/asmjit/core/osutils.h @@ -0,0 +1,61 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_OSUTILS_H_INCLUDED +#define ASMJIT_CORE_OSUTILS_H_INCLUDED + +#include "../core/globals.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \addtogroup asmjit_utilities +//! \{ + +//! Operating system utilities. +namespace OSUtils { + //! Gets the current CPU tick count, used for benchmarking (1ms resolution). + ASMJIT_API uint32_t getTickCount() noexcept; +}; + + +//! \cond INTERNAL +//! Lock. +//! +//! Lock is internal, it cannot be used outside of AsmJit, however, its internal +//! layout is exposed as it's used by some other classes, which are public. +class Lock { +public: + ASMJIT_NONCOPYABLE(Lock) + +#if defined(_WIN32) +#pragma pack(push, 8) + struct ASMJIT_MAY_ALIAS Handle { + void* DebugInfo; + long LockCount; + long RecursionCount; + void* OwningThread; + void* LockSemaphore; + unsigned long* SpinCount; + }; + Handle _handle; +#pragma pack(pop) +#elif !defined(__EMSCRIPTEN__) + typedef pthread_mutex_t Handle; + Handle _handle; +#endif + + ASMJIT_FORCE_INLINE Lock() noexcept; + ASMJIT_FORCE_INLINE ~Lock() noexcept; + + ASMJIT_FORCE_INLINE void lock() noexcept; + ASMJIT_FORCE_INLINE void unlock() noexcept; +}; +//! \endcond + +//! \} + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_OSUTILS_H_INCLUDED diff --git a/src/asmjit/core/osutils_p.h b/src/asmjit/core/osutils_p.h new file mode 100644 index 0000000..fd87e73 --- /dev/null +++ b/src/asmjit/core/osutils_p.h @@ -0,0 +1,68 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_OSUTILS_P_H_INCLUDED +#define ASMJIT_CORE_OSUTILS_P_H_INCLUDED + +#include "../core/osutils.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_utilities +//! \{ + +#if defined(_WIN32) + +// Windows implementation. +static_assert(sizeof(Lock::Handle) == sizeof(CRITICAL_SECTION), "asmjit::Lock::Handle layout must match CRITICAL_SECTION"); +static_assert(alignof(Lock::Handle) == alignof(CRITICAL_SECTION), "asmjit::Lock::Handle alignment must match CRITICAL_SECTION"); + +ASMJIT_FORCE_INLINE Lock::Lock() noexcept { InitializeCriticalSection(reinterpret_cast(&_handle)); } +ASMJIT_FORCE_INLINE Lock::~Lock() noexcept { DeleteCriticalSection(reinterpret_cast(&_handle)); } +ASMJIT_FORCE_INLINE void Lock::lock() noexcept { EnterCriticalSection(reinterpret_cast(&_handle)); } +ASMJIT_FORCE_INLINE void Lock::unlock() noexcept { LeaveCriticalSection(reinterpret_cast(&_handle)); } + +#elif !defined(__EMSCRIPTEN__) + +// PThread implementation. +#ifdef PTHREAD_MUTEX_INITIALIZER +ASMJIT_FORCE_INLINE Lock::Lock() noexcept : _handle(PTHREAD_MUTEX_INITIALIZER) {} +#else +ASMJIT_FORCE_INLINE Lock::Lock() noexcept { pthread_mutex_init(&_handle, nullptr); } +#endif +ASMJIT_FORCE_INLINE Lock::~Lock() noexcept { pthread_mutex_destroy(&_handle); } +ASMJIT_FORCE_INLINE void Lock::lock() noexcept { pthread_mutex_lock(&_handle); } +ASMJIT_FORCE_INLINE void Lock::unlock() noexcept { pthread_mutex_unlock(&_handle); } + +#else + +// Dummy implementation - Emscripten or other unsupported platform. +ASMJIT_FORCE_INLINE Lock::Lock() noexcept {} +ASMJIT_FORCE_INLINE Lock::~Lock() noexcept {} +ASMJIT_FORCE_INLINE void Lock::lock() noexcept {} +ASMJIT_FORCE_INLINE void Lock::unlock() noexcept {} + +#endif + +//! Scoped lock. +class LockGuard { +public: + ASMJIT_NONCOPYABLE(LockGuard) + + Lock& _target; + + inline LockGuard(Lock& target) noexcept + : _target(target) { _target.lock(); } + inline ~LockGuard() noexcept { _target.unlock(); } +}; + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // ASMJIT_CORE_OSUTILS_P_H_INCLUDED + diff --git a/src/asmjit/core/raassignment_p.h b/src/asmjit/core/raassignment_p.h new file mode 100644 index 0000000..5418329 --- /dev/null +++ b/src/asmjit/core/raassignment_p.h @@ -0,0 +1,418 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED +#define ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_COMPILER + +#include "../core/radefs_p.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_ra +//! \{ + +//! Holds the current register assignment. +//! +//! Has two purposes: +//! +//! 1. Holds register assignment of a local register allocator (see \ref RALocalAllocator). +//! 2. Holds register assignment of the entry of basic blocks (see \ref RABlock). +class RAAssignment { +public: + ASMJIT_NONCOPYABLE(RAAssignment) + + enum Ids : uint32_t { + kPhysNone = 0xFF, + kWorkNone = RAWorkReg::kIdNone + }; + + enum DirtyBit : uint32_t { + kClean = 0, + kDirty = 1 + }; + + struct Layout { + //! Index of architecture registers per group. + RARegIndex physIndex; + //! Count of architecture registers per group. + RARegCount physCount; + //! Count of physical registers of all groups. + uint32_t physTotal; + //! Count of work registers. + uint32_t workCount; + //! WorkRegs data (vector). + const RAWorkRegs* workRegs; + + inline void reset() noexcept { + physIndex.reset(); + physCount.reset(); + physTotal = 0; + workCount = 0; + workRegs = nullptr; + } + }; + + struct PhysToWorkMap { + //! Assigned registers (each bit represents one physical reg). + RARegMask assigned; + //! Dirty registers (spill slot out of sync or no spill slot). + RARegMask dirty; + //! PhysReg to WorkReg mapping. + uint32_t workIds[1 /* ... */]; + + static inline size_t sizeOf(size_t count) noexcept { + return sizeof(PhysToWorkMap) - sizeof(uint32_t) + count * sizeof(uint32_t); + } + + inline void reset(size_t count) noexcept { + assigned.reset(); + dirty.reset(); + + for (size_t i = 0; i < count; i++) + workIds[i] = kWorkNone; + } + + inline void copyFrom(const PhysToWorkMap* other, size_t count) noexcept { + size_t size = sizeOf(count); + memcpy(this, other, size); + } + + inline void unassign(RegGroup group, uint32_t physId, uint32_t indexInWorkIds) noexcept { + assigned.clear(group, Support::bitMask(physId)); + dirty.clear(group, Support::bitMask(physId)); + workIds[indexInWorkIds] = kWorkNone; + } + }; + + struct WorkToPhysMap { + //! WorkReg to PhysReg mapping + uint8_t physIds[1 /* ... */]; + + static inline size_t sizeOf(size_t count) noexcept { + return size_t(count) * sizeof(uint8_t); + } + + inline void reset(size_t count) noexcept { + for (size_t i = 0; i < count; i++) + physIds[i] = kPhysNone; + } + + inline void copyFrom(const WorkToPhysMap* other, size_t count) noexcept { + size_t size = sizeOf(count); + if (ASMJIT_LIKELY(size)) + memcpy(this, other, size); + } + }; + + //! \name Members + //! \{ + + //! Physical registers layout. + Layout _layout; + //! WorkReg to PhysReg mapping. + WorkToPhysMap* _workToPhysMap; + //! PhysReg to WorkReg mapping and assigned/dirty bits. + PhysToWorkMap* _physToWorkMap; + //! Optimization to translate PhysRegs to WorkRegs faster. + Support::Array _physToWorkIds; + + //! \} + + //! \name Construction & Destruction + //! \{ + + inline RAAssignment() noexcept { + _layout.reset(); + resetMaps(); + } + + ASMJIT_FORCE_INLINE void initLayout(const RARegCount& physCount, const RAWorkRegs& workRegs) noexcept { + // Layout must be initialized before data. + ASMJIT_ASSERT(_physToWorkMap == nullptr); + ASMJIT_ASSERT(_workToPhysMap == nullptr); + + _layout.physIndex.buildIndexes(physCount); + _layout.physCount = physCount; + _layout.physTotal = uint32_t(_layout.physIndex[RegGroup::kMaxVirt]) + + uint32_t(_layout.physCount[RegGroup::kMaxVirt]) ; + _layout.workCount = workRegs.size(); + _layout.workRegs = &workRegs; + } + + ASMJIT_FORCE_INLINE void initMaps(PhysToWorkMap* physToWorkMap, WorkToPhysMap* workToPhysMap) noexcept { + _physToWorkMap = physToWorkMap; + _workToPhysMap = workToPhysMap; + for (RegGroup group : RegGroupVirtValues{}) + _physToWorkIds[group] = physToWorkMap->workIds + _layout.physIndex.get(group); + } + + ASMJIT_FORCE_INLINE void resetMaps() noexcept { + _physToWorkMap = nullptr; + _workToPhysMap = nullptr; + _physToWorkIds.fill(nullptr); + } + + //! \} + + //! \name Accessors + //! \{ + + inline PhysToWorkMap* physToWorkMap() const noexcept { return _physToWorkMap; } + inline WorkToPhysMap* workToPhysMap() const noexcept { return _workToPhysMap; } + + inline RARegMask& assigned() noexcept { return _physToWorkMap->assigned; } + inline const RARegMask& assigned() const noexcept { return _physToWorkMap->assigned; } + inline uint32_t assigned(RegGroup group) const noexcept { return _physToWorkMap->assigned[group]; } + + inline RARegMask& dirty() noexcept { return _physToWorkMap->dirty; } + inline const RARegMask& dirty() const noexcept { return _physToWorkMap->dirty; } + inline RegMask dirty(RegGroup group) const noexcept { return _physToWorkMap->dirty[group]; } + + inline uint32_t workToPhysId(RegGroup group, uint32_t workId) const noexcept { + DebugUtils::unused(group); + ASMJIT_ASSERT(workId != kWorkNone); + ASMJIT_ASSERT(workId < _layout.workCount); + return _workToPhysMap->physIds[workId]; + } + + inline uint32_t physToWorkId(RegGroup group, uint32_t physId) const noexcept { + ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); + return _physToWorkIds[group][physId]; + } + + inline bool isPhysAssigned(RegGroup group, uint32_t physId) const noexcept { + ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); + return Support::bitTest(_physToWorkMap->assigned[group], physId); + } + + inline bool isPhysDirty(RegGroup group, uint32_t physId) const noexcept { + ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); + return Support::bitTest(_physToWorkMap->dirty[group], physId); + } + + //! \} + + //! \name Assignment + //! + //! These are low-level allocation helpers that are used to update the current mappings between physical and + //! virt/work registers and also to update masks that represent allocated and dirty registers. These functions + //! don't emit any code; they are only used to update and keep all mappings in sync. + //! + //! \{ + + //! Assign [VirtReg/WorkReg] to a physical register. + inline void assign(RegGroup group, uint32_t workId, uint32_t physId, bool dirty) noexcept { + ASMJIT_ASSERT(workToPhysId(group, workId) == kPhysNone); + ASMJIT_ASSERT(physToWorkId(group, physId) == kWorkNone); + ASMJIT_ASSERT(!isPhysAssigned(group, physId)); + ASMJIT_ASSERT(!isPhysDirty(group, physId)); + + _workToPhysMap->physIds[workId] = uint8_t(physId); + _physToWorkIds[group][physId] = workId; + + RegMask regMask = Support::bitMask(physId); + _physToWorkMap->assigned[group] |= regMask; + _physToWorkMap->dirty[group] |= regMask & Support::bitMaskFromBool(dirty); + + verify(); + } + + //! Reassign [VirtReg/WorkReg] to `dstPhysId` from `srcPhysId`. + inline void reassign(RegGroup group, uint32_t workId, uint32_t dstPhysId, uint32_t srcPhysId) noexcept { + ASMJIT_ASSERT(dstPhysId != srcPhysId); + ASMJIT_ASSERT(workToPhysId(group, workId) == srcPhysId); + ASMJIT_ASSERT(physToWorkId(group, srcPhysId) == workId); + ASMJIT_ASSERT(isPhysAssigned(group, srcPhysId) == true); + ASMJIT_ASSERT(isPhysAssigned(group, dstPhysId) == false); + + _workToPhysMap->physIds[workId] = uint8_t(dstPhysId); + _physToWorkIds[group][srcPhysId] = kWorkNone; + _physToWorkIds[group][dstPhysId] = workId; + + RegMask srcMask = Support::bitMask(srcPhysId); + RegMask dstMask = Support::bitMask(dstPhysId); + + bool dirty = (_physToWorkMap->dirty[group] & srcMask) != 0; + RegMask regMask = dstMask | srcMask; + + _physToWorkMap->assigned[group] ^= regMask; + _physToWorkMap->dirty[group] ^= regMask & Support::bitMaskFromBool(dirty); + + verify(); + } + + inline void swap(RegGroup group, uint32_t aWorkId, uint32_t aPhysId, uint32_t bWorkId, uint32_t bPhysId) noexcept { + ASMJIT_ASSERT(aPhysId != bPhysId); + ASMJIT_ASSERT(workToPhysId(group, aWorkId) == aPhysId); + ASMJIT_ASSERT(workToPhysId(group, bWorkId) == bPhysId); + ASMJIT_ASSERT(physToWorkId(group, aPhysId) == aWorkId); + ASMJIT_ASSERT(physToWorkId(group, bPhysId) == bWorkId); + ASMJIT_ASSERT(isPhysAssigned(group, aPhysId)); + ASMJIT_ASSERT(isPhysAssigned(group, bPhysId)); + + _workToPhysMap->physIds[aWorkId] = uint8_t(bPhysId); + _workToPhysMap->physIds[bWorkId] = uint8_t(aPhysId); + _physToWorkIds[group][aPhysId] = bWorkId; + _physToWorkIds[group][bPhysId] = aWorkId; + + RegMask aMask = Support::bitMask(aPhysId); + RegMask bMask = Support::bitMask(bPhysId); + RegMask flipMask = Support::bitMaskFromBool(((_physToWorkMap->dirty[group] & aMask) != 0) ^ ((_physToWorkMap->dirty[group] & bMask) != 0)); + RegMask regMask = aMask | bMask; + _physToWorkMap->dirty[group] ^= regMask & flipMask; + + verify(); + } + + //! Unassign [VirtReg/WorkReg] from a physical register. + inline void unassign(RegGroup group, uint32_t workId, uint32_t physId) noexcept { + ASMJIT_ASSERT(physId < Globals::kMaxPhysRegs); + ASMJIT_ASSERT(workToPhysId(group, workId) == physId); + ASMJIT_ASSERT(physToWorkId(group, physId) == workId); + ASMJIT_ASSERT(isPhysAssigned(group, physId)); + + _workToPhysMap->physIds[workId] = kPhysNone; + _physToWorkIds[group][physId] = kWorkNone; + + RegMask regMask = Support::bitMask(physId); + _physToWorkMap->assigned[group] &= ~regMask; + _physToWorkMap->dirty[group] &= ~regMask; + + verify(); + } + + inline void makeClean(RegGroup group, uint32_t workId, uint32_t physId) noexcept { + DebugUtils::unused(workId); + RegMask regMask = Support::bitMask(physId); + _physToWorkMap->dirty[group] &= ~regMask; + } + + inline void makeDirty(RegGroup group, uint32_t workId, uint32_t physId) noexcept { + DebugUtils::unused(workId); + RegMask regMask = Support::bitMask(physId); + _physToWorkMap->dirty[group] |= regMask; + } + + //! \} + + //! \name Utilities + //! \{ + + ASMJIT_FORCE_INLINE void swap(RAAssignment& other) noexcept { + std::swap(_workToPhysMap, other._workToPhysMap); + std::swap(_physToWorkMap, other._physToWorkMap); + _physToWorkIds.swap(other._physToWorkIds); + } + + inline void assignWorkIdsFromPhysIds() noexcept { + memset(_workToPhysMap, uint8_t(BaseReg::kIdBad), WorkToPhysMap::sizeOf(_layout.workCount)); + + for (RegGroup group : RegGroupVirtValues{}) { + uint32_t physBaseIndex = _layout.physIndex[group]; + Support::BitWordIterator it(_physToWorkMap->assigned[group]); + + while (it.hasNext()) { + uint32_t physId = it.next(); + uint32_t workId = _physToWorkMap->workIds[physBaseIndex + physId]; + + ASMJIT_ASSERT(workId != kWorkNone); + _workToPhysMap->physIds[workId] = uint8_t(physId); + } + } + } + + inline void copyFrom(const PhysToWorkMap* physToWorkMap) noexcept { + memcpy(_physToWorkMap, physToWorkMap, PhysToWorkMap::sizeOf(_layout.physTotal)); + assignWorkIdsFromPhysIds(); + } + + inline void copyFrom(const PhysToWorkMap* physToWorkMap, const WorkToPhysMap* workToPhysMap) noexcept { + memcpy(_physToWorkMap, physToWorkMap, PhysToWorkMap::sizeOf(_layout.physTotal)); + memcpy(_workToPhysMap, workToPhysMap, WorkToPhysMap::sizeOf(_layout.workCount)); + } + + inline void copyFrom(const RAAssignment& other) noexcept { + copyFrom(other.physToWorkMap(), other.workToPhysMap()); + } + + // Not really useful outside of debugging. + bool equals(const RAAssignment& other) const noexcept { + // Layout should always match. + if (_layout.physIndex != other._layout.physIndex || + _layout.physCount != other._layout.physCount || + _layout.physTotal != other._layout.physTotal || + _layout.workCount != other._layout.workCount || + _layout.workRegs != other._layout.workRegs) + return false; + + uint32_t physTotal = _layout.physTotal; + uint32_t workCount = _layout.workCount; + + for (uint32_t physId = 0; physId < physTotal; physId++) { + uint32_t thisWorkId = _physToWorkMap->workIds[physId]; + uint32_t otherWorkId = other._physToWorkMap->workIds[physId]; + if (thisWorkId != otherWorkId) + return false; + } + + for (uint32_t workId = 0; workId < workCount; workId++) { + uint32_t thisPhysId = _workToPhysMap->physIds[workId]; + uint32_t otherPhysId = other._workToPhysMap->physIds[workId]; + if (thisPhysId != otherPhysId) + return false; + } + + if (_physToWorkMap->assigned != other._physToWorkMap->assigned || + _physToWorkMap->dirty != other._physToWorkMap->dirty ) + return false; + + return true; + } + +#if defined(ASMJIT_BUILD_DEBUG) + ASMJIT_NOINLINE void verify() noexcept { + // Verify WorkToPhysMap. + { + for (uint32_t workId = 0; workId < _layout.workCount; workId++) { + uint32_t physId = _workToPhysMap->physIds[workId]; + if (physId != kPhysNone) { + const RAWorkReg* workReg = _layout.workRegs->at(workId); + RegGroup group = workReg->group(); + ASMJIT_ASSERT(_physToWorkIds[group][physId] == workId); + } + } + } + + // Verify PhysToWorkMap. + { + for (RegGroup group : RegGroupVirtValues{}) { + uint32_t physCount = _layout.physCount[group]; + for (uint32_t physId = 0; physId < physCount; physId++) { + uint32_t workId = _physToWorkIds[group][physId]; + if (workId != kWorkNone) { + ASMJIT_ASSERT(_workToPhysMap->physIds[workId] == physId); + } + } + } + } + } +#else + inline void verify() noexcept {} +#endif + + //! \} +}; + +//! \} +//! \endcond + +ASMJIT_END_NAMESPACE + +#endif // !ASMJIT_NO_COMPILER +#endif // ASMJIT_CORE_RAASSIGNMENT_P_H_INCLUDED diff --git a/src/asmjit/core/rabuilders_p.h b/src/asmjit/core/rabuilders_p.h new file mode 100644 index 0000000..9d9b428 --- /dev/null +++ b/src/asmjit/core/rabuilders_p.h @@ -0,0 +1,612 @@ +// This file is part of AsmJit project +// +// See asmjit.h or LICENSE.md for license and copyright information +// SPDX-License-Identifier: Zlib + +#ifndef ASMJIT_CORE_RABUILDERS_P_H_INCLUDED +#define ASMJIT_CORE_RABUILDERS_P_H_INCLUDED + +#include "../core/api-config.h" +#ifndef ASMJIT_NO_COMPILER + +#include "../core/formatter.h" +#include "../core/rapass_p.h" + +ASMJIT_BEGIN_NAMESPACE + +//! \cond INTERNAL +//! \addtogroup asmjit_ra +//! \{ + +template +class RACFGBuilderT { +public: + enum : uint32_t { + kRootIndentation = 2, + kCodeIndentation = 4, + + // NOTE: This is a bit hacky. There are some nodes which are processed twice (see `onBeforeInvoke()` and + // `onBeforeRet()`) as they can insert some nodes around them. Since we don't have any flags to mark these + // we just use their position that is [at that time] unassigned. + kNodePositionDidOnBefore = 0xFFFFFFFFu + }; + + //! \name Members + //! \{ + + BaseRAPass* _pass = nullptr; + BaseCompiler* _cc = nullptr; + RABlock* _curBlock = nullptr; + RABlock* _retBlock = nullptr; + FuncNode* _funcNode = nullptr; + RARegsStats _blockRegStats {}; + uint32_t _exitLabelId = Globals::kInvalidId; + ZoneVector _sharedAssignmentsMap {}; + + // Only used by logging, it's fine to be here to prevent more #ifdefs... + bool _hasCode = false; + RABlock* _lastLoggedBlock = nullptr; + +#ifndef ASMJIT_NO_LOGGING + Logger* _logger = nullptr; + FormatOptions _formatOptions {}; + StringTmp<512> _sb; +#endif + + //! \} + + inline RACFGBuilderT(BaseRAPass* pass) noexcept + : _pass(pass), + _cc(pass->cc()) { +#ifndef ASMJIT_NO_LOGGING + _logger = _pass->hasDiagnosticOption(DiagnosticOptions::kRADebugCFG) ? _pass->logger() : nullptr; + if (_logger) + _formatOptions = _logger->options(); +#endif + } + + inline BaseCompiler* cc() const noexcept { return _cc; } + + //! \name Run + //! \{ + + //! Called per function by an architecture-specific CFG builder. + Error run() noexcept { + log("[BuildCFG]\n"); + ASMJIT_PROPAGATE(prepare()); + + logNode(_funcNode, kRootIndentation); + logBlock(_curBlock, kRootIndentation); + + RABlock* entryBlock = _curBlock; + BaseNode* node = _funcNode->next(); + if (ASMJIT_UNLIKELY(!node)) + return DebugUtils::errored(kErrorInvalidState); + + _curBlock->setFirst(_funcNode); + _curBlock->setLast(_funcNode); + + RAInstBuilder ib; + ZoneVector blocksWithUnknownJumps; + + for (;;) { + BaseNode* next = node->next(); + ASMJIT_ASSERT(node->position() == 0 || node->position() == kNodePositionDidOnBefore); + + if (node->isInst()) { + // Instruction | Jump | Invoke | Return + // ------------------------------------ + + // Handle `InstNode`, `InvokeNode`, and `FuncRetNode`. All of them share the same interface that provides + // operands that have read/write semantics. + if (ASMJIT_UNLIKELY(!_curBlock)) { + // Unreachable code has to be removed, we cannot allocate registers in such code as we cannot do proper + // liveness analysis in such case. + removeNode(node); + node = next; + continue; + } + + _hasCode = true; + + if (node->isInvoke() || node->isFuncRet()) { + if (node->position() != kNodePositionDidOnBefore) { + // Call and Reg are complicated as they may insert some surrounding code around them. The simplest + // approach is to get the previous node, call the `onBefore()` handlers and then check whether + // anything changed and restart if so. By restart we mean that the current `node` would go back to + // the first possible inserted node by `onBeforeInvoke()` or `onBeforeRet()`. + BaseNode* prev = node->prev(); + + if (node->type() == NodeType::kInvoke) + ASMJIT_PROPAGATE(static_cast(this)->onBeforeInvoke(node->as())); + else + ASMJIT_PROPAGATE(static_cast(this)->onBeforeRet(node->as())); + + if (prev != node->prev()) { + // If this was the first node in the block and something was + // inserted before it then we have to update the first block. + if (_curBlock->first() == node) + _curBlock->setFirst(prev->next()); + + node->setPosition(kNodePositionDidOnBefore); + node = prev->next(); + + // `onBeforeInvoke()` and `onBeforeRet()` can only insert instructions. + ASMJIT_ASSERT(node->isInst()); + } + + // Necessary if something was inserted after `node`, but nothing before. + next = node->next(); + } + else { + // Change the position back to its original value. + node->setPosition(0); + } + } + + InstNode* inst = node->as(); + logNode(inst, kCodeIndentation); + + InstControlFlow cf = InstControlFlow::kRegular; + ib.reset(); + ASMJIT_PROPAGATE(static_cast(this)->onInst(inst, cf, ib)); + + if (node->isInvoke()) { + ASMJIT_PROPAGATE(static_cast(this)->onInvoke(inst->as(), ib)); + } + + if (node->isFuncRet()) { + ASMJIT_PROPAGATE(static_cast(this)->onRet(inst->as(), ib)); + cf = InstControlFlow::kReturn; + } + + if (cf == InstControlFlow::kJump) { + uint32_t fixedRegCount = 0; + for (RATiedReg& tiedReg : ib) { + RAWorkReg* workReg = _pass->workRegById(tiedReg.workId()); + if (workReg->group() == RegGroup::kGp) { + uint32_t useId = tiedReg.useId(); + if (useId == BaseReg::kIdBad) { + useId = _pass->_scratchRegIndexes[fixedRegCount++]; + tiedReg.setUseId(useId); + } + _curBlock->addExitScratchGpRegs(Support::bitMask(useId)); + } + } + } + + ASMJIT_PROPAGATE(_pass->assignRAInst(inst, _curBlock, ib)); + _blockRegStats.combineWith(ib._stats); + + if (cf != InstControlFlow::kRegular) { + // Support for conditional and unconditional jumps. + if (cf == InstControlFlow::kJump || cf == InstControlFlow::kBranch) { + _curBlock->setLast(node); + _curBlock->addFlags(RABlockFlags::kHasTerminator); + _curBlock->makeConstructed(_blockRegStats); + + if (!inst->hasOption(InstOptions::kUnfollow)) { + // Jmp/Jcc/Call/Loop/etc... + uint32_t opCount = inst->opCount(); + const Operand* opArray = inst->operands(); + + // Cannot jump anywhere without operands. + if (ASMJIT_UNLIKELY(!opCount)) + return DebugUtils::errored(kErrorInvalidState); + + if (opArray[opCount - 1].isLabel()) { + // Labels are easy for constructing the control flow. + LabelNode* labelNode; + ASMJIT_PROPAGATE(cc()->labelNodeOf(&labelNode, opArray[opCount - 1].as