Skip to content

Commit 33dbc8b

Browse files
committed
Intel x64 ASM: intrinsic output
Add another implementation of the assembly code for Intel x64 that uses intrinsics and C code.
1 parent bc808cb commit 33dbc8b

20 files changed

Lines changed: 415407 additions & 28 deletions

‎.wolfssl_known_macro_extras‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1231,6 +1231,7 @@ WOLFSSL_X509_STORE_CERTS
12311231
WOLFSSL_X509_TINY
12321232
WOLFSSL_X509_TRUSTED_CERTIFICATE_CALLBACK
12331233
WOLFSSL_X509_VERIFY_ONLY
1234+
WOLFSSL_X86_64_INTRIN_ALLOW_WIDE_ISA
12341235
WOLFSSL_XFREE_NO_NULLNESS_CHECK
12351236
WOLFSSL_XILINX_CRYPTO_OLD
12361237
WOLFSSL_XILINX_PATCH
@@ -1315,6 +1316,7 @@ __ATOMIC_RELAXED
13151316
__ATOMIC_SEQ_CST
13161317
__AVR_ARCH__
13171318
__AVR__
1319+
__AVX2__
13181320
__AVX512F__
13191321
__BCPLUSPLUS__
13201322
__BIG_ENDIAN__

‎CMakeLists.txt‎

Lines changed: 140 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2229,8 +2229,7 @@ else()
22292229
endif()
22302230
endif()
22312231

2232-
# TODO: - AES-XTS
2233-
# - Web server
2232+
# TODO: - Web server
22342233
# - Web client
22352234
add_option("WOLFSSL_CMAC"
22362235
"Enable CMAC (default: disabled)"
@@ -2879,10 +2878,123 @@ if("${WOLFSSL_TARGET_PROCESSOR}" MATCHES "x86_64|AMD64")
28792878
set(WOLFSSL_X86_64_BUILD ON)
28802879
add_option("WOLFSSL_X86_64_BUILD_ASM" "Build ASM files" "yes" "yes;no")
28812880
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_64_BUILD")
2881+
elseif("${WOLFSSL_TARGET_PROCESSOR}" MATCHES "^(i[3-6]86|x86)$")
2882+
# 32-bit x86. The distinction matters beyond the define: the AES-GCM
2883+
# speedup has its OWN assembly there (aes_gcm_x86_asm.S) rather than a
2884+
# 32-bit build of the x86-64 one, which is what BUILD_X86_ASM selects
2885+
# below - the same split autotools makes with ENABLED_X86_ASM.
2886+
set(WOLFSSL_X86_BUILD ON)
2887+
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_X86_BUILD")
28822888
elseif("${WOLFSSL_TARGET_PROCESSOR}" MATCHES "aarch64|arm64")
28832889
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AARCH64_BUILD")
28842890
endif()
28852891

2892+
# Intel assembly speedups. cmake/functions.cmake already selects the asm
2893+
# sources from this variable; declaring it makes it discoverable and gives it
2894+
# a default. USE_INTEL_SPEEDUP is what makes the C dispatch to those sources -
2895+
# without it they compile but nothing ever calls them.
2896+
add_option("WOLFSSL_INTEL_ASM"
2897+
"Enable Intel assembly speedups (default: disabled)"
2898+
"no" "yes;no")
2899+
2900+
if(WOLFSSL_INTEL_ASM)
2901+
list(APPEND WOLFSSL_DEFINITIONS "-DUSE_INTEL_SPEEDUP")
2902+
endif()
2903+
2904+
# AES-NI. Same shape: functions.cmake already reads WOLFSSL_AESNI to select
2905+
# the AES assembly, but WOLFSSL_AESNI has to reach the compiler as well or
2906+
# aes.c leaves every AES-NI path out and the assembly it does build is never
2907+
# called. --enable-intelasm implies it in the autotools build; match that.
2908+
add_option("WOLFSSL_AESNI"
2909+
"Enable AES-NI (default: disabled)"
2910+
"no" "yes;no")
2911+
2912+
if(WOLFSSL_AESNI OR WOLFSSL_INTEL_ASM)
2913+
# x86 only. Both the define and the -maes below are meaningless
2914+
# elsewhere, and -maes lands on CMAKE_C_FLAGS for the library AND every
2915+
# example and test target, so a cross compiler rejects the whole build.
2916+
# Fail with a clear message instead, as WOLFSSL_INTELASM_INTRINSICS and
2917+
# WOLFSSL_FALCON_ASM do below.
2918+
if(NOT WOLFSSL_X86_64_BUILD AND NOT WOLFSSL_X86_BUILD)
2919+
message(FATAL_ERROR
2920+
"WOLFSSL_AESNI / WOLFSSL_INTEL_ASM are x86 only")
2921+
endif()
2922+
set(WOLFSSL_AESNI "yes")
2923+
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AESNI")
2924+
# aes.c uses AES-NI intrinsics directly in the key schedule, so the
2925+
# instructions have to be available to the compiler, not just to the
2926+
# assembler. A flag, not a define, so it goes on CMAKE_C_FLAGS rather
2927+
# than WOLFSSL_DEFINITIONS. This is an ISA the runtime dispatch already
2928+
# requires before calling in, so it does not widen what any dispatched
2929+
# routine may use.
2930+
if(NOT CMAKE_C_COMPILER_ID STREQUAL "MSVC")
2931+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes")
2932+
endif()
2933+
endif()
2934+
2935+
# AES-XTS. The XTS assembly is wrapped in WOLFSSL_AES_XTS, so without this
2936+
# option the file is an empty translation unit.
2937+
add_option("WOLFSSL_AESXTS"
2938+
"Enable AES-XTS (default: disabled)"
2939+
"no" "yes;no")
2940+
2941+
if(WOLFSSL_AESXTS)
2942+
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AES_XTS")
2943+
# --enable-aesxts also turns the streaming API on by default for
2944+
# non-ARMASM builds (configure.ac defaults ENABLED_AESXTS_STREAM from
2945+
# ENABLED_AESXTS). Without this, cmake -DWOLFSSL_AESXTS=yes silently
2946+
# gives a smaller API than the autotools equivalent - no
2947+
# wc_AesXtsEncryptInit/Update/Final - which breaks the documented
2948+
# --enable-foo == -DWOLFSSL_FOO=yes mapping.
2949+
if(NOT WOLFSSL_ARM_ASM)
2950+
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_AESXTS_STREAM")
2951+
endif()
2952+
endif()
2953+
2954+
# Build the Intel speedups from the generated C intrinsics rather than from
2955+
# the generated assembly. Both come out of one generator, so the two forms
2956+
# compute the same thing; the C form is visible to the optimiser, to LTO and
2957+
# to the sanitisers. Coverage is per algorithm - anything without an
2958+
# intrinsics translation keeps building from assembly.
2959+
# Falcon's floating-point backend. Falcon needs binary64 arithmetic that is
2960+
# bit-exact everywhere it runs, so the fpr layer is written in SSE2 rather
2961+
# than left to the compiler; this is autotools' --enable-falcon=asm. It is
2962+
# x86-64 only, and it selects a different fpr representation from the
2963+
# native-double backends, which is why those are mutually exclusive there.
2964+
add_option("WOLFSSL_FALCON_ASM"
2965+
"Build Falcon's fpr layer from the generated x86-64 assembly (default: disabled)"
2966+
"no" "yes;no")
2967+
2968+
if (WOLFSSL_FALCON_ASM)
2969+
if (NOT WOLFSSL_FALCON)
2970+
message(FATAL_ERROR "WOLFSSL_FALCON_ASM requires WOLFSSL_FALCON")
2971+
endif()
2972+
if (NOT WOLFSSL_X86_64_BUILD)
2973+
message(FATAL_ERROR "WOLFSSL_FALCON_ASM is x86-64 only")
2974+
endif()
2975+
list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_FALCON_FPR_ASM")
2976+
set_wolfssl_definitions("WOLFSSL_FALCON_FPR_ASM" RESULT)
2977+
endif()
2978+
2979+
add_option("WOLFSSL_INTELASM_INTRINSICS"
2980+
"Build the x86-64 speedups from generated C intrinsics rather than assembly (default: disabled)"
2981+
"no" "yes;no")
2982+
2983+
if(WOLFSSL_INTELASM_INTRINSICS)
2984+
if(NOT WOLFSSL_INTEL_ASM)
2985+
message(FATAL_ERROR
2986+
"WOLFSSL_INTELASM_INTRINSICS requires WOLFSSL_INTEL_ASM")
2987+
endif()
2988+
if(NOT WOLFSSL_X86_64_BUILD)
2989+
message(FATAL_ERROR "WOLFSSL_INTELASM_INTRINSICS is x86-64 only")
2990+
endif()
2991+
# Unlike the assembly it replaces, this code is compiled, and CMake adds
2992+
# no -O of its own unless a build type asks for one. That is handled for
2993+
# these sources specifically by WOLFSSL_INTRIN_OPT further down, which
2994+
# puts -O1 on each *_intrin.c whatever the build type - so there is
2995+
# nothing to warn about here.
2996+
endif()
2997+
28862998
# SP math all
28872999
add_option("WOLFSSL_SP_MATH_ALL"
28883000
"Enable Single Precision math implementation for full algorithm suite (default: enabled)"
@@ -4526,6 +4638,32 @@ endif()
45264638

45274639
add_library(wolfssl::wolfssl ALIAS wolfssl)
45284640

4641+
# The generated x86-64 intrinsics are a literal transliteration of the assembly
4642+
# - one C statement per instruction - so their functions are enormous.
4643+
# Unoptimised, the compiler gives every intermediate its own stack slot and
4644+
# never reuses one: mlkem_encapsulate_avx512 gets a 718KB frame on gcc and
4645+
# 1.9MB on clang, enough to overflow a thread stack. At -O1 and above the slots
4646+
# are shared and the frame drops to about 2KB. CMake emits no -O at all when
4647+
# CMAKE_BUILD_TYPE is unset, which is the default, so force the level on just
4648+
# these sources - not on the rest of the library, whose flags stay the user's.
4649+
# Override with -DWOLFSSL_INTRIN_OPT=-O2 if you want a different level.
4650+
set(WOLFSSL_INTRIN_OPT "" CACHE STRING
4651+
"Optimisation flag for the generated *_intrin.c files (default -O1, /O1 on MSVC)")
4652+
if(NOT WOLFSSL_INTRIN_OPT)
4653+
if(MSVC)
4654+
set(WOLFSSL_INTRIN_OPT "/O1")
4655+
else()
4656+
set(WOLFSSL_INTRIN_OPT "-O1")
4657+
endif()
4658+
endif()
4659+
foreach(_intrin_src ${LIB_SOURCES})
4660+
if(_intrin_src MATCHES "_intrin\\.c$")
4661+
set_source_files_properties("${_intrin_src}" PROPERTIES
4662+
COMPILE_OPTIONS "${WOLFSSL_INTRIN_OPT}")
4663+
endif()
4664+
endforeach()
4665+
unset(_intrin_src)
4666+
45294667
if (NOT "$ENV{ARIA_DIR}" STREQUAL "")
45304668
message(STATUS "Found Environment variable ARIA_DIR=$ENV{ARIA_DIR}")
45314669
if(WOLFSSL_ARIA)

0 commit comments

Comments
 (0)