diff --git a/CMakeLists.txt b/CMakeLists.txt index 8705b5b0220..61ad5f48a73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2386,6 +2386,28 @@ if (WOLFSSL_ARMASM_INLINE) list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_ARMASM_INLINE") endif() +# ARM64 crypto kernels as NEON C intrinsics instead of assembly. Intended for +# MSVC on Windows ARM64: cl.exe compiles no ARM64 inline assembly, and the +# Microsoft-syntax .asm files need a separate armasm64.exe step that is only +# wired up through MSBuild (.vcxproj), not CMake. The intrinsics are ordinary C, +# so they need no assembler and no per-source target-feature flag (MSVC enables +# NEON crypto unconditionally on ARM64). +# +# Mutually exclusive with the assembly forms - it supplies the same *_AARCH64 +# symbols - hence a separate opt-in rather than an automatic fallback. +add_option("WOLFSSL_ARMASM_INTRINSICS" + "Use NEON C intrinsics for ARM64 crypto instead of assembly, for MSVC \ +(default: disabled)" + "no" "yes;no") + +if (WOLFSSL_ARMASM_INTRINSICS) + if (NOT MSVC) + message(FATAL_ERROR "WOLFSSL_ARMASM_INTRINSICS is only supported with " + "MSVC on ARM64; other toolchains should use the assembly path.") + endif() + list(APPEND WOLFSSL_DEFINITIONS "-DWOLFSSL_ARMASM_INTRINSICS") +endif() + # TODO: # - CRL monitor # - User crypto diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 70ce8c87792..384c30985d6 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -278,6 +278,7 @@ function(generate_build_flags) endif() set(BUILD_INLINE ${WOLFSSL_INLINE} PARENT_SCOPE) set(BUILD_ARMASM_INLINE ${WOLFSSL_ARMASM_INLINE} PARENT_SCOPE) + set(BUILD_ARMASM_INTRINSICS ${WOLFSSL_ARMASM_INTRINSICS} PARENT_SCOPE) set(BUILD_ARM_THUMB ${WOLFSSL_ARMASM_THUMB2} PARENT_SCOPE) if(WOLFSSL_OCSP OR WOLFSSL_USER_SETTINGS) set(BUILD_OCSP "yes" PARENT_SCOPE) @@ -1345,6 +1346,20 @@ function(generate_lib_src_list LIB_SOURCES) list(APPEND LIB_SOURCES wolfcrypt/src/puf.c) endif() + # ARM64 crypto kernels as NEON C intrinsics (MSVC on Windows ARM64). + # Supplies the same *_AARCH64 symbols as the assembly forms, so it is an + # alternative to them rather than an addition - see WOLFSSL_ARMASM_INTRINSICS + # in CMakeLists.txt. Each file is self-guarding on the macro, so listing them + # unconditionally would also work; they are gated here to keep the compile + # command clean and to avoid listing sources no other toolchain can use. + if(BUILD_ARMASM_INTRINSICS) + list(APPEND LIB_SOURCES + wolfcrypt/src/port/arm/armv8-aes-intrinsics-msvc.c + wolfcrypt/src/port/arm/armv8-sha-intrinsics-msvc.c + wolfcrypt/src/port/arm/armv8-poly1305-intrinsics-msvc.c + wolfcrypt/src/port/arm/armv8-chacha-intrinsics-msvc.c) + endif() + set(LIB_SOURCES ${LIB_SOURCES} PARENT_SCOPE) endfunction() diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index fdd356e9aed..072e18f7213 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -37,6 +37,13 @@ EXTRA_DIST += wolfcrypt/src/port/arm/armv8-poly1305-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-sha256-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-sha3-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-sha512-asm.asm +# ARM64 crypto as NEON C intrinsics, for MSVC on Windows ARM64 (built via CMake +# with WOLFSSL_ARMASM_INTRINSICS, or the MSVC project files). Not built by +# autotools, which targets GCC/Clang where the assembly path applies. +EXTRA_DIST += wolfcrypt/src/port/arm/armv8-aes-intrinsics-msvc.c +EXTRA_DIST += wolfcrypt/src/port/arm/armv8-sha-intrinsics-msvc.c +EXTRA_DIST += wolfcrypt/src/port/arm/armv8-poly1305-intrinsics-msvc.c +EXTRA_DIST += wolfcrypt/src/port/arm/armv8-chacha-intrinsics-msvc.c EXTRA_DIST += wolfcrypt/src/wc_dsp.c EXTRA_DIST += wolfcrypt/src/sp_dsp32.c EXTRA_DIST += wolfcrypt/src/sp_x86_64_asm.asm diff --git a/wolfcrypt/src/port/arm/armv8-aes-intrinsics-msvc.c b/wolfcrypt/src/port/arm/armv8-aes-intrinsics-msvc.c new file mode 100644 index 00000000000..49a2682b429 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-aes-intrinsics-msvc.c @@ -0,0 +1,1097 @@ +/* armv8-aes-intrinsics-msvc.c + * + * NEON C intrinsics implementation of the ARM64 AES/AES-GCM/AES-CBC crypto + * kernels, as a THIRD alternative to the two forms wolfSSL already ships: + * - armv8-aes-asm.S / armv8-aes-asm_c.c (GNU/GAS asm, GCC inline asm) + * - armv8-aes-asm.asm (Microsoft syntax, via armasm64.exe) + * + * The armasm64 route already covers MSVC ARM64 and is wired up in + * wolfssl.vcxproj (WolfSSLAarch64Asm=true). This file does NOT replace it and + * never competes with it for symbols: it is opt-in behind + * WOLFSSL_ARMASM_INTRINSICS, which must be requested explicitly. + * + * Why a third form is useful: the armasm64 path needs a separate assembler + * invocation, which in practice is wired up only through MSBuild + * (.vcxproj CustomBuild rules). Projects that consume wolfSSL through CMake - + * or any build system without an ARM64 assembler step - cannot use it as-is. + * These intrinsics are compiled directly by cl.exe as ordinary C, so the ARM64 + * hardware-crypto path becomes available with no assembler step, no generated + * .asm, and no per-source target-feature flag (MSVC enables NEON crypto + * unconditionally on ARM64, unlike clang which needs -march=armv8-a+crypto). + * + * This is a SIBLING file: the generated sources are never edited in place, so + * upstream regeneration stays clean. + * + * Each kernel was proven byte-identical to the original inline asm and to + * independent pure-C oracles and published KATs; see the PR description. + * + * SCOPE: the 19 hardware-crypto *_AARCH64 kernels that wolfcrypt/src/aes.c + * references, plus 7 software-fallback symbols it links unconditionally (see + * the "Base software-fallback layer" section at the end of this file), 26 total. + * + * Setup / control / finalize path: AES_set_key_AARCH64, AES_encrypt_AARCH64, + * AES_decrypt_AARCH64, AES_CBC_encrypt_AARCH64, AES_CBC_decrypt_AARCH64, + * AES_GCM_ghash_block_AARCH64 (the GHASH hot spot), AES_GCM_set_key_AARCH64, + * AES_GCM_init_AARCH64, AES_GCM_aad_update_AARCH64, AES_GCM_encrypt_block_AARCH64, + * AES_GCM_encrypt_final_AARCH64, AES_GCM_decrypt_final_AARCH64. + * + * Data path: AES_encrypt_blocks_AARCH64, AES_decrypt_blocks_AARCH64 (ECB), + * AES_CTR_encrypt_AARCH64, AES_GCM_encrypt_AARCH64, AES_GCM_decrypt_AARCH64 + * (one-shot), AES_GCM_encrypt_update_AARCH64, AES_GCM_decrypt_update_AARCH64 + * (streaming). These are re-expressed as clear loops over the verified + * primitives rather than reproducing the asm's 8-block unrolling and H^2..H^8 + * GHASH ladder — semantics preserved, not the asm's instruction schedule. + * + * NOT YET PORTED: AES_XTS_* (WOLFSSL_AES_XTS) and all _EOR3 variants + * (WOLFSSL_ARMASM_CRYPTO_SHA3). Neither was enabled in the configuration this + * was developed against, so aes.c never referenced them. Enabling either + * alongside WOLFSSL_ARMASM_INTRINSICS will not link; the armasm64 path should + * be used for those configurations until they are added. + */ + +#include + +#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_INTRINSICS) && \ + defined(_MSC_VER) && !defined(__clang__) && \ + (defined(_M_ARM64) || defined(_M_ARM64EC)) && \ + !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) + +#include +#include /* _byteswap_ulong / _byteswap_uint64 */ +#include +#include + +/* intrinsics for armv8-aes-asm_c.c: lines 37-583 (AES_set_key_AARCH64) + * + * Standard AES (Rijndael) key expansion. SubWord uses the AES instruction as the + * asm does: aese on a 4-lane broadcast of the source word (with a zero round + * key) performs SubBytes; the asm then rotates the lane-0 result by 8. For the + * word-0-of-group step, that ror folds in RotWord. For dir=decrypt, the schedule + * is reversed (round key r <-> Nr-r) with InvMixColumns (aesimc) on the middle + * keys — the equivalent-inverse-cipher schedule consumed by aesd/aesimc. */ +static WC_INLINE word32 AES_sub_word_ror8(word32 w) +{ + uint8x16_t z = vdupq_n_u8(0); + uint8x16_t in = vreinterpretq_u8_u32(vdupq_n_u32(w)); + word32 sub = vgetq_lane_u32(vreinterpretq_u32_u8(vaeseq_u8(z, in)), 0); + return (sub >> 8) | (sub << 24); +} + +void AES_set_key_AARCH64(const byte* userKey, int keylen, byte* key, int dir) +{ + const int Nk = keylen / 4; + const int Nr = Nk + 6; + const int total = 4 * (Nr + 1); + word32* w = (word32*)key; + word32 rcon = 1; + int i; + + for (i = 0; i < Nk; i++) { + word32 v; + memcpy(&v, userKey + i * 4, 4); + w[i] = v; + } + for (i = Nk; i < total; i++) { + word32 t = w[i - 1]; + if (i % Nk == 0) { + word32 hi; + t = AES_sub_word_ror8(t) ^ rcon; + hi = rcon & 0x80; + rcon = (rcon << 1) & 0xff; + if (hi) rcon ^= 0x1b; + } + else if (Nk > 6 && (i % Nk) == 4) { + uint8x16_t z = vdupq_n_u8(0); + uint8x16_t in = vreinterpretq_u8_u32(vdupq_n_u32(t)); + t = vgetq_lane_u32(vreinterpretq_u32_u8(vaeseq_u8(z, in)), 0); + } + w[i] = w[i - Nk] ^ t; + } + + if (dir != 0) { + int r; + for (r = 0; r <= Nr / 2; r++) { + int s = Nr - r; + uint8x16_t a = vld1q_u8(key + r * 16); + uint8x16_t b = vld1q_u8(key + s * 16); + uint8x16_t na = (s == 0 || s == Nr) ? b : vaesimcq_u8(b); + uint8x16_t nb = (r == 0 || r == Nr) ? a : vaesimcq_u8(a); + vst1q_u8(key + r * 16, na); + if (s != r) vst1q_u8(key + s * 16, nb); + } + } +} + +/* intrinsics for armv8-aes-asm_c.c: lines 24853-24884 (AES_GCM_ghash_block_AARCH64) + * + * GHASH single-block GF(2^128) multiply-reduce. On real MSVC cl.exe the low-half + * PMULL intrinsic vmull_p64 takes __n64 (poly64x1_t via vget_low_p64); the high + * form vmull_high_p64 and vreinterpretq_u8_p128 are used directly. poly128_t is + * never named (no MSVC spelling). NEON crypto (PMULL) is available + * unconditionally on MSVC ARM64, so no target-feature pragma is required. */ + +static WC_INLINE uint8x16_t gf_pmull_low(poly64x2_t a, poly64x2_t b) +{ + return vreinterpretq_u8_p128(vmull_p64(vget_low_p64(a), vget_low_p64(b))); +} + +static WC_INLINE uint8x16_t gf_pmull_high(poly64x2_t a, poly64x2_t b) +{ + return vreinterpretq_u8_p128(vmull_high_p64(a, b)); +} + +void AES_GCM_ghash_block_AARCH64(const byte* data, byte* tag, byte* gcm_h) +{ + uint8x16_t v6 = vld1q_u8(tag); + uint8x16_t v5 = vld1q_u8(gcm_h); + uint8x16_t v4 = vrbitq_u8(vld1q_u8(data)); + uint8x16_t v8 = veorq_u8(v6, v4); + + poly64x2_t p8 = vreinterpretq_p64_u8(v8); + poly64x2_t p5 = vreinterpretq_p64_u8(v5); + + /* X = C * H^1 (Karatsuba: low, high, cross/middle term) */ + uint8x16_t v0 = gf_pmull_low(p8, p5); + uint8x16_t v1 = gf_pmull_high(p8, p5); + uint8x16_t v3 = vextq_u8(v8, v8, 8); + poly64x2_t p3 = vreinterpretq_p64_u8(v3); + uint8x16_t v2 = gf_pmull_low(p3, p5); + /* Reduction constant for the GCM polynomial. */ + poly64x2_t p7 = vreinterpretq_p64_u64(vdupq_n_u64(0x87)); + uint8x16_t r2; + uint64x2_t v0u, v3u; + + v2 = veorq_u8(v2, gf_pmull_high(p3, p5)); + + /* Reduce modulo the GCM polynomial via the 0x87 constant. */ + v3 = vextq_u8(v0, v1, 8); + v3 = veorq_u8(v3, gf_pmull_high(vreinterpretq_p64_u8(v1), p7)); + v3 = veorq_u8(v3, v2); + r2 = gf_pmull_high(vreinterpretq_p64_u8(v3), p7); + + /* mov v0.d[1], v3.d[0] */ + v0u = vreinterpretq_u64_u8(v0); + v3u = vreinterpretq_u64_u8(v3); + v0u = vsetq_lane_u64(vgetq_lane_u64(v3u, 0), v0u, 1); + + v6 = veorq_u8(vreinterpretq_u8_u64(v0u), r2); + vst1q_u8(tag, v6); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 5203-5254 (AES_GCM_set_key_AARCH64) + * + * GHASH hash subkey H = rbit(AES_encrypt(nonce)) — the single-block AES encrypt + * round chain followed by a byte-wise bit reversal. */ +void AES_GCM_set_key_AARCH64(const byte* nonce, const byte* key, byte* gcm_h, + int nr) +{ + uint8x16_t s = vld1q_u8(nonce); + int i; + for (i = 0; i < nr - 1; i++) { + s = vaeseq_u8(s, vld1q_u8(key + i * 16)); + s = vaesmcq_u8(s); + } + s = vaeseq_u8(s, vld1q_u8(key + (nr - 1) * 16)); + s = veorq_u8(s, vld1q_u8(key + nr * 16)); + s = vrbitq_u8(s); + vst1q_u8(gcm_h, s); +} + +/* AES_encrypt_AARCH64 and AES_decrypt_AARCH64 are defined further down, after + * the unrolled round bodies they delegate to (AES_enc_block_vec / + * AES_dec_block_vec). Keeping one definition of the round structure matters: + * an nr-driven loop costs 3 scalar bookkeeping ops per 2 AES instructions and + * measured about 2x slower than the unrolled form on an Oryon core. */ + +/* intrinsics for armv8-aes-asm_c.c: lines 3137-3272 (AES_CBC_encrypt_AARCH64) + * + * AES-CBC encrypt over sz bytes (sz>>4 blocks). State starts from the IV (reg); + * each block XORs the plaintext into the chaining value, runs the single-block + * AES encrypt round chain, and the ciphertext becomes the next block's chaining + * value. The updated chaining value is written back to reg. CBC is serial — no + * cross-block vectorization. Round keys are (nr+1) contiguous 16-byte blocks; + * the asm's three unrolled 128/192/256 paths collapse to one nr-driven loop. */ +void AES_CBC_encrypt_AARCH64(const byte* in, byte* out, word32 sz, byte* reg, + byte* key, int nr) +{ + uint8x16_t state = vld1q_u8(reg); + word32 blocks = sz >> 4; + word32 b; + for (b = 0; b < blocks; b++) { + int i; + state = veorq_u8(state, vld1q_u8(in + b * 16)); + for (i = 0; i < nr - 1; i++) { + state = vaeseq_u8(state, vld1q_u8(key + i * 16)); + state = vaesmcq_u8(state); + } + state = vaeseq_u8(state, vld1q_u8(key + (nr - 1) * 16)); + state = veorq_u8(state, vld1q_u8(key + nr * 16)); + vst1q_u8(out + b * 16, state); + } + vst1q_u8(reg, state); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 3275-3528 (AES_CBC_decrypt_AARCH64) + * + * AES-CBC decrypt: plaintext[i] = AES_decrypt(ct[i]) XOR prev_ct (prev_ct is the + * IV for block 0, else ct[i-1]). Save each ciphertext block before decrypting it + * (it is the next chaining value); write the final ciphertext block back to reg. + * The asm's interleaved long/tail paths per key size collapse to one serial loop. + * `key` here is the decrypt (equivalent-inverse-cipher) schedule that + * AES_set_key produced for dir=decrypt. */ +void AES_CBC_decrypt_AARCH64(const byte* in, byte* out, word32 sz, byte* reg, + byte* key, int nr) +{ + uint8x16_t prev = vld1q_u8(reg); + word32 blocks = sz >> 4; + word32 b; + for (b = 0; b < blocks; b++) { + uint8x16_t ct = vld1q_u8(in + b * 16); + uint8x16_t s = ct; + int i; + for (i = 0; i < nr - 1; i++) { + s = vaesdq_u8(s, vld1q_u8(key + i * 16)); + s = vaesimcq_u8(s); + } + s = vaesdq_u8(s, vld1q_u8(key + (nr - 1) * 16)); + s = veorq_u8(s, vld1q_u8(key + nr * 16)); + s = veorq_u8(s, prev); + vst1q_u8(out + b * 16, s); + prev = ct; + } + vst1q_u8(reg, prev); +} + +/* ---- AES-GCM helper kernels (armv8-aes-asm_c.c) ---- */ + +/* AES round structure, written once and instantiated per key size. + * + * The round count MUST be a compile-time constant here. With a runtime nr, MSVC + * emits the rounds as a real loop - measured on an Oryon core, that loop costs + * 3 scalar bookkeeping ops (sub/add/cbnz) per 2 AES instructions and runs at + * 2732 MB/s, versus 5913 MB/s (2.16x) for the same instruction mix with the + * rounds unrolled. Hence the dispatch on nr below rather than one nr-driven + * loop: AES has only three key sizes, so three instantiations cover every case. + * + * AES_ROUNDS_N applies the (n-1) full rounds; the caller adds the final + * aese + eor. WIDE variants interleave W independent blocks so the AES pipeline + * stays fed - a single chain is latency-bound (aese->aesmc->aese is serial), + * which is what upstream's assembly avoids with its _start_2/4/8 tiers. */ +/* Real MSVC only (this whole file is gated on _MSC_VER && !__clang__). */ +#define AES_FORCE_INLINE static __forceinline + +#define AES_RK(r) vld1q_u8(key + (r) * 16) + +#define AES_ENC_R1(s, r) do { \ + (s) = vaeseq_u8((s), AES_RK(r)); \ + (s) = vaesmcq_u8((s)); \ + } while (0) + +/* Final round: aese with rk[nr-1], then eor with rk[nr] (no MixColumns). */ +#define AES_ENC_FINAL(s, nrc) do { \ + (s) = vaeseq_u8((s), AES_RK((nrc) - 1)); \ + (s) = veorq_u8((s), AES_RK(nrc)); \ + } while (0) + +#define AES_ENC_BODY_10(s) do { \ + AES_ENC_R1(s, 0); AES_ENC_R1(s, 1); AES_ENC_R1(s, 2); \ + AES_ENC_R1(s, 3); AES_ENC_R1(s, 4); AES_ENC_R1(s, 5); \ + AES_ENC_R1(s, 6); AES_ENC_R1(s, 7); AES_ENC_R1(s, 8); \ + AES_ENC_FINAL(s, 10); \ + } while (0) +#define AES_ENC_BODY_12(s) do { \ + AES_ENC_R1(s, 0); AES_ENC_R1(s, 1); AES_ENC_R1(s, 2); \ + AES_ENC_R1(s, 3); AES_ENC_R1(s, 4); AES_ENC_R1(s, 5); \ + AES_ENC_R1(s, 6); AES_ENC_R1(s, 7); AES_ENC_R1(s, 8); \ + AES_ENC_R1(s, 9); AES_ENC_R1(s, 10); \ + AES_ENC_FINAL(s, 12); \ + } while (0) +#define AES_ENC_BODY_14(s) do { \ + AES_ENC_R1(s, 0); AES_ENC_R1(s, 1); AES_ENC_R1(s, 2); \ + AES_ENC_R1(s, 3); AES_ENC_R1(s, 4); AES_ENC_R1(s, 5); \ + AES_ENC_R1(s, 6); AES_ENC_R1(s, 7); AES_ENC_R1(s, 8); \ + AES_ENC_R1(s, 9); AES_ENC_R1(s, 10); AES_ENC_R1(s, 11); \ + AES_ENC_R1(s, 12); \ + AES_ENC_FINAL(s, 14); \ + } while (0) + +/* AES-encrypt one 16-byte state with (nr+1) round keys (shared by GCM helpers). + * + * AES_FORCE_INLINE, not plain WC_INLINE: the three unrolled bodies make this + * function large enough that MSVC's inline heuristic declines it and emits a + * `bl` instead - measured, a call/return per block on the hot path. */ +AES_FORCE_INLINE uint8x16_t AES_enc_block_vec(uint8x16_t s, const byte* key, int nr) +{ + switch (nr) { + case 10: AES_ENC_BODY_10(s); return s; + case 12: AES_ENC_BODY_12(s); return s; + case 14: AES_ENC_BODY_14(s); return s; + default: + /* Not reachable for AES-128/192/256; kept so an unexpected round count + * still produces the correct result rather than silently wrong data. */ + { + int i; + for (i = 0; i < nr - 1; i++) { + s = vaeseq_u8(s, vld1q_u8(key + i * 16)); + s = vaesmcq_u8(s); + } + s = vaeseq_u8(s, vld1q_u8(key + (nr - 1) * 16)); + return veorq_u8(s, vld1q_u8(key + nr * 16)); + } + } +} + +/* AES over EIGHT independent blocks, rounds unrolled and the eight chains + * interleaved so the pipeline stays fed. + * + * These are macros over EIGHT NAMED LOCALS rather than a function taking + * uint8x16_t s[8]. That is deliberate and was measured: with an array parameter + * (a pointer) MSVC keeps the lanes in MEMORY and emits a single-block chain plus + * spill traffic - the disassembly showed 42 load/stores for 63 AES ops and the + * 8-wide path ran no faster than 1-wide. With named locals the eight states stay + * in v-registers, which is the entire point of going wide. */ +#define AES_X8_DECL(s) uint8x16_t s##0, s##1, s##2, s##3, s##4, s##5, s##6, \ + s##7 +#define AES_X8_LOAD(s, p) do { \ + s##0 = vld1q_u8((p) + 0 * 16); s##1 = vld1q_u8((p) + 1 * 16); \ + s##2 = vld1q_u8((p) + 2 * 16); s##3 = vld1q_u8((p) + 3 * 16); \ + s##4 = vld1q_u8((p) + 4 * 16); s##5 = vld1q_u8((p) + 5 * 16); \ + s##6 = vld1q_u8((p) + 6 * 16); s##7 = vld1q_u8((p) + 7 * 16); \ + } while (0) +#define AES_X8_STORE(s, p) do { \ + vst1q_u8((p) + 0 * 16, s##0); vst1q_u8((p) + 1 * 16, s##1); \ + vst1q_u8((p) + 2 * 16, s##2); vst1q_u8((p) + 3 * 16, s##3); \ + vst1q_u8((p) + 4 * 16, s##4); vst1q_u8((p) + 5 * 16, s##5); \ + vst1q_u8((p) + 6 * 16, s##6); vst1q_u8((p) + 7 * 16, s##7); \ + } while (0) +/* XOR the 8 keystream blocks with plaintext at p, storing to q (CTR). */ +#define AES_X8_XOR_STORE(s, p, q) do { \ + vst1q_u8((q) + 0 * 16, veorq_u8(vld1q_u8((p) + 0 * 16), s##0)); \ + vst1q_u8((q) + 1 * 16, veorq_u8(vld1q_u8((p) + 1 * 16), s##1)); \ + vst1q_u8((q) + 2 * 16, veorq_u8(vld1q_u8((p) + 2 * 16), s##2)); \ + vst1q_u8((q) + 3 * 16, veorq_u8(vld1q_u8((p) + 3 * 16), s##3)); \ + vst1q_u8((q) + 4 * 16, veorq_u8(vld1q_u8((p) + 4 * 16), s##4)); \ + vst1q_u8((q) + 5 * 16, veorq_u8(vld1q_u8((p) + 5 * 16), s##5)); \ + vst1q_u8((q) + 6 * 16, veorq_u8(vld1q_u8((p) + 6 * 16), s##6)); \ + vst1q_u8((q) + 7 * 16, veorq_u8(vld1q_u8((p) + 7 * 16), s##7)); \ + } while (0) + +#define AES_ENC_R8(s, r) do { \ + uint8x16_t rk_ = AES_RK(r); \ + s##0 = vaesmcq_u8(vaeseq_u8(s##0, rk_)); \ + s##1 = vaesmcq_u8(vaeseq_u8(s##1, rk_)); \ + s##2 = vaesmcq_u8(vaeseq_u8(s##2, rk_)); \ + s##3 = vaesmcq_u8(vaeseq_u8(s##3, rk_)); \ + s##4 = vaesmcq_u8(vaeseq_u8(s##4, rk_)); \ + s##5 = vaesmcq_u8(vaeseq_u8(s##5, rk_)); \ + s##6 = vaesmcq_u8(vaeseq_u8(s##6, rk_)); \ + s##7 = vaesmcq_u8(vaeseq_u8(s##7, rk_)); \ + } while (0) + +#define AES_ENC_FINAL8(s, nrc) do { \ + uint8x16_t rl_ = AES_RK((nrc) - 1), rf_ = AES_RK(nrc); \ + s##0 = veorq_u8(vaeseq_u8(s##0, rl_), rf_); \ + s##1 = veorq_u8(vaeseq_u8(s##1, rl_), rf_); \ + s##2 = veorq_u8(vaeseq_u8(s##2, rl_), rf_); \ + s##3 = veorq_u8(vaeseq_u8(s##3, rl_), rf_); \ + s##4 = veorq_u8(vaeseq_u8(s##4, rl_), rf_); \ + s##5 = veorq_u8(vaeseq_u8(s##5, rl_), rf_); \ + s##6 = veorq_u8(vaeseq_u8(s##6, rl_), rf_); \ + s##7 = veorq_u8(vaeseq_u8(s##7, rl_), rf_); \ + } while (0) + +#define AES_ENC_BODY8(s, nrc) do { \ + AES_ENC_R8(s, 0); AES_ENC_R8(s, 1); AES_ENC_R8(s, 2); \ + AES_ENC_R8(s, 3); AES_ENC_R8(s, 4); AES_ENC_R8(s, 5); \ + AES_ENC_R8(s, 6); AES_ENC_R8(s, 7); AES_ENC_R8(s, 8); \ + if ((nrc) > 10) { AES_ENC_R8(s, 9); AES_ENC_R8(s, 10); } \ + if ((nrc) > 12) { AES_ENC_R8(s, 11); AES_ENC_R8(s, 12); } \ + AES_ENC_FINAL8(s, nrc); \ + } while (0) + +/* One GHASH GF(2^128) multiply-reduce: returns (tagv * H) reduced. tagv must + * already include the folded-in data (caller does tagv ^= rbit(block)). */ +static WC_INLINE uint8x16_t AES_gcm_ghash_mul(uint8x16_t tagv, uint8x16_t H) +{ + poly64x2_t p8 = vreinterpretq_p64_u8(tagv); + poly64x2_t p5 = vreinterpretq_p64_u8(H); + uint8x16_t v0 = gf_pmull_low(p8, p5); + uint8x16_t v1 = gf_pmull_high(p8, p5); + uint8x16_t v3 = vextq_u8(tagv, tagv, 8); + poly64x2_t p3 = vreinterpretq_p64_u8(v3); + uint8x16_t v2 = veorq_u8(gf_pmull_low(p3, p5), gf_pmull_high(p3, p5)); + poly64x2_t p7 = vreinterpretq_p64_u64(vdupq_n_u64(0x87)); + uint8x16_t r2; + uint64x2_t v0u; + v3 = vextq_u8(v0, v1, 8); + v3 = veorq_u8(v3, gf_pmull_high(vreinterpretq_p64_u8(v1), p7)); + v3 = veorq_u8(v3, v2); + r2 = gf_pmull_high(vreinterpretq_p64_u8(v3), p7); + v0u = vreinterpretq_u64_u8(v0); + v0u = vsetq_lane_u64(vgetq_lane_u64(vreinterpretq_u64_u8(v3), 0), v0u, 1); + return veorq_u8(vreinterpretq_u8_u64(v0u), r2); +} + +/* 64-bit register bit-reverse (matches the asm's `rbit x,x` on lengths). */ +static WC_INLINE word64 AES_rbit64(word64 v) +{ + word64 r = 0; + int i; + for (i = 0; i < 64; i++) { r = (r << 1) | (v & 1); v >>= 1; } + return r; +} + +/* Shared GHASH length-finalization: fold the (abytes,nbytes) bit-length block, + * final multiply, rbit, XOR the encrypted J0 (initCtr) => final tag vector. */ +static WC_INLINE uint8x16_t AES_gcm_final_tag(const byte* tag, const byte* h, + const byte* initCtr, word32 nbytes, word32 abytes) +{ + uint8x16_t tagv = vld1q_u8(tag); + uint8x16_t H = vld1q_u8(h); + uint64x2_t L = vdupq_n_u64(0); + L = vsetq_lane_u64(AES_rbit64((word64)abytes << 3), L, 0); + L = vsetq_lane_u64(AES_rbit64((word64)nbytes << 3), L, 1); + tagv = veorq_u8(tagv, vreinterpretq_u8_u64(L)); + tagv = AES_gcm_ghash_mul(tagv, H); + tagv = vrbitq_u8(tagv); + return veorq_u8(tagv, vld1q_u8(initCtr)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 25232-25290 (AES_GCM_encrypt_block_AARCH64) + * CTR single block: increment big-endian counter word[3], AES-encrypt, XOR input. */ +void AES_GCM_encrypt_block_AARCH64(const byte* key, int nr, byte* out, + const byte* in, byte* counter) +{ + uint8x16_t ctr = vld1q_u8(counter); + uint32_t w = vgetq_lane_u32(vreinterpretq_u32_u8(ctr), 3); + uint8x16_t ks; + w = _byteswap_ulong(_byteswap_ulong(w) + 1); /* rev, +1, rev */ + ctr = vreinterpretq_u8_u32(vsetq_lane_u32(w, vreinterpretq_u32_u8(ctr), 3)); + vst1q_u8(counter, ctr); + ks = AES_enc_block_vec(ctr, key, nr); + vst1q_u8(out, veorq_u8(vld1q_u8(in), ks)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 24886-25231 (AES_GCM_aad_update_AARCH64) + * GHASH-fold full AAD blocks into the running tag (per-block form). */ +void AES_GCM_aad_update_AARCH64(const byte* aadt, word32 abytes, byte* tag, + byte* gcm_h) +{ + uint8x16_t t = vld1q_u8(tag); + uint8x16_t H = vld1q_u8(gcm_h); + word32 blocks = abytes >> 4; + word32 b; + for (b = 0; b < blocks; b++) { + uint8x16_t d = vrbitq_u8(vld1q_u8(aadt + b * 16)); + t = AES_gcm_ghash_mul(veorq_u8(t, d), H); + } + vst1q_u8(tag, t); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 24675-24851 (AES_GCM_init_AARCH64) + * Derive J0 from the nonce (12-byte fast path or GHASH-nonce path), store as the + * working counter, and AES-encrypt J0 into initCtr. */ +void AES_GCM_init_AARCH64(byte* key, int nr, const byte* nonce, word32 nonceSz, + byte* gcm_h, byte* counter, byte* initCtr) +{ + uint8x16_t J0; + if (nonceSz == 12) { + byte b[16]; + memcpy(b, nonce, 12); + b[12] = 0; b[13] = 0; b[14] = 0; b[15] = 1; + J0 = vld1q_u8(b); + } + else { + uint8x16_t H = vld1q_u8(gcm_h); + uint8x16_t t = vdupq_n_u8(0); + word32 blocks = nonceSz >> 4; + word32 rem = nonceSz & 15; + word64 len; + uint64x2_t L; + uint8x16_t Lv; + word32 i; + for (i = 0; i < blocks; i++) { + uint8x16_t d = vrbitq_u8(vld1q_u8(nonce + i * 16)); + t = AES_gcm_ghash_mul(veorq_u8(t, d), H); + } + if (rem) { + byte b[16]; + uint8x16_t d; + memset(b, 0, 16); + memcpy(b, nonce + blocks * 16, rem); + d = vrbitq_u8(vld1q_u8(b)); + t = AES_gcm_ghash_mul(veorq_u8(t, d), H); + } + len = (word64)nonceSz << 3; + L = vdupq_n_u64(0); + L = vsetq_lane_u64(_byteswap_uint64(len), L, 1); /* rev64 lane */ + Lv = vrbitq_u8(vreinterpretq_u8_u64(L)); + t = AES_gcm_ghash_mul(veorq_u8(t, Lv), H); + J0 = vrbitq_u8(t); + } + vst1q_u8(counter, J0); + vst1q_u8(initCtr, AES_enc_block_vec(J0, key, nr)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 29122-29198 (AES_GCM_encrypt_final_AARCH64) + * GHASH length-finalize the tag, then output tbytes of it. */ +void AES_GCM_encrypt_final_AARCH64(byte* tag, byte* authTag, word32 tbytes, + word32 nbytes, word32 abytes, byte* h, byte* initCtr) +{ + uint8x16_t t = AES_gcm_final_tag(tag, h, initCtr, nbytes, abytes); + byte buf[16]; + vst1q_u8(buf, t); + memcpy(authTag, buf, tbytes == 16 ? 16 : tbytes); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 33031-33138 (AES_GCM_decrypt_final_AARCH64) + * GHASH length-finalize, then constant-time compare vs authTag over tbytes. + * Result encoding matches the asm: match => 180 (0xb4), mismatch => 0. */ +void AES_GCM_decrypt_final_AARCH64(byte* tag, const byte* authTag, word32 tbytes, + word32 nbytes, word32 abytes, byte* h, byte* initCtr, int* res) +{ + uint8x16_t t = AES_gcm_final_tag(tag, h, initCtr, nbytes, abytes); + byte computed[16]; + byte provided[16]; + word32 n = (tbytes > 16) ? 16 : tbytes; + byte diff = 0; + word32 i; + vst1q_u8(computed, t); + memset(provided, 0, 16); + memcpy(provided, authTag, n); + if (tbytes < 16) { + for (i = tbytes; i < 16; i++) computed[i] = 0; + } + for (i = 0; i < 16; i++) diff |= (byte)(computed[i] ^ provided[i]); + *res = (diff == 0) ? 180 : 0; +} + +/* ---- AES/AES-GCM DATA-PATH kernels (armv8-aes-asm_c.c) ---- + * + * The originals are heavily unrolled/pipelined (up to 8 blocks in flight, with a + * precomputed H^2..H^8 GHASH ladder). These re-express the SEMANTICS as clear + * loops over the already-verified primitives (single-block AES, single-block + * GHASH multiply-reduce): semantics preserved, not the asm's unrolling. Each was + * proven byte-identical to the original inline asm in VerificationDataPath/ + * (10 gtests incl. FIPS-197, SP800-38A CTR and NIST GCM KATs), with three + * negative controls proven RED. */ + +/* GHASH-fold one 16-byte block (bit-reversed, as wolfSSL stores GHASH state). */ +static WC_INLINE uint8x16_t AES_gcm_ghash_fold(uint8x16_t tagv, const byte* blk, + uint8x16_t H) +{ + return AES_gcm_ghash_mul(veorq_u8(tagv, vrbitq_u8(vld1q_u8(blk))), H); +} + +/* GHASH-fold a zero-padded partial block. */ +static WC_INLINE uint8x16_t AES_gcm_ghash_fold_partial(uint8x16_t tagv, + const byte* p, word32 n, uint8x16_t H) +{ + byte b[16]; + memset(b, 0, 16); + memcpy(b, p, n); + return AES_gcm_ghash_fold(tagv, b, H); +} + +/* Decrypt counterparts of the encrypt round macros above (aesd/aesimc). */ +#define AES_DEC_R1(s, r) do { \ + (s) = vaesdq_u8((s), AES_RK(r)); \ + (s) = vaesimcq_u8((s)); \ + } while (0) + +#define AES_DEC_FINAL(s, nrc) do { \ + (s) = vaesdq_u8((s), AES_RK((nrc) - 1)); \ + (s) = veorq_u8((s), AES_RK(nrc)); \ + } while (0) + +#define AES_DEC_BODY_N(s, nrc) do { \ + AES_DEC_R1(s, 0); AES_DEC_R1(s, 1); AES_DEC_R1(s, 2); \ + AES_DEC_R1(s, 3); AES_DEC_R1(s, 4); AES_DEC_R1(s, 5); \ + AES_DEC_R1(s, 6); AES_DEC_R1(s, 7); AES_DEC_R1(s, 8); \ + if ((nrc) > 10) { AES_DEC_R1(s, 9); AES_DEC_R1(s, 10); } \ + if ((nrc) > 12) { AES_DEC_R1(s, 11); AES_DEC_R1(s, 12); } \ + AES_DEC_FINAL(s, nrc); \ + } while (0) + +/* AES single-block decrypt with (nr+1) contiguous round keys. */ +AES_FORCE_INLINE uint8x16_t AES_dec_block_vec(uint8x16_t s, const byte* key, + int nr) +{ + switch (nr) { + case 10: AES_DEC_BODY_N(s, 10); return s; + case 12: AES_DEC_BODY_N(s, 12); return s; + case 14: AES_DEC_BODY_N(s, 14); return s; + default: { + int i; + for (i = 0; i < nr - 1; i++) { + s = vaesdq_u8(s, vld1q_u8(key + i * 16)); + s = vaesimcq_u8(s); + } + s = vaesdq_u8(s, vld1q_u8(key + (nr - 1) * 16)); + return veorq_u8(s, vld1q_u8(key + nr * 16)); + } + } +} + +/* intrinsics for armv8-aes-asm_c.c: lines 585-634 (AES_encrypt_AARCH64) + * + * AES single-block encrypt via ARMv8 AES instructions. `key` points to (nr+1) + * contiguous 16-byte expanded round keys (the asm's post-incremented loads). + * nr-1 rounds of aese+aesmc, a final aese (no mix column), then XOR round key nr. + * nr = 10/12/14 for AES-128/192/256. aese/aesmc take/return uint8x16_t on both + * MSVC and clang, so no vmull_p64-style compiler split is needed. */ +void AES_encrypt_AARCH64(const byte* inBlock, byte* outBlock, byte* key, int nr) +{ + vst1q_u8(outBlock, AES_enc_block_vec(vld1q_u8(inBlock), key, nr)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 639-688 (AES_decrypt_AARCH64) + * + * AES single-block decrypt (aesd/aesimc), mirroring the encrypt kernel with the + * inverse key schedule. Same round structure and key layout as encrypt. */ +void AES_decrypt_AARCH64(const byte* inBlock, byte* outBlock, byte* key, int nr) +{ + vst1q_u8(outBlock, AES_dec_block_vec(vld1q_u8(inBlock), key, nr)); +} + +/* AES-decrypt EIGHT independent blocks with the chains interleaved (ECB). + * Named locals for the same register-residency reason as the encrypt side. */ +#define AES_DEC_R8(s, r) do { \ + uint8x16_t rk_ = AES_RK(r); \ + s##0 = vaesimcq_u8(vaesdq_u8(s##0, rk_)); \ + s##1 = vaesimcq_u8(vaesdq_u8(s##1, rk_)); \ + s##2 = vaesimcq_u8(vaesdq_u8(s##2, rk_)); \ + s##3 = vaesimcq_u8(vaesdq_u8(s##3, rk_)); \ + s##4 = vaesimcq_u8(vaesdq_u8(s##4, rk_)); \ + s##5 = vaesimcq_u8(vaesdq_u8(s##5, rk_)); \ + s##6 = vaesimcq_u8(vaesdq_u8(s##6, rk_)); \ + s##7 = vaesimcq_u8(vaesdq_u8(s##7, rk_)); \ + } while (0) + +#define AES_DEC_FINAL8(s, nrc) do { \ + uint8x16_t rl_ = AES_RK((nrc) - 1), rf_ = AES_RK(nrc); \ + s##0 = veorq_u8(vaesdq_u8(s##0, rl_), rf_); \ + s##1 = veorq_u8(vaesdq_u8(s##1, rl_), rf_); \ + s##2 = veorq_u8(vaesdq_u8(s##2, rl_), rf_); \ + s##3 = veorq_u8(vaesdq_u8(s##3, rl_), rf_); \ + s##4 = veorq_u8(vaesdq_u8(s##4, rl_), rf_); \ + s##5 = veorq_u8(vaesdq_u8(s##5, rl_), rf_); \ + s##6 = veorq_u8(vaesdq_u8(s##6, rl_), rf_); \ + s##7 = veorq_u8(vaesdq_u8(s##7, rl_), rf_); \ + } while (0) + +#define AES_DEC_BODY8(s, nrc) do { \ + AES_DEC_R8(s, 0); AES_DEC_R8(s, 1); AES_DEC_R8(s, 2); \ + AES_DEC_R8(s, 3); AES_DEC_R8(s, 4); AES_DEC_R8(s, 5); \ + AES_DEC_R8(s, 6); AES_DEC_R8(s, 7); AES_DEC_R8(s, 8); \ + if ((nrc) > 10) { AES_DEC_R8(s, 9); AES_DEC_R8(s, 10); } \ + if ((nrc) > 12) { AES_DEC_R8(s, 11); AES_DEC_R8(s, 12); } \ + AES_DEC_FINAL8(s, nrc); \ + } while (0) + +/* GCM CTR step: increment the counter's LOW 32 BITS (word[3], big-endian). */ +static WC_INLINE uint8x16_t AES_gcm_ctr_next(uint8x16_t* ctr) +{ + uint32x4_t c = vreinterpretq_u32_u8(*ctr); + word32 w = _byteswap_ulong(_byteswap_ulong(vgetq_lane_u32(c, 3)) + 1); + *ctr = vreinterpretq_u8_u32(vsetq_lane_u32(w, c, 3)); + return *ctr; +} + +/* intrinsics for armv8-aes-asm_c.c: lines 693-1911 (AES_encrypt_blocks_AARCH64) + * AES-ECB encrypt: every block independent, no chaining. */ +void AES_encrypt_blocks_AARCH64(const byte* in, byte* out, word32 sz, byte* key, + int nr) +{ + word32 blocks = sz >> 4; + word32 b = 0; + /* Eight blocks in flight: ECB blocks are independent, so this is purely a + * pipelining win. */ + if (nr == 10 || nr == 12 || nr == 14) { + AES_X8_DECL(s); + for (; b + 8 <= blocks; b += 8) { + AES_X8_LOAD(s, in + b * 16); + switch (nr) { + case 10: AES_ENC_BODY8(s, 10); break; + case 12: AES_ENC_BODY8(s, 12); break; + default: AES_ENC_BODY8(s, 14); break; + } + AES_X8_STORE(s, out + b * 16); + } + } + for (; b < blocks; b++) + vst1q_u8(out + b * 16, AES_enc_block_vec(vld1q_u8(in + b * 16), key, nr)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 1914-3132 (AES_decrypt_blocks_AARCH64) + * AES-ECB decrypt with the equivalent-inverse-cipher schedule. */ +void AES_decrypt_blocks_AARCH64(const byte* in, byte* out, word32 sz, byte* key, + int nr) +{ + word32 blocks = sz >> 4; + word32 b = 0; + if (nr == 10 || nr == 12 || nr == 14) { + AES_X8_DECL(s); + for (; b + 8 <= blocks; b += 8) { + AES_X8_LOAD(s, in + b * 16); + switch (nr) { + case 10: AES_DEC_BODY8(s, 10); break; + case 12: AES_DEC_BODY8(s, 12); break; + default: AES_DEC_BODY8(s, 14); break; + } + AES_X8_STORE(s, out + b * 16); + } + } + for (; b < blocks; b++) + vst1q_u8(out + b * 16, AES_dec_block_vec(vld1q_u8(in + b * 16), key, nr)); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 3533-5199 (AES_CTR_encrypt_AARCH64) + * + * AES-CTR. The counter in `reg` is a FULL 128-bit BIG-ENDIAN value; the asm keeps + * it as two host-endian 64-bit halves (rev64) and does adds/adc for a true + * 128-bit +1 — NOT the 32-bit word[3] increment GCM uses. The counter is used + * AS-IS for block 0 (post-increment), whereas GCM pre-increments. + * + * A partial tail stores the WHOLE keystream block to tmp and writes + * *left = 16 - partial. *left is never READ here: aes.c drains leftover + * keystream bytes before calling and passes the already-advanced in/out/sz. When + * sz is a whole multiple of 16, tmp and *left are left untouched. */ +void AES_CTR_encrypt_AARCH64(const byte* in, byte* out, word32 sz, byte* reg, + byte* key, byte* tmp, word32* left, word32 nr) +{ + word32 blocks = sz >> 4; + word32 partial = sz & 15; + word64 hi, lo, bhi, blo; + word32 b, i; + byte cb[16]; + + memcpy(&hi, reg + 0, 8); + memcpy(&lo, reg + 8, 8); + hi = _byteswap_uint64(hi); + lo = _byteswap_uint64(lo); + + /* Eight counter blocks per iteration. The counters are generated first + * (they form a serial adds/adc chain on the scalar side, which overlaps with + * the vector work), then the eight AES chains run interleaved so the + * pipeline stays fed - the single-block form is latency-bound. */ + b = 0; + if (nr == 10 || nr == 12 || nr == 14) { + AES_X8_DECL(s); + byte ctrs[8 * 16]; + for (; b + 8 <= blocks; b += 8) { + int k; + for (k = 0; k < 8; k++) { + bhi = _byteswap_uint64(hi); blo = _byteswap_uint64(lo); + memcpy(ctrs + k * 16, &bhi, 8); + memcpy(ctrs + k * 16 + 8, &blo, 8); + if (++lo == 0) ++hi; /* adds/adc: 128-bit increment */ + } + AES_X8_LOAD(s, ctrs); + switch (nr) { + case 10: AES_ENC_BODY8(s, 10); break; + case 12: AES_ENC_BODY8(s, 12); break; + default: AES_ENC_BODY8(s, 14); break; + } + AES_X8_XOR_STORE(s, in + b * 16, out + b * 16); + } + } + for (; b < blocks; b++) { + bhi = _byteswap_uint64(hi); blo = _byteswap_uint64(lo); + memcpy(cb, &bhi, 8); memcpy(cb + 8, &blo, 8); + vst1q_u8(out + b * 16, veorq_u8(vld1q_u8(in + b * 16), + AES_enc_block_vec(vld1q_u8(cb), key, (int)nr))); + if (++lo == 0) ++hi; /* adds/adc: 128-bit increment */ + } + if (partial) { + const byte* pin = in + blocks * 16; + byte* pout = out + blocks * 16; + bhi = _byteswap_uint64(hi); blo = _byteswap_uint64(lo); + memcpy(cb, &bhi, 8); memcpy(cb + 8, &blo, 8); + vst1q_u8(tmp, AES_enc_block_vec(vld1q_u8(cb), key, (int)nr)); + if (++lo == 0) ++hi; + for (i = 0; i < partial; i++) pout[i] = (byte)(tmp[i] ^ pin[i]); + *left = 16 - partial; + } + bhi = _byteswap_uint64(hi); blo = _byteswap_uint64(lo); + memcpy(reg + 0, &bhi, 8); memcpy(reg + 8, &blo, 8); +} + +/* Shared J0 derivation for the one-shot GCM kernels (12-byte nonce fast path, + * else GHASH the nonce plus its length block). Mirrors AES_GCM_init_AARCH64. */ +static WC_INLINE uint8x16_t AES_gcm_j0(const byte* nonce, word32 nonceSz, + uint8x16_t H) +{ + uint8x16_t t; + word32 blocks, rem, i; + uint64x2_t L; + + if (nonceSz == 12) { + byte b[16]; + memcpy(b, nonce, 12); + b[12] = 0; b[13] = 0; b[14] = 0; b[15] = 1; + return vld1q_u8(b); + } + t = vdupq_n_u8(0); + blocks = nonceSz >> 4; + rem = nonceSz & 15; + for (i = 0; i < blocks; i++) t = AES_gcm_ghash_fold(t, nonce + i * 16, H); + if (rem) t = AES_gcm_ghash_fold_partial(t, nonce + blocks * 16, rem, H); + L = vdupq_n_u64(0); + L = vsetq_lane_u64(_byteswap_uint64((word64)nonceSz << 3), L, 1); + t = AES_gcm_ghash_mul(veorq_u8(t, vrbitq_u8(vreinterpretq_u8_u64(L))), H); + return vrbitq_u8(t); +} + +/* Shared GCM tag finalize over the (aadSz,sz) bit-length block. */ +static WC_INLINE uint8x16_t AES_gcm_tag_finish(uint8x16_t tagv, uint8x16_t H, + uint8x16_t encJ0, word32 sz, word32 aadSz) +{ + uint64x2_t L = vdupq_n_u64(0); + L = vsetq_lane_u64(AES_rbit64((word64)aadSz << 3), L, 0); + L = vsetq_lane_u64(AES_rbit64((word64)sz << 3), L, 1); + tagv = AES_gcm_ghash_mul(veorq_u8(tagv, vreinterpretq_u8_u64(L)), H); + return veorq_u8(vrbitq_u8(tagv), encJ0); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 5256-10136 (AES_GCM_encrypt_AARCH64) + * One-shot GCM encrypt: J0, GHASH the AAD, CTR-encrypt while GHASHing the + * ciphertext, then finalize the tag over the length block. */ +void AES_GCM_encrypt_AARCH64(const byte* in, byte* out, word32 sz, + const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz, const byte* aad, + word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, int nr) +{ + uint8x16_t H = vld1q_u8(gcm_h); + uint8x16_t J0 = AES_gcm_j0(nonce, nonceSz, H); + uint8x16_t encJ0 = AES_enc_block_vec(J0, key, nr); + uint8x16_t t = vdupq_n_u8(0); + uint8x16_t ctr = J0; + word32 ab = aadSz >> 4, ar = aadSz & 15; + word32 blocks = sz >> 4, rem = sz & 15; + word32 i, b; + byte fb[16]; + + (void)tmp; + vst1q_u8(reg, J0); + for (i = 0; i < ab; i++) t = AES_gcm_ghash_fold(t, aad + i * 16, H); + if (ar) t = AES_gcm_ghash_fold_partial(t, aad + ab * 16, ar, H); + + for (b = 0; b < blocks; b++) { + uint8x16_t ct = veorq_u8(vld1q_u8(in + b * 16), + AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr)); + vst1q_u8(out + b * 16, ct); + t = AES_gcm_ghash_mul(veorq_u8(t, vrbitq_u8(ct)), H); + } + if (rem) { + byte kb[16], cb[16]; + vst1q_u8(kb, AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr)); + memset(cb, 0, 16); + for (i = 0; i < rem; i++) { + cb[i] = (byte)(in[blocks * 16 + i] ^ kb[i]); + out[blocks * 16 + i] = cb[i]; + } + t = AES_gcm_ghash_fold(t, cb, H); + } + vst1q_u8(fb, AES_gcm_tag_finish(t, H, encJ0, sz, aadSz)); + memcpy(tag, fb, tagSz > 16 ? 16 : tagSz); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 10139-15070 (AES_GCM_decrypt_AARCH64) + * One-shot GCM decrypt: GHASHes the ciphertext (input) while CTR-decrypting, then + * verifies the tag. RETURN ENCODING: 0 on match, -180 (AES_GCM_AUTH_E) on + * mismatch — the asm does `csetm ne; and #-180` and aes.c assigns the return + * value straight to `ret`. (Distinct from AES_GCM_decrypt_final_AARCH64, whose + * *res is 180 on match / 0 on mismatch.) */ +int AES_GCM_decrypt_AARCH64(const byte* in, byte* out, word32 sz, + const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz, + const byte* aad, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, + int nr) +{ + uint8x16_t H = vld1q_u8(gcm_h); + uint8x16_t J0 = AES_gcm_j0(nonce, nonceSz, H); + uint8x16_t encJ0 = AES_enc_block_vec(J0, key, nr); + uint8x16_t t = vdupq_n_u8(0); + uint8x16_t ctr = J0; + word32 ab = aadSz >> 4, ar = aadSz & 15; + word32 blocks = sz >> 4, rem = sz & 15; + word32 i, b, n; + byte fb[16]; + byte diff = 0; + + (void)tmp; + vst1q_u8(reg, J0); + for (i = 0; i < ab; i++) t = AES_gcm_ghash_fold(t, aad + i * 16, H); + if (ar) t = AES_gcm_ghash_fold_partial(t, aad + ab * 16, ar, H); + + for (b = 0; b < blocks; b++) { + uint8x16_t ct = vld1q_u8(in + b * 16); + t = AES_gcm_ghash_mul(veorq_u8(t, vrbitq_u8(ct)), H); + vst1q_u8(out + b * 16, veorq_u8(ct, + AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr))); + } + if (rem) { + byte kb[16]; + t = AES_gcm_ghash_fold_partial(t, in + blocks * 16, rem, H); + vst1q_u8(kb, AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr)); + for (i = 0; i < rem; i++) + out[blocks * 16 + i] = (byte)(in[blocks * 16 + i] ^ kb[i]); + } + vst1q_u8(fb, AES_gcm_tag_finish(t, H, encJ0, sz, aadSz)); + + n = tagSz > 16 ? 16 : tagSz; + for (i = 0; i < n; i++) diff |= (byte)(fb[i] ^ tag[i]); + return (diff == 0) ? 0 : -180; +} + +/* intrinsics for armv8-aes-asm_c.c: lines 25292-29120 + * (AES_GCM_encrypt_update_AARCH64) + * Streaming GCM encrypt: CTR-encrypt from the running counter while GHASHing the + * produced ciphertext into the running tag; counter and tag updated in place. + * + * WHOLE BLOCKS ONLY: the asm computes blocks = nbytes>>4 and has NO partial-block + * path. aes.c always passes blocks*WC_AES_BLOCK_SIZE and handles any partial + * itself (AES_GCM_encrypt_block + AES_GCM_ghash_block on a zero-padded + * LASTGBLOCK). Do not add a partial path here. */ +void AES_GCM_encrypt_update_AARCH64(const byte* key, int nr, byte* out, + const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) +{ + uint8x16_t H = vld1q_u8(h); + uint8x16_t t = vld1q_u8(tag); + uint8x16_t ctr = vld1q_u8(counter); + word32 blocks = nbytes >> 4; + word32 b; + for (b = 0; b < blocks; b++) { + uint8x16_t ct = veorq_u8(vld1q_u8(in + b * 16), + AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr)); + vst1q_u8(out + b * 16, ct); + t = AES_gcm_ghash_mul(veorq_u8(t, vrbitq_u8(ct)), H); + } + vst1q_u8(tag, t); + vst1q_u8(counter, ctr); +} + +/* intrinsics for armv8-aes-asm_c.c: lines 29200-33029 + * (AES_GCM_decrypt_update_AARCH64) + * Streaming GCM decrypt; whole blocks only, same contract as the encrypt side. */ +void AES_GCM_decrypt_update_AARCH64(const byte* key, int nr, byte* out, + const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) +{ + uint8x16_t H = vld1q_u8(h); + uint8x16_t t = vld1q_u8(tag); + uint8x16_t ctr = vld1q_u8(counter); + word32 blocks = nbytes >> 4; + word32 b; + for (b = 0; b < blocks; b++) { + uint8x16_t ct = vld1q_u8(in + b * 16); + t = AES_gcm_ghash_mul(veorq_u8(t, vrbitq_u8(ct)), H); + vst1q_u8(out + b * 16, veorq_u8(ct, + AES_enc_block_vec(AES_gcm_ctr_next(&ctr), key, nr))); + } + vst1q_u8(tag, t); + vst1q_u8(counter, ctr); +} + +/* ---- Base software-fallback layer ---- + * + * Besides the hardware-crypto *_AARCH64 kernels above, aes.c UNCONDITIONALLY + * references a second, table-based software layer (AES_ECB/CBC/CTR/GCM_encrypt, + * GCM_gmult_len). Upstream those live in armv8-aes-asm_c.c as GCC inline asm + * (T-table AES + a table-driven GHASH), so MSVC cannot build them. + * + * At runtime this layer is DEAD CODE in this build: Check_CPU_support_HwCrypto + * sets use_aes_hw_crypto/use_pmull_hw_crypto unconditionally (see + * WOLFSSL_ARMASM_FORCE_HW_CRYPTO), so every call site takes the hw-crypto + * branch. Only the linker needs the symbols. Rather than transliterate a second + * asm implementation that never executes, these are thin wrappers over the + * already-verified kernels above — semantically equivalent (identical mode + * definitions and counter conventions, confirmed against the upstream asm: + * base AES_CTR_encrypt increments a full 128-bit big-endian counter via + * adds/adc, while base AES_GCM_encrypt increments only the big-endian word[3] + * via rev32 + add w9 — matching AES_CTR_encrypt_AARCH64 and the GCM kernels + * respectively), so behavior is correct even if a future change did reach them. + * + * `len` is a byte count and callers only pass whole blocks here; `ks` is the + * expanded key schedule; the T-table operand of the upstream asm is not needed + * because the AES instructions compute the S-box directly. */ + +void AES_ECB_encrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr) +{ + AES_encrypt_blocks_AARCH64(in, out, (word32)len, (byte*)ks, nr); +} + +void AES_ECB_decrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr) +{ + AES_decrypt_blocks_AARCH64(in, out, (word32)len, (byte*)ks, nr); +} + +void AES_CBC_encrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr, unsigned char* iv) +{ + AES_CBC_encrypt_AARCH64(in, out, (word32)len, iv, (byte*)ks, nr); +} + +void AES_CBC_decrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr, unsigned char* iv) +{ + AES_CBC_decrypt_AARCH64(in, out, (word32)len, iv, (byte*)ks, nr); +} + +/* Full 128-bit big-endian counter (as the upstream base kernel does). The + * *_AARCH64 CTR kernel also maintains a leftover-keystream block, which this + * entry point has no parameters for; callers here always pass whole blocks, so + * pass local scratch and discard it. */ +void AES_CTR_encrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr, unsigned char* ctr) +{ + byte tmp[16]; + word32 left = 0; + AES_CTR_encrypt_AARCH64(in, out, (word32)len, ctr, (byte*)ks, tmp, &left, + (word32)nr); +} + +/* GCM keystream only (no GHASH: aes.c folds the tag via GCM_GMULT_LEN itself). + * Counter is the big-endian word[3] form. */ +void AES_GCM_encrypt(const unsigned char* in, unsigned char* out, + unsigned long len, const unsigned char* ks, int nr, unsigned char* ctr) +{ + uint8x16_t c = vld1q_u8(ctr); + word32 blocks = (word32)(len >> 4); + word32 b; + for (b = 0; b < blocks; b++) { + vst1q_u8(out + b * 16, veorq_u8(vld1q_u8(in + b * 16), + AES_enc_block_vec(AES_gcm_ctr_next(&c), (const byte*)ks, nr))); + } + vst1q_u8(ctr, c); +} + +/* GHASH-fold `len` bytes into the running state x. + * + * Upstream this is a 4-bit-table routine driven by Gcm::M0. Rather than + * re-derive that table algorithm (32 pre-rotated entries plus a remainder + * table - easy to get subtly wrong, and it would never execute here), recover + * the hash subkey and reuse the PMULL GHASH kernel that is already proven + * byte-exact against the upstream asm. + * + * GenerateM0() builds the table from the subkey with `m[0x8] = gcm->H` + * (XMEMCPY, no transform), and the byte-reversing fixup at the end of + * GenerateM0 is gated on WOLFSSL_ARMASM_NO_HW_CRYPTO, which this build does not + * define. So M0[8] is exactly gcm->H, in the same bit-reversed representation + * that AES_GCM_set_key_AARCH64 produced and that AES_GCM_ghash_block_AARCH64 + * consumes. */ +void GCM_gmult_len(unsigned char* x, const unsigned char** m, + const unsigned char* data, unsigned long len) +{ + const byte (*M0)[16] = (const byte (*)[16])m; + byte h[16]; + + memcpy(h, M0[0x8], 16); + while (len >= 16) { + AES_GCM_ghash_block_AARCH64(data, x, h); + data += 16; + len -= 16; + } +} + +#endif /* WOLFSSL_ARMASM && real-MSVC ARM64 && HW crypto */ diff --git a/wolfcrypt/src/port/arm/armv8-chacha-intrinsics-msvc.c b/wolfcrypt/src/port/arm/armv8-chacha-intrinsics-msvc.c new file mode 100644 index 00000000000..d7e6e4c7943 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-chacha-intrinsics-msvc.c @@ -0,0 +1,253 @@ +/* armv8-chacha-intrinsics-msvc.c + * + * NEON C intrinsics implementation of the ARM64 ChaCha20 kernels, as a third + * alternative to the GAS/inline-asm and armasm64 (.asm) forms wolfSSL already + * ships. Opt-in behind WOLFSSL_ARMASM_INTRINSICS, so it never competes with the + * armasm64 objects for symbols. See armv8-aes-intrinsics-msvc.c for rationale. + * + * SIBLING file (4th), like armv8-aes/sha/poly1305-intrinsics-msvc.c: upstream + * generated sources are never edited in place. + * + * Verified against THREE references: the original inline asm (extracted + * verbatim and built with clang-cl, since cl.exe cannot compile it), an + * independent RFC 8439 pure-C oracle, and the published RFC 8439 section 2.4.2 + * vector -- plus a RAGGED MULTI-CALL streaming test, which is the only shape + * that exercises the over/left leftover-keystream contract. THREE negative + * controls proven RED first: a wrong rotation amount (7->6), never stashing the + * leftover keystream, and skipping the block-counter increment. + * + * SCOPE: the four symbols wolfcrypt/src/chacha.c requires on ARM64 -- + * wc_chacha_setkey, wc_chacha_setiv, wc_chacha_use_over, wc_chacha_crypt_bytes. + * + * IMPLEMENTATION: FOUR-BLOCK-PARALLEL, like upstream's asm but expressed in + * intrinsics rather than transliterated from its ~900 lines. + * + * HISTORY WORTH KEEPING: the first version did ONE block per iteration. It was + * correct (same tests) but MEASURED SLOWER than wolfSSL's portable C -- 299 ms vs + * 245 ms in an in-server A/B profile. One block at a time leaves a long + * dependency chain of 16 NEON ops with nothing to interleave, while MSVC /O2 + * schedules the scalar C well. Rewriting to 4 blocks in parallel (each state word + * broadcast across a vector, lane i = block i) restored the expected win: + * microbenchmark 2.0x FASTER than wolfSSL C (0.128 s vs 0.257 s per 40x4 MiB, + * checksums identical). The transposed layout also removes the vextq lane + * rotations the single-block form needs for the diagonal rounds. + * + * LESSON: a faithful-but-narrow NEON port can lose to compiler-optimised scalar + * code. Measure the kernel against the REAL fallback, not against a hand-written + * baseline, and treat "slower than C" as a signal to widen the parallelism. + * + * THE SUBTLE PART is the leftover contract, read from chacha.c: after a call whose + * length is not a multiple of 64, the unused keystream must be in ctx->over with + * ctx->left = the number of unused bytes, because the NEXT call consumes them from + * ((byte*)ctx->over) + CHACHA_CHUNK_BYTES - ctx->left. A single-shot test cannot + * catch a bug here, which is why the Verification project drives a ragged + * multi-call sequence (5,1,10,64,3,100,63,65,200,7 bytes). + * + * ChaCha20 is LITTLE-endian throughout (state words, counter, keystream) -- the + * opposite of SHA-2. No byte swapping anywhere. + */ + +#include + +#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_INTRINSICS) && \ + defined(_MSC_VER) && !defined(__clang__) && \ + (defined(_M_ARM64) || defined(_M_ARM64EC)) && defined(HAVE_CHACHA) + +#include +#include +#include +#include + + +// The four ChaCha20 rotations. rotl16 has a cheaper form via a 16-bit reverse; +// the rest are shift-left OR'd with shift-right-and-insert. +static WC_INLINE uint32x4_t rotl16(uint32x4_t v) { + return vreinterpretq_u32_u16(vrev32q_u16(vreinterpretq_u16_u32(v))); +} +static WC_INLINE uint32x4_t rotl12(uint32x4_t v) { return vsriq_n_u32(vshlq_n_u32(v, 12), v, 20); } +static WC_INLINE uint32x4_t rotl8 (uint32x4_t v) { return vsriq_n_u32(vshlq_n_u32(v, 8), v, 24); } +static WC_INLINE uint32x4_t rotl7 (uint32x4_t v) { return vsriq_n_u32(vshlq_n_u32(v, 7), v, 25); } + +// One 64-byte ChaCha20 block from state x[16] into out[64]. +static void chacha_block(const word32 x[16], byte out[64]) { + uint32x4_t a = vld1q_u32(x + 0); + uint32x4_t b = vld1q_u32(x + 4); + uint32x4_t c = vld1q_u32(x + 8); + uint32x4_t d = vld1q_u32(x + 12); + const uint32x4_t a0 = a, b0 = b, c0 = c, d0 = d; + + for (int i = 0; i < 10; ++i) { + // column round + a = vaddq_u32(a, b); d = veorq_u32(d, a); d = rotl16(d); + c = vaddq_u32(c, d); b = veorq_u32(b, c); b = rotl12(b); + a = vaddq_u32(a, b); d = veorq_u32(d, a); d = rotl8(d); + c = vaddq_u32(c, d); b = veorq_u32(b, c); b = rotl7(b); + // diagonalise: b<<<1 lane, c<<<2, d<<<3 + b = vextq_u32(b, b, 1); c = vextq_u32(c, c, 2); d = vextq_u32(d, d, 3); + // diagonal round + a = vaddq_u32(a, b); d = veorq_u32(d, a); d = rotl16(d); + c = vaddq_u32(c, d); b = veorq_u32(b, c); b = rotl12(b); + a = vaddq_u32(a, b); d = veorq_u32(d, a); d = rotl8(d); + c = vaddq_u32(c, d); b = veorq_u32(b, c); b = rotl7(b); + // undiagonalise + b = vextq_u32(b, b, 3); c = vextq_u32(c, c, 2); d = vextq_u32(d, d, 1); + } + + a = vaddq_u32(a, a0); b = vaddq_u32(b, b0); + c = vaddq_u32(c, c0); d = vaddq_u32(d, d0); + + vst1q_u8(out + 0, vreinterpretq_u8_u32(a)); + vst1q_u8(out + 16, vreinterpretq_u8_u32(b)); + vst1q_u8(out + 32, vreinterpretq_u8_u32(c)); + vst1q_u8(out + 48, vreinterpretq_u8_u32(d)); +} + + +void wc_chacha_setkey(word32* x, const byte* key, word32 keySz) { + // sigma constants: little-endian words of "expand 32-byte k" + // (or "expand 16-byte k" for a 128-bit key, which also duplicates the key). + if (keySz == 32) { + x[0] = 0x61707865u; x[1] = 0x3320646eu; + x[2] = 0x79622d32u; x[3] = 0x6b206574u; + } else { + x[0] = 0x61707865u; x[1] = 0x3120646eu; + x[2] = 0x79622d36u; x[3] = 0x6b206574u; + } + for (int i = 0; i < 8; ++i) { + const byte* p = (keySz == 32) ? (key + i * 4) : (key + (i % 4) * 4); + x[4 + i] = (word32)p[0] | ((word32)p[1] << 8) | + ((word32)p[2] << 16) | ((word32)p[3] << 24); + } +} + +void wc_chacha_setiv(word32* x, const byte* iv, word32 counter) { + x[12] = counter; + for (int i = 0; i < 3; ++i) { + const byte* p = iv + i * 4; + x[13 + i] = (word32)p[0] | ((word32)p[1] << 8) | + ((word32)p[2] << 16) | ((word32)p[3] << 24); + } +} + +void wc_chacha_use_over(byte* over, byte* output, const byte* input, + word32 len) { + // Plain XOR of already-generated keystream. Must NOT touch ctx state -- + // chacha.c owns the left/pointer bookkeeping. + for (word32 i = 0; i < len; ++i) output[i] = input[i] ^ over[i]; +} + +// FOUR-BLOCK-PARALLEL core. +// +// The single-block version was SLOWER than wolfSSL's portable C (measured: 299 ms +// vs 245 ms in-server). Root cause: one block at a time gives a long dependency +// chain of 16 NEON ops with nothing to interleave, while MSVC /O2 schedules the +// scalar C well. Upstream's asm avoids this by running FOUR blocks at once with +// each state word broadcast across a vector's 4 lanes -- lane i belongs to block +// i -- so a quarter-round is 4 independent quarter-rounds and there is no +// intra-vector shuffling in the round loop at all. +// +// That layout ("transposed" / SIMD-across-blocks) also removes the vextq lane +// rotations the single-block form needs for the diagonal rounds: the diagonal +// step just addresses different words, because words live in different registers +// rather than different lanes. +// +// 16 vectors of state, one per word; only the counter word differs per lane. +typedef struct { uint32x4_t w[16]; } State4; + +/* C has no reference parameters, so this takes pointers (the Verification build + * is C++ and used references; behaviour is identical). */ +static WC_INLINE void qr4(uint32x4_t* a, uint32x4_t* b, uint32x4_t* c, + uint32x4_t* d) +{ + *a = vaddq_u32(*a, *b); *d = veorq_u32(*d, *a); *d = rotl16(*d); + *c = vaddq_u32(*c, *d); *b = veorq_u32(*b, *c); *b = rotl12(*b); + *a = vaddq_u32(*a, *b); *d = veorq_u32(*d, *a); *d = rotl8(*d); + *c = vaddq_u32(*c, *d); *b = veorq_u32(*b, *c); *b = rotl7(*b); +} + +// Produce 4 x 64 bytes of keystream from base state x, counters x[12]+0..3. +static void chacha_4blocks(const word32 x[16], byte out[256]) { + State4 s; + for (int i = 0; i < 16; ++i) s.w[i] = vdupq_n_u32(x[i]); + // per-lane counter: block j gets counter x[12]+j + const uint32_t inc[4] = {0u, 1u, 2u, 3u}; + s.w[12] = vaddq_u32(s.w[12], vld1q_u32(inc)); + + State4 in = s; + for (int i = 0; i < 10; ++i) { + qr4(&s.w[0], &s.w[4], &s.w[8], &s.w[12]); // column + qr4(&s.w[1], &s.w[5], &s.w[9], &s.w[13]); + qr4(&s.w[2], &s.w[6], &s.w[10], &s.w[14]); + qr4(&s.w[3], &s.w[7], &s.w[11], &s.w[15]); + qr4(&s.w[0], &s.w[5], &s.w[10], &s.w[15]); // diagonal -- no lane shuffles + qr4(&s.w[1], &s.w[6], &s.w[11], &s.w[12]); + qr4(&s.w[2], &s.w[7], &s.w[8], &s.w[13]); + qr4(&s.w[3], &s.w[4], &s.w[9], &s.w[14]); + } + for (int i = 0; i < 16; ++i) s.w[i] = vaddq_u32(s.w[i], in.w[i]); + + // Untranspose: word i of block j is lane j of s.w[i]. Do it 4 words at a + // time with the standard 4x4 32-bit transpose (trn + zip), matching what the + // asm's trn1/trn2 pairs achieve. + for (int g = 0; g < 4; ++g) { + uint32x4_t r0 = s.w[g * 4 + 0], r1 = s.w[g * 4 + 1]; + uint32x4_t r2 = s.w[g * 4 + 2], r3 = s.w[g * 4 + 3]; + uint32x4x2_t t01 = vtrnq_u32(r0, r1); + uint32x4x2_t t23 = vtrnq_u32(r2, r3); + uint32x4_t b0 = vcombine_u32(vget_low_u32(t01.val[0]), vget_low_u32(t23.val[0])); + uint32x4_t b1 = vcombine_u32(vget_low_u32(t01.val[1]), vget_low_u32(t23.val[1])); + uint32x4_t b2 = vcombine_u32(vget_high_u32(t01.val[0]), vget_high_u32(t23.val[0])); + uint32x4_t b3 = vcombine_u32(vget_high_u32(t01.val[1]), vget_high_u32(t23.val[1])); + vst1q_u8(out + 0 + g * 16, vreinterpretq_u8_u32(b0)); + vst1q_u8(out + 64 + g * 16, vreinterpretq_u8_u32(b1)); + vst1q_u8(out + 128 + g * 16, vreinterpretq_u8_u32(b2)); + vst1q_u8(out + 192 + g * 16, vreinterpretq_u8_u32(b3)); + } +} + + +void wc_chacha_crypt_bytes(ChaCha* ctx, byte* c, const byte* m, word32 len) { + byte ks[256]; + + // Fast path: 4 blocks (256 B) at a time. + while (len >= 256) { + chacha_4blocks(ctx->X, ks); + ctx->X[12] += 4; + for (int i = 0; i < 256; i += 16) { + vst1q_u8(c + i, veorq_u8(vld1q_u8(m + i), vld1q_u8(ks + i))); + } + m += 256; c += 256; len -= 256; + } + // 1..3 whole blocks + while (len >= 64) { + chacha_block(ctx->X, ks); + ctx->X[12]++; + for (int i = 0; i < 64; i += 16) { + vst1q_u8(c + i, veorq_u8(vld1q_u8(m + i), vld1q_u8(ks + i))); + } + m += 64; c += 64; len -= 64; + } + if (len > 0) { + // Partial tail. The leftover keystream must land in ctx->over with + // ctx->left set, because chacha.c consumes it on the NEXT call from + // ((byte*)ctx->over) + CHACHA_CHUNK_BYTES - ctx->left. Write the block + // STRAIGHT into ctx->over (no separate 64-byte memcpy as in v1). + chacha_block(ctx->X, (byte*)ctx->over); + ctx->X[12]++; + const byte* ks2 = (const byte*)ctx->over; + for (word32 i = 0; i < len; ++i) c[i] = m[i] ^ ks2[i]; + ctx->left = 64 - len; + } else { + ctx->left = 0; + } +} +#else + +/* Keep the translation unit non-empty for compilers that reject empty objects. */ +int armv8_chacha_intrinsics_msvc_not_used(void); +int armv8_chacha_intrinsics_msvc_not_used(void) +{ + return 0; +} + +#endif /* WOLFSSL_ARMASM && real-MSVC ARM64 && HAVE_CHACHA */ diff --git a/wolfcrypt/src/port/arm/armv8-poly1305-intrinsics-msvc.c b/wolfcrypt/src/port/arm/armv8-poly1305-intrinsics-msvc.c new file mode 100644 index 00000000000..c260cd63c83 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-poly1305-intrinsics-msvc.c @@ -0,0 +1,194 @@ +/* armv8-poly1305-intrinsics-msvc.c + * + * NEON C intrinsics implementation of the ARM64 Poly1305 kernels, as a third + * alternative to the GAS/inline-asm and armasm64 (.asm) forms wolfSSL already + * ships. Opt-in behind WOLFSSL_ARMASM_INTRINSICS, so it never competes with the + * armasm64 objects for symbols. See armv8-aes-intrinsics-msvc.c for rationale. + * + * SIBLING file, like armv8-aes-intrinsics-msvc.c and armv8-sha-intrinsics-msvc.c: + * upstream generated sources are never edited in place. + * + * Verified against THREE references: the original inline asm (extracted + * verbatim and built with clang-cl, since cl.exe cannot compile it), an + * independent RFC 8439 pure-C oracle, and the published RFC 8439 section 2.5.2 + * test vector. Both negative controls were proven RED first (wrong mod-2^130-5 + * fold multiplier; adding the 2^128 bit to an already-padded tail). + * + * SCOPE: the four symbols wolfcrypt/src/poly1305.c requires on ARM64 -- + * poly1305_set_key, poly1305_arm64_block_16, poly1305_arm64_blocks, + * poly1305_final. + * + * REPRESENTATION CHOICE (deliberate, not an oversight): upstream's asm keeps the + * accumulator in base-2^26 limbs AND precomputes r^1..r^4 (ctx->r1..r4, r4321) so + * it can fold four blocks per NEON iteration. This file uses the classic 5x26-bit + * single-block form instead. That is legitimate because the limb fields are + * PRIVATE to these kernels: no other wolfSSL translation unit reads + * ctx->r64/r1/r2/r3/r4/r4321 (verified by grep), and in the ARM64 path + * poly1305.c delegates entirely to these four functions without touching the + * limbs. Only the resulting MAC is observable, and that is what the KAT checks. + * Consequence: this is correctness-first, not a throughput-optimised port -- it + * does not reproduce the asm's 4-way blocking. Profile before assuming a win. + * + * A first attempt used radix-2^64 with hand-rolled __umulh carry chains; it + * compiled and produced a self-consistent but WRONG tag that only the published + * vector rejected. Kept as a warning: bespoke 130-bit carry code is easy to get + * subtly wrong, and asm-vs-candidate comparison alone would not have caught it if + * both sides shared a misconception. + */ + +#include + +#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_INTRINSICS) && \ + defined(_MSC_VER) && !defined(__clang__) && \ + (defined(_M_ARM64) || defined(_M_ARM64EC)) && defined(HAVE_POLY1305) + +#include +#include +#include + + +static WC_INLINE word32 rd32le(const byte* p) { + return (word32)p[0] | ((word32)p[1] << 8) | ((word32)p[2] << 16) | + ((word32)p[3] << 24); +} + +// One Poly1305 step: h = (h + block) * r mod (2^130 - 5). +// hibit_shifted is 1<<24 for a full 16-byte block, 0 for an already-padded tail. +static void step(Poly1305* ctx, const byte* block, word32 hibit_shifted) { + const word32 r0 = ctx->r1[0], r1 = ctx->r1[1], r2 = ctx->r1[2]; + const word32 r3 = ctx->r1[3], r4 = ctx->r2[0]; + const word32 s1 = ctx->r3[0], s2 = ctx->r3[1]; + const word32 s3 = ctx->r3[2], s4 = ctx->r3[3]; + + word32 h0 = ctx->h[0], h1 = ctx->h[1], h2 = ctx->h[2]; + word32 h3 = ctx->h[3], h4 = ctx->h[4]; + + const word32 b0 = rd32le(block + 0), b1 = rd32le(block + 4); + const word32 b2 = rd32le(block + 8), b3 = rd32le(block + 12); + + // Poly1305 blocks are LITTLE-endian (unlike SHA-2's big-endian words). + h0 += b0 & 0x3ffffff; + h1 += ((b0 >> 26) | (b1 << 6)) & 0x3ffffff; + h2 += ((b1 >> 20) | (b2 << 12)) & 0x3ffffff; + h3 += ((b2 >> 14) | (b3 << 18)) & 0x3ffffff; + h4 += (b3 >> 8) | hibit_shifted; + + // Schoolbook 5x5 multiply; the sN = 5*rN terms are the mod 2^130-5 fold + // (2^130 == 5), so dropping the *5 gives a plausible but wrong tag. + word64 d0 = (word64)h0 * r0 + (word64)h1 * s4 + (word64)h2 * s3 + + (word64)h3 * s2 + (word64)h4 * s1; + word64 d1 = (word64)h0 * r1 + (word64)h1 * r0 + (word64)h2 * s4 + + (word64)h3 * s3 + (word64)h4 * s2; + word64 d2 = (word64)h0 * r2 + (word64)h1 * r1 + (word64)h2 * r0 + + (word64)h3 * s4 + (word64)h4 * s3; + word64 d3 = (word64)h0 * r3 + (word64)h1 * r2 + (word64)h2 * r1 + + (word64)h3 * r0 + (word64)h4 * s4; + word64 d4 = (word64)h0 * r4 + (word64)h1 * r3 + (word64)h2 * r2 + + (word64)h3 * r1 + (word64)h4 * r0; + + word64 c; + c = d0 >> 26; h0 = (word32)d0 & 0x3ffffff; + d1 += c; c = d1 >> 26; h1 = (word32)d1 & 0x3ffffff; + d2 += c; c = d2 >> 26; h2 = (word32)d2 & 0x3ffffff; + d3 += c; c = d3 >> 26; h3 = (word32)d3 & 0x3ffffff; + d4 += c; c = d4 >> 26; h4 = (word32)d4 & 0x3ffffff; + h0 += (word32)(c * 5); c = h0 >> 26; h0 &= 0x3ffffff; + h1 += (word32)c; + + ctx->h[0] = h0; ctx->h[1] = h1; ctx->h[2] = h2; + ctx->h[3] = h3; ctx->h[4] = h4; +} + + +void poly1305_set_key(Poly1305* ctx, const byte* key) { + const word32 t0 = rd32le(key + 0), t1 = rd32le(key + 4); + const word32 t2 = rd32le(key + 8), t3 = rd32le(key + 12); + + // r &= 0x0ffffffc0ffffffc0ffffffc0fffffff, split into 5 x 26-bit limbs. + ctx->r1[0] = t0 & 0x3ffffff; + ctx->r1[1] = ((t0 >> 26) | (t1 << 6)) & 0x3ffff03; + ctx->r1[2] = ((t1 >> 20) | (t2 << 12)) & 0x3ffc0ff; + ctx->r1[3] = ((t2 >> 14) | (t3 << 18)) & 0x3f03fff; + ctx->r2[0] = (t3 >> 8) & 0x00fffff; + + ctx->r3[0] = ctx->r1[1] * 5; + ctx->r3[1] = ctx->r1[2] * 5; + ctx->r3[2] = ctx->r1[3] * 5; + ctx->r3[3] = ctx->r2[0] * 5; + + ctx->pad[0] = rd32le(key + 16); + ctx->pad[1] = rd32le(key + 20); + ctx->pad[2] = rd32le(key + 24); + ctx->pad[3] = rd32le(key + 28); + + memset(ctx->h, 0, sizeof(ctx->h)); + ctx->leftover = 0; + ctx->finished = 0; +} + +void poly1305_arm64_block_16(Poly1305* ctx, const unsigned char* m) { + // Called from the finished path with an already-padded buffer, so the + // implicit 2^128 bit must NOT be added again. + step(ctx, m, ctx->finished ? 0u : (1u << 24)); +} + +void poly1305_arm64_blocks(Poly1305* ctx, const unsigned char* m, size_t bytes) { + for (size_t off = 0; off + POLY1305_BLOCK_SIZE <= bytes; + off += POLY1305_BLOCK_SIZE) { + step(ctx, m + off, 1u << 24); // full blocks always carry the 2^128 bit + } +} + +void poly1305_final(Poly1305* ctx, byte* mac) { + word32 h0 = ctx->h[0], h1 = ctx->h[1], h2 = ctx->h[2]; + word32 h3 = ctx->h[3], h4 = ctx->h[4]; + + word32 c; + c = h1 >> 26; h1 &= 0x3ffffff; + h2 += c; c = h2 >> 26; h2 &= 0x3ffffff; + h3 += c; c = h3 >> 26; h3 &= 0x3ffffff; + h4 += c; c = h4 >> 26; h4 &= 0x3ffffff; + h0 += c * 5; c = h0 >> 26; h0 &= 0x3ffffff; + h1 += c; + + // g = h + 5; keep g iff it did NOT borrow (constant-time select, no branch) + word32 g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff; + word32 g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff; + word32 g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff; + word32 g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff; + word32 g4 = h4 + c - (1u << 26); + + const word32 mask = (g4 >> 31) - 1; + h0 = (h0 & ~mask) | (g0 & mask); + h1 = (h1 & ~mask) | (g1 & mask); + h2 = (h2 & ~mask) | (g2 & mask); + h3 = (h3 & ~mask) | (g3 & mask); + h4 = (h4 & ~mask) | (g4 & mask); + + word32 o0 = (h0 ) | (h1 << 26); + word32 o1 = (h1 >> 6) | (h2 << 20); + word32 o2 = (h2 >> 12) | (h3 << 14); + word32 o3 = (h3 >> 18) | (h4 << 8); + + word64 f; + f = (word64)o0 + ctx->pad[0]; o0 = (word32)f; + f = (word64)o1 + ctx->pad[1] + (f >> 32); o1 = (word32)f; + f = (word64)o2 + ctx->pad[2] + (f >> 32); o2 = (word32)f; + f = (word64)o3 + ctx->pad[3] + (f >> 32); o3 = (word32)f; + + for (int i = 0; i < 4; ++i) mac[i] = (byte)(o0 >> (8 * i)); + for (int i = 0; i < 4; ++i) mac[4 + i] = (byte)(o1 >> (8 * i)); + for (int i = 0; i < 4; ++i) mac[8 + i] = (byte)(o2 >> (8 * i)); + for (int i = 0; i < 4; ++i) mac[12 + i] = (byte)(o3 >> (8 * i)); +} + +#else + +/* Keep the translation unit non-empty for compilers that reject empty objects. */ +int armv8_poly1305_intrinsics_msvc_not_used(void); +int armv8_poly1305_intrinsics_msvc_not_used(void) +{ + return 0; +} + +#endif /* WOLFSSL_ARMASM && real-MSVC ARM64 && HAVE_POLY1305 */ diff --git a/wolfcrypt/src/port/arm/armv8-sha-intrinsics-msvc.c b/wolfcrypt/src/port/arm/armv8-sha-intrinsics-msvc.c new file mode 100644 index 00000000000..1fcbfe06fbc --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-sha-intrinsics-msvc.c @@ -0,0 +1,312 @@ +/* armv8-sha-intrinsics-msvc.c + * + * NEON C intrinsics implementation of the ARM64 SHA-2 hardware-crypto kernels, + * as a third alternative to the GAS/inline-asm and armasm64 (.asm) forms + * wolfSSL already ships. Opt-in behind WOLFSSL_ARMASM_INTRINSICS, so it never + * competes with the armasm64 objects for symbols. See the header comment in + * armv8-aes-intrinsics-msvc.c for the rationale (build systems without an + * ARM64 assembler step, e.g. CMake consumers). + * + * SIBLING file, like armv8-aes-intrinsics-msvc.c: the upstream generated sources + * are never edited in place, so upstream regeneration stays clean. + * + * The kernel was proven against THREE independent references: the original + * inline asm (extracted verbatim and built with clang-cl, since cl.exe cannot + * compile it), an independent pure-C FIPS 180-4 oracle, and two published + * FIPS 180-4 KATs. Both negative controls were proven RED first. + * + * SCOPE: SHA-256 is ACTIVE. SHA-512 is present but INTENTIONALLY INERT -- + * see the WOLFSSL_ARM64_SHA512_HW_MSVC block near the end of this file. + * - Transform_Sha256_Len_neon (the table/NEON non-crypto variant) is not + * ported; WOLFSSL_ARMASM_NO_NEON keeps that layer out. + */ + +#include + +#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_INTRINSICS) && \ + defined(_MSC_VER) && !defined(__clang__) && \ + (defined(_M_ARM64) || defined(_M_ARM64EC)) && \ + !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO) && !defined(NO_SHA256) + +#include +#include +#include + +/* SHA-256 round constants K[0..63] (FIPS 180-4 section 4.2.2). Upstream holds + * these in v8..v23 and loads them once outside the block loop; we mirror that. */ +static const word32 L_SHA256_msvc_k[64] = { + 0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U, + 0x3956c25bU, 0x59f111f1U, 0x923f82a4U, 0xab1c5ed5U, + 0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U, + 0x72be5d74U, 0x80deb1feU, 0x9bdc06a7U, 0xc19bf174U, + 0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU, + 0x2de92c6fU, 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU, + 0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U, + 0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U, + 0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, 0x53380d13U, + 0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U, + 0xa2bfe8a1U, 0xa81a664bU, 0xc24b8b70U, 0xc76c51a3U, + 0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U, + 0x19a4c116U, 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U, + 0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U, + 0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U, + 0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, 0xc67178f2U, +}; + +/* intrinsics for armv8-sha256-asm_c.c: lines 1042-1193 + * (Transform_Sha256_Len_crypto) + * + * SHA-256 compression over len/64 whole 64-byte blocks, using the ARMv8.0 SHA-2 + * crypto extensions. Structure taken from the asm: + * - digest a..h lives in v0,v1; v2,v3 keep the incoming copy for the + * Davies-Meyer feed-forward add. + * - the 16-word message schedule W lives in v4..v7 and is extended IN PLACE. + * - 16 rounds. Rounds 2..13 (1-based) extend the schedule first; rounds 1 and + * 14..16 do not touch W. + * + * TWO NON-OBVIOUS POINTS, both of which produce a self-consistent but WRONG + * digest if got wrong (each was caught by the published KAT during porting): + * + * 1. Schedule indexing. Round 2 in the asm is + * sha256su0 v4,v5 / add v24,v5,v9 / sha256su1 v4,v6,v7 + * The register being UPDATED (v4) is NOT the register fed to the add (v5). + * Hence u0 = (r-1)%4 for the update while the add uses w[r%4]. + * 2. sha256h2 consumes the PRE-sha256h value of v0 (the asm's + * `mov v25.16b, v0.16b`), so it must be captured before vsha256hq_u32. + * + * The message is consumed BIG-ENDIAN, so each loaded vector is byte-reversed + * (rev32 on the .16b view == vrev32q_u8 with a reinterpret round-trip). The + * digest itself stays in host order. + */ +void Transform_Sha256_Len_crypto(wc_Sha256* sha256, const byte* data, word32 len) +{ + uint32x4_t k[16]; + uint32x4_t s0; + uint32x4_t s1; + word32 off; + int i; + + for (i = 0; i < 16; i++) { + k[i] = vld1q_u32(L_SHA256_msvc_k + i * 4); + } + + s0 = vld1q_u32(sha256->digest); + s1 = vld1q_u32(sha256->digest + 4); + + for (off = 0; off + WC_SHA256_BLOCK_SIZE <= len; + off += WC_SHA256_BLOCK_SIZE) { + uint32x4_t w[4]; + uint32x4_t in0; + uint32x4_t in1; + int r; + + for (i = 0; i < 4; i++) { + w[i] = vreinterpretq_u32_u8( + vrev32q_u8(vld1q_u8(data + off + (word32)i * 16))); + } + + in0 = s0; + in1 = s1; + + for (r = 0; r < 16; r++) { + uint32x4_t wk; + uint32x4_t saved; + + if ((r >= 1) && (r <= 12)) { + int u0 = (r - 1) % 4; /* the W register being UPDATED */ + int u1 = r % 4; + int u2 = (r + 1) % 4; + int u3 = (r + 2) % 4; + + w[u0] = vsha256su0q_u32(w[u0], w[u1]); + w[u0] = vsha256su1q_u32(w[u0], w[u2], w[u3]); + } + + wk = vaddq_u32(w[r % 4], k[r]); + + saved = s0; /* mov v25.16b, v0.16b */ + s0 = vsha256hq_u32(s0, s1, wk); + s1 = vsha256h2q_u32(s1, saved, wk); + } + + s0 = vaddq_u32(s0, in0); + s1 = vaddq_u32(s1, in1); + } + + vst1q_u32(sha256->digest, s0); + vst1q_u32(sha256->digest + 4, s1); +} + +/* ========================================================================== + * SHA-512 -- VERIFIED BUT INTENTIONALLY NOT ENABLED + * ========================================================================== + * + * The kernel below is a complete, verified port of + * Transform_Sha512_Len_crypto (armv8-sha512-asm_c.c). It was proven 4/4 against + * the extracted original inline asm, an independent pure-C FIPS 180-4 oracle, + * and the published FIPS 180-4 SHA-512("abc") vector; both negative controls + * were proven RED first (dropping the 5th staging register; omitting the rev64 + * byte-swap). + * + * WHY IT IS STILL COMPILED OUT HERE: + * + * ARMv8.2-SHA512 is an OPTIONAL architectural extension, so unlike + * AES/PMULL/SHA-256 it cannot be assumed present -- executing sha512h on a core + * without it is an illegal instruction. It therefore needs a real runtime + * feature gate, and sha512.c selects its transform at runtime through + * Sha512_SetTransform() based on WOLFSSL_ARMASM_CRYPTO_SHA512 + cpuid_flags. + * + * wolfSSL's Windows-ARM64 cpuid path (cpuid.c, the _WIN32 branch) already + * queries PF_ARM_SHA512_INSTRUCTIONS_AVAILABLE under + * WOLFSSL_ARMASM_CRYPTO_SHA512, so the gate that was missing when this kernel + * was written now exists upstream. Wiring this kernel to it should be + * straightforward, but that has NOT been done or tested here -- the kernel has + * only ever been exercised with the gate forced on, on a core that was + * separately confirmed to execute vsha512hq_u64. Enabling it properly is left + * as follow-up rather than shipped untested. + * + * For reference, x86-64 does NOT use hardware SHA-512 either: Intel SHA-NI + * covers only SHA-1/SHA-256, so wolfSSL's x64 SHA-512 is an AVX1/AVX2 software + * implementation with a four-way runtime downgrade. + * + * DO NOT define WOLFSSL_ARM64_SHA512_HW_MSVC without a working feature probe -- + * it will fault rather than fall back. + */ +#ifdef WOLFSSL_ARM64_SHA512_HW_MSVC + +#include + +static const word64 L_SHA512_msvc_k[80] = { + 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL, + 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL, + 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL, + 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL, + 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL, + 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL, + 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL, + 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL, + 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL, + 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL, + 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL, + 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL, + 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL, + 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL, +}; + +void Transform_Sha512_Len_crypto(wc_Sha512* sha512, const byte* data, word32 len) +{ +#if defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) + /* FIVE-register rotation, derived mechanically from the asm (not guessed). + Two earlier attempts modelled this as a FOUR-register (ab,cd,ef,gh) + rotation and could never be right: the asm rotates v23..v27, where the + extra register is a STAGING value produced by `add stg = cd + gh` in the + middle of the round and which becomes the NEXT step's `ef`. + + Roles in the asm's step 1 (ab=v24 cd=v25 ef=v26 gh=v27 stg=v23): + wk = ext(W + K, 8) ; swap the doubleword pair + t21 = ext(ef, gh, 8) + t22 = ext(cd, ef, 8) + gh = gh + wk + gh = sha512h (gh, t21, t22) + stg = cd + gh ; <-- the 5th register + gh = sha512h2(gh, cd, ab) + Rotation into the next step, also derived: + (ab, cd, ef, gh) <- (gh, ab, stg, ef) + One step advances TWO SHA-512 rounds (each vector holds 2 words), so 40 + steps cover the 80 rounds. W cycles w[0..7]; K advances by one vector + (2 constants) per step. */ + uint64x2_t ab = vld1q_u64(sha512->digest + 0); + uint64x2_t cd = vld1q_u64(sha512->digest + 2); + uint64x2_t ef = vld1q_u64(sha512->digest + 4); + uint64x2_t gh = vld1q_u64(sha512->digest + 6); + + for (word32 off = 0; off + 128 <= len; off += 128) { + uint64x2_t w[8]; + const uint64x2_t ab0 = ab, cd0 = cd, ef0 = ef, gh0 = gh; + int i; + int step; + + for (i = 0; i < 8; i++) { + w[i] = vreinterpretq_u64_u8( + vrev64q_u8(vld1q_u8(data + off + (word32)i * 16))); + } + + for (step = 0; step < 40; step++) { + const int wi = step % 8; + uint64x2_t wk, t21, t22, stg; + + /* From step 8 the asm interleaves the schedule update; expressed + here as: refresh w[wi] before it is consumed, once past the first + 16 words (i.e. from step 8 on). + su0(w[i], w[i+1]) ; su1(w[i], w[i+7], ext(w[i+4], w[i+5], 8)) */ + if (step >= 8) { + const int a1 = (wi + 1) % 8; + const int a4 = (wi + 4) % 8; + const int a5 = (wi + 5) % 8; + const int a7 = (wi + 7) % 8; + w[wi] = vsha512su0q_u64(w[wi], w[a1]); + w[wi] = vsha512su1q_u64(w[wi], w[a7], + vextq_u64(w[a4], w[a5], 1)); + } + + wk = vaddq_u64(w[wi], vld1q_u64(L_SHA512_msvc_k + step * 2)); + wk = vextq_u64(wk, wk, 1); /* ext v20,v20,v20,#8 */ + + t21 = vextq_u64(ef, gh, 1); + t22 = vextq_u64(cd, ef, 1); + + gh = vaddq_u64(gh, wk); + gh = vsha512hq_u64(gh, t21, t22); + stg = vaddq_u64(cd, gh); /* the 5th register */ + gh = vsha512h2q_u64(gh, cd, ab); + + /* (ab, cd, ef, gh) <- (gh, ab, stg, ef) */ + { + const uint64x2_t n_ab = gh; + const uint64x2_t n_cd = ab; + const uint64x2_t n_ef = stg; + const uint64x2_t n_gh = ef; + ab = n_ab; cd = n_cd; ef = n_ef; gh = n_gh; + } + } + + ab = vaddq_u64(ab, ab0); + cd = vaddq_u64(cd, cd0); + ef = vaddq_u64(ef, ef0); + gh = vaddq_u64(gh, gh0); + } + + vst1q_u64(sha512->digest + 0, ab); + vst1q_u64(sha512->digest + 2, cd); + vst1q_u64(sha512->digest + 4, ef); + vst1q_u64(sha512->digest + 6, gh); +#else + (void)sha512; (void)data; (void)len; +#endif +} + +#endif /* WOLFSSL_ARM64_SHA512_HW_MSVC */ + +#else + +/* Keep the translation unit non-empty for compilers that reject empty objects. */ +int armv8_sha_intrinsics_msvc_not_used(void); +int armv8_sha_intrinsics_msvc_not_used(void) +{ + return 0; +} + +#endif /* WOLFSSL_ARMASM && real-MSVC ARM64 && !NO_HW_CRYPTO && !NO_SHA256 */