From 16cc781ea32a5177a120bebcdb50043f040434c2 Mon Sep 17 00:00:00 2001 From: Patrik Wenger Date: Fri, 17 Jul 2026 02:28:09 +0200 Subject: [PATCH] salsa20: SIMD-accelerated backends via fearless_simd Add opt-in SIMD backends (`simd` feature) that process 4 blocks (SSE2/NEON) or 8 blocks (AVX2) in parallel, plus a diagonal single-block path for HSalsa20 and tail processing. The `std` feature implies `simd` and adds runtime SIMD level detection via LazyLock. Without `simd`, the soft backend is used as before and the MSRV stays at 1.85. Backend4 (ParBlocksSize=U4) uses u32x4 for 128-bit SIMD. Backend8 (ParBlocksSize=U8) uses the native-width vector for 256-bit SIMD, with chunked fallback for narrower levels. HSalsa20 uses diagonal SIMD layout where each u32x4 holds one role across all four quarter rounds, with byte shuffles to rotate between column and row rounds. Adds dudect constant-time verification example (ctbench). Restricts --all-features CI test to stable (fearless_simd requires rustc 1.89, above the crate MSRV). Benchmarked on i7-8700B (AVX2), Salsa20/20 throughput: 16B: 372 -> 372 MB/s (1.0x) 256B: 533 -> 579 MB/s (1.1x) 1KB: 537 -> 1855 MB/s (3.5x) 16KB: 542 -> 1949 MB/s (3.6x) --- .github/workflows/salsa20.yml | 1 + Cargo.lock | 328 +++++++++++++++++++++++++++++++ salsa20/CHANGELOG.md | 8 + salsa20/Cargo.toml | 4 + salsa20/README.md | 5 + salsa20/examples/ctbench.rs | 144 ++++++++++++++ salsa20/src/backends.rs | 2 + salsa20/src/backends/simd.rs | 355 ++++++++++++++++++++++++++++++++++ salsa20/src/backends/soft.rs | 2 + salsa20/src/lib.rs | 28 ++- salsa20/src/xsalsa.rs | 43 ++-- 11 files changed, 899 insertions(+), 21 deletions(-) create mode 100644 salsa20/examples/ctbench.rs create mode 100644 salsa20/src/backends/simd.rs diff --git a/.github/workflows/salsa20.yml b/.github/workflows/salsa20.yml index dffe4221..0f3b2a23 100644 --- a/.github/workflows/salsa20.yml +++ b/.github/workflows/salsa20.yml @@ -57,3 +57,4 @@ jobs: toolchain: ${{ matrix.rust }} - run: cargo test - run: cargo test --all-features + if: matrix.rust == 'stable' diff --git a/Cargo.lock b/Cargo.lock index c49e7b75..16a2a18e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,38 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "blobby" version = "0.4.0" @@ -18,12 +50,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -36,6 +83,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "cipher" version = "0.5.2" @@ -49,6 +107,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -67,6 +140,62 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "dudect-bencher" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6174adfb35811a6845001876823965032406784c36e795edb84a74353455e1" +dependencies = [ + "clap", + "ctrlc", + "rand", + "rand_chacha", +] + +[[package]] +name = "fearless_simd" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f5f895dac0865bc235f1364793899569a7db538137f1024dccea56bf41c78d" +dependencies = [ + "libm", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + [[package]] name = "hc-256" version = "0.6.0" @@ -75,6 +204,15 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex-literal" version = "1.1.0" @@ -106,6 +244,72 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rabbit" version = "0.5.0" @@ -114,6 +318,27 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -134,15 +359,118 @@ version = "0.11.0" dependencies = [ "cfg-if", "cipher", + "dudect-bencher", + "fearless_simd", "hex-literal", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index efc51c61..f3581920 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -5,6 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- SIMD-accelerated backends via `fearless_simd` (SSE2/AVX2/NEON), + opt-in via `simd` feature +- SIMD-accelerated HSalsa20 using diagonal vector layout +- `simd` feature for SIMD backends (requires rustc 1.89+) +- `std` feature for runtime SIMD level detection (implies `simd`) + ## 0.11.0 (2026-03-30) ### Added - SSE2 backend ([#333]) diff --git a/salsa20/Cargo.toml b/salsa20/Cargo.toml index 3f3e549a..9d5bf40b 100644 --- a/salsa20/Cargo.toml +++ b/salsa20/Cargo.toml @@ -15,12 +15,16 @@ description = "Pure Rust implementation of the Salsa20 stream cipher" [dependencies] cfg-if = "1" cipher = { version = "0.5", features = ["stream-wrapper"] } +fearless_simd = { version = "0.6", default-features = false, features = ["libm"], optional = true } [dev-dependencies] cipher = { version = "0.5", features = ["dev"] } +dudect-bencher = "0.7" hex-literal = "1" [features] +simd = ["dep:fearless_simd"] +std = ["simd", "fearless_simd/std"] zeroize = ["cipher/zeroize"] [package.metadata.docs.rs] diff --git a/salsa20/README.md b/salsa20/README.md index 9b7b38be..1063c778 100644 --- a/salsa20/README.md +++ b/salsa20/README.md @@ -11,6 +11,10 @@ Implementation of the [Salsa] family of stream ciphers, including the [XSalsa] variants with an extended 192-bit (24-byte) nonce. +SIMD-accelerated on x86/x86\_64 (SSE2, AVX2) and aarch64 (NEON) via +[fearless\_simd]. Enable the `std` feature for runtime SIMD detection; +without it, the SIMD level is determined at compile time. + ## ⚠️ Security Warning: [Hazmat!][hazmat-link] This crate does not ensure ciphertexts are authentic (i.e. by using a MAC to @@ -96,3 +100,4 @@ dual licensed as above, without any additional terms or conditions. [Salsa]: https://en.wikipedia.org/wiki/Salsa20 [XSalsa]: https://cr.yp.to/snuffle/xsalsa-20081128.pdf +[fearless\_simd]: https://crates.io/crates/fearless_simd diff --git a/salsa20/examples/ctbench.rs b/salsa20/examples/ctbench.rs new file mode 100644 index 00000000..00e5492f --- /dev/null +++ b/salsa20/examples/ctbench.rs @@ -0,0 +1,144 @@ +//! Constant-time verification using dudect statistical timing tests. +//! +//! Run with: cargo run --release --features std --example ctbench + +use dudect_bencher::rand::{Rng, RngExt}; +use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main}; +use salsa20::cipher::{KeyIvInit, StreamCipher}; + +fn hsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng) { + use salsa20::hsalsa; + + let input = [0x13u8; 16]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(100_000); + for _ in 0..100_000 { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + for (key, class) in &inputs { + let key: salsa20::Key = (*key).into(); + let input: salsa20::cipher::array::Array = input.into(); + runner.run_one(*class, || { + let _ = hsalsa::(&key, &input); + }); + } +} + +fn xsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let nonce = [0x13u8; 24]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (key, class) in &inputs { + buf.fill(0); + let key = *key; + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_data(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let key = [0x42u8; 32]; + let nonce = [0x13u8; 24]; + + let mut inputs: Vec<(Vec, Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((vec![0u8; size], Class::Left)); + } else { + let mut data = vec![0u8; size]; + rng.fill_bytes(&mut data); + inputs.push((data, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (data, class) in &inputs { + buf.copy_from_slice(data); + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_key_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 64, 100_000); +} +fn xsalsa20_key_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 256, 50_000); +} +fn xsalsa20_key_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 1024, 10_000); +} +fn xsalsa20_key_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 4096, 5_000); +} +fn xsalsa20_key_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 16384, 2_000); +} +fn xsalsa20_key_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 65536, 1_000); +} + +fn xsalsa20_data_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 64, 100_000); +} +fn xsalsa20_data_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 256, 50_000); +} +fn xsalsa20_data_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 1024, 10_000); +} +fn xsalsa20_data_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 4096, 5_000); +} +fn xsalsa20_data_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 16384, 2_000); +} +fn xsalsa20_data_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 65536, 1_000); +} + +ctbench_main!( + hsalsa20_key, + xsalsa20_key_64, + xsalsa20_key_256, + xsalsa20_key_1024, + xsalsa20_key_4096, + xsalsa20_key_16384, + xsalsa20_key_65536, + xsalsa20_data_64, + xsalsa20_data_256, + xsalsa20_data_1024, + xsalsa20_data_4096, + xsalsa20_data_16384, + xsalsa20_data_65536 +); diff --git a/salsa20/src/backends.rs b/salsa20/src/backends.rs index fbc9393c..c904fc83 100644 --- a/salsa20/src/backends.rs +++ b/salsa20/src/backends.rs @@ -1 +1,3 @@ +#[cfg(feature = "simd")] +pub(crate) mod simd; pub(crate) mod soft; diff --git a/salsa20/src/backends/simd.rs b/salsa20/src/backends/simd.rs new file mode 100644 index 00000000..ca749fb7 --- /dev/null +++ b/salsa20/src/backends/simd.rs @@ -0,0 +1,355 @@ +//! SIMD-accelerated backends using fearless_simd. +//! +//! Provides two backends: +//! - `Backend4`: processes 4 blocks in parallel (128-bit SIMD, SSE2/NEON) +//! - `Backend8`: processes 8 blocks in parallel (256-bit SIMD, AVX2) +//! +//! Single-block generation uses a diagonal SIMD layout where each `u32x4` +//! holds one role (a/b/c/d) across all four quarter rounds, avoiding +//! scalar extract/insert between rounds. + +use crate::{STATE_WORDS, SalsaCore, Unsigned}; +use cipher::{ + Block, BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherSeekCore, + consts::{U4, U8, U64}, +}; +use core::ops::{Add, AddAssign, BitOr, BitXor, BitXorAssign, Shl, Shr}; +use fearless_simd::prelude::*; +use fearless_simd::{Simd, u32x4}; + +// --------------------------------------------------------------------------- +// SIMD helpers +// --------------------------------------------------------------------------- + +pub(crate) trait SimdOps: + Copy + + Add + + AddAssign + + BitXor + + BitXorAssign + + BitOr + + Shl + + Shr +{ +} + +impl SimdOps for T where + T: Copy + + Add + + AddAssign + + BitXor + + BitXorAssign + + BitOr + + Shl + + Shr +{ +} + +#[inline(always)] +fn rotl(x: V, n: u32) -> V { + (x << n) | (x >> (32 - n)) +} + +#[inline(always)] +fn salsa_qr_simd(s: &mut [V; 16], a: usize, b: usize, c: usize, d: usize) { + s[b] ^= rotl(s[a] + s[d], 7); + s[c] ^= rotl(s[b] + s[a], 9); + s[d] ^= rotl(s[c] + s[b], 13); + s[a] ^= rotl(s[d] + s[c], 18); +} + +#[inline(always)] +fn salsa_double_round_simd(s: &mut [V; 16]) { + // Column rounds + salsa_qr_simd(s, 0, 4, 8, 12); + salsa_qr_simd(s, 5, 9, 13, 1); + salsa_qr_simd(s, 10, 14, 2, 6); + salsa_qr_simd(s, 15, 3, 7, 11); + // Diagonal rounds + salsa_qr_simd(s, 0, 1, 2, 3); + salsa_qr_simd(s, 5, 6, 7, 4); + salsa_qr_simd(s, 10, 11, 8, 9); + salsa_qr_simd(s, 15, 12, 13, 14); +} + +// --------------------------------------------------------------------------- +// Diagonal single-block keystream generation +// --------------------------------------------------------------------------- + +/// Generate one 64-byte keystream block using diagonal SIMD layout. +/// +/// Each `u32x4` holds one diagonal of the Salsa20 state matrix, so all four +/// quarter rounds execute in parallel within a single SIMD instruction +/// sequence. Shuffles rotate elements between column and row rounds. +#[inline(always)] +fn gen_ks_block_diagonal( + simd: S, + state: &[u32; STATE_WORDS], + block: &mut [u8], +) { + let shuffle_rot1 = + simd.load_array_u8x16([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3]); + let shuffle_rot2 = + simd.load_array_u8x16([8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7]); + let shuffle_rot3 = + simd.load_array_u8x16([12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + let init_v0: u32x4 = u32x4::from_fn(simd, |i| state[[0, 5, 10, 15][i]]); + let init_v1: u32x4 = u32x4::from_fn(simd, |i| state[[4, 9, 14, 3][i]]); + let init_v2: u32x4 = u32x4::from_fn(simd, |i| state[[8, 13, 2, 7][i]]); + let init_v3: u32x4 = u32x4::from_fn(simd, |i| state[[12, 1, 6, 11][i]]); + + let (mut v0, mut v1, mut v2, mut v3) = (init_v0, init_v1, init_v2, init_v3); + + for _ in 0..R::USIZE { + // Column round (diagonal layout: v0=a, v1=b, v2=c, v3=d) + v1 ^= rotl(v0 + v3, 7); + v2 ^= rotl(v1 + v0, 9); + v3 ^= rotl(v2 + v1, 13); + v0 ^= rotl(v3 + v2, 18); + + // Shuffle to row-round alignment + v1 = v1.swizzle_dyn_within_blocks(shuffle_rot3); + v2 = v2.swizzle_dyn_within_blocks(shuffle_rot2); + v3 = v3.swizzle_dyn_within_blocks(shuffle_rot1); + + // Row round (v0=a, v3=b, v2=c, v1=d) + v3 ^= rotl(v0 + v1, 7); + v2 ^= rotl(v3 + v0, 9); + v1 ^= rotl(v2 + v3, 13); + v0 ^= rotl(v1 + v2, 18); + + // Unshuffle back to diagonal layout + v1 = v1.swizzle_dyn_within_blocks(shuffle_rot1); + v2 = v2.swizzle_dyn_within_blocks(shuffle_rot2); + v3 = v3.swizzle_dyn_within_blocks(shuffle_rot3); + } + + // Feedforward + v0 += init_v0; + v1 += init_v1; + v2 += init_v2; + v3 += init_v3; + + // Diagonal layout → sequential byte order + // v0=[s0,s5,s10,s15] v1=[s4,s9,s14,s3] v2=[s8,s13,s2,s7] v3=[s12,s1,s6,s11] + block[0..4].copy_from_slice(&v0[0].to_le_bytes()); + block[4..8].copy_from_slice(&v3[1].to_le_bytes()); + block[8..12].copy_from_slice(&v2[2].to_le_bytes()); + block[12..16].copy_from_slice(&v1[3].to_le_bytes()); + block[16..20].copy_from_slice(&v1[0].to_le_bytes()); + block[20..24].copy_from_slice(&v0[1].to_le_bytes()); + block[24..28].copy_from_slice(&v3[2].to_le_bytes()); + block[28..32].copy_from_slice(&v2[3].to_le_bytes()); + block[32..36].copy_from_slice(&v2[0].to_le_bytes()); + block[36..40].copy_from_slice(&v1[1].to_le_bytes()); + block[40..44].copy_from_slice(&v0[2].to_le_bytes()); + block[44..48].copy_from_slice(&v3[3].to_le_bytes()); + block[48..52].copy_from_slice(&v3[0].to_le_bytes()); + block[52..56].copy_from_slice(&v2[1].to_le_bytes()); + block[56..60].copy_from_slice(&v1[2].to_le_bytes()); + block[60..64].copy_from_slice(&v0[3].to_le_bytes()); +} + +// --------------------------------------------------------------------------- +// Wide parallel-block keystream generation +// --------------------------------------------------------------------------- + +/// Generate N keystream blocks in parallel using wide SIMD. +/// +/// Each SIMD lane processes an independent block with its own counter value. +/// The state words are splatted across lanes, with per-lane counters in s[8] +/// and s[9]. After the rounds, the interleaved output is scattered into +/// separate block buffers. +macro_rules! gen_wide_ks_blocks { + ($vec:ty, $simd:expr, $state:expr, $counter:expr, $n:expr, + $rounds:expr, $blocks:expr) => {{ + let mut s: [$vec; 16] = core::array::from_fn(|i| <$vec>::splat($simd, $state[i])); + s[8] = <$vec>::from_fn($simd, |lane| ($counter + lane as u64) as u32); + s[9] = <$vec>::from_fn($simd, |lane| (($counter + lane as u64) >> 32) as u32); + let initial = s; + + for _ in 0..$rounds { + salsa_double_round_simd(&mut s); + } + + for i in 0..16 { + s[i] += initial[i]; + } + + let n: usize = $n; + for lane in 0..n { + for word in 0..16 { + let val = s[word][lane]; + $blocks[lane][word * 4..word * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } + }}; +} + +// --------------------------------------------------------------------------- +// Backend4: 4 parallel blocks (128-bit SIMD) +// --------------------------------------------------------------------------- + +pub(crate) struct Backend4<'a, R: Unsigned, S: Simd> { + core: &'a mut SalsaCore, + simd: S, +} + +impl<'a, R: Unsigned, S: Simd> Backend4<'a, R, S> { + #[inline(always)] + pub(crate) fn new(core: &'a mut SalsaCore, simd: S) -> Self { + Self { core, simd } + } +} + +impl BlockSizeUser for Backend4<'_, R, S> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend4<'_, R, S> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend4<'_, R, S> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + gen_ks_block_diagonal::(self.simd, &self.core.state, block); + self.core.set_block_pos(self.core.get_block_pos() + 1); + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + let counter = self.core.get_block_pos(); + gen_wide_ks_blocks!( + u32x4, + self.simd, + self.core.state, + counter, + 4, + R::USIZE, + blocks + ); + self.core.set_block_pos(counter + 4); + } +} + +// --------------------------------------------------------------------------- +// Backend8: 8 parallel blocks (256-bit SIMD) +// --------------------------------------------------------------------------- + +pub(crate) struct Backend8<'a, R: Unsigned, S: Simd> { + core: &'a mut SalsaCore, + simd: S, +} + +impl<'a, R: Unsigned, S: Simd> Backend8<'a, R, S> { + #[inline(always)] + pub(crate) fn new(core: &'a mut SalsaCore, simd: S) -> Self { + Self { core, simd } + } +} + +impl BlockSizeUser for Backend8<'_, R, S> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend8<'_, R, S> { + type ParBlocksSize = U8; +} + +impl StreamCipherBackend for Backend8<'_, R, S> +where + S::u32s: SimdOps + core::ops::Index, +{ + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut Block) { + gen_ks_block_diagonal::(self.simd, &self.core.state, block); + self.core.set_block_pos(self.core.get_block_pos() + 1); + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + let counter = self.core.get_block_pos(); + let n = S::u32s::N; + let mut offset = 0usize; + while offset < 8 { + let base = counter + offset as u64; + let batch = core::cmp::min(n, 8 - offset); + gen_wide_ks_blocks!( + S::u32s, + self.simd, + self.core.state, + base, + batch, + R::USIZE, + &mut blocks[offset..] + ); + offset += n; + } + self.core.set_block_pos(counter + 8); + } +} + +// --------------------------------------------------------------------------- +// Dispatch entry point +// --------------------------------------------------------------------------- + +/// Dispatch to the widest available SIMD backend. +#[inline(always)] +pub(crate) fn process( + simd: S, + core: &mut SalsaCore, + f: impl cipher::StreamCipherClosure, +) where + S::u32s: SimdOps + core::ops::Index, +{ + if S::u32s::N >= 8 { + f.call(&mut Backend8::new(core, simd)); + } else { + f.call(&mut Backend4::new(core, simd)); + } +} + +// --------------------------------------------------------------------------- +// HSalsa20 SIMD +// --------------------------------------------------------------------------- + +/// SIMD-accelerated HSalsa20 using diagonal layout. +#[inline(always)] +pub(crate) fn hsalsa_simd(simd: S, state: &[u32; STATE_WORDS]) -> [u32; 8] { + let shuffle_rot1 = + simd.load_array_u8x16([4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3]); + let shuffle_rot2 = + simd.load_array_u8x16([8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7]); + let shuffle_rot3 = + simd.load_array_u8x16([12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + + let mut v0: u32x4 = u32x4::from_fn(simd, |i| state[[0, 5, 10, 15][i]]); + let mut v1: u32x4 = u32x4::from_fn(simd, |i| state[[4, 9, 14, 3][i]]); + let mut v2: u32x4 = u32x4::from_fn(simd, |i| state[[8, 13, 2, 7][i]]); + let mut v3: u32x4 = u32x4::from_fn(simd, |i| state[[12, 1, 6, 11][i]]); + + for _ in 0..R::USIZE { + v1 ^= rotl(v0 + v3, 7); + v2 ^= rotl(v1 + v0, 9); + v3 ^= rotl(v2 + v1, 13); + v0 ^= rotl(v3 + v2, 18); + + v1 = v1.swizzle_dyn_within_blocks(shuffle_rot3); + v2 = v2.swizzle_dyn_within_blocks(shuffle_rot2); + v3 = v3.swizzle_dyn_within_blocks(shuffle_rot1); + + v3 ^= rotl(v0 + v1, 7); + v2 ^= rotl(v3 + v0, 9); + v1 ^= rotl(v2 + v3, 13); + v0 ^= rotl(v1 + v2, 18); + + v1 = v1.swizzle_dyn_within_blocks(shuffle_rot1); + v2 = v2.swizzle_dyn_within_blocks(shuffle_rot2); + v3 = v3.swizzle_dyn_within_blocks(shuffle_rot3); + } + + // HSalsa20 output: words [0,5,10,15,6,7,8,9], no feedforward + [v0[0], v0[1], v0[2], v0[3], v3[2], v2[3], v2[0], v1[1]] +} diff --git a/salsa20/src/backends/soft.rs b/salsa20/src/backends/soft.rs index caf2693f..99dc29dc 100644 --- a/salsa20/src/backends/soft.rs +++ b/salsa20/src/backends/soft.rs @@ -1,6 +1,8 @@ //! Portable implementation which does not rely on architecture-specific //! intrinsics. +#![allow(dead_code)] + use crate::{Block, STATE_WORDS, SalsaCore, Unsigned}; use cipher::{ BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherSeekCore, diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index e794b593..9e445692 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -1,4 +1,4 @@ -#![no_std] +#![cfg_attr(not(feature = "std"), no_std)] #![doc = include_str!("../README.md")] #![doc( html_logo_url = "https://raw.githubusercontent.com/RustCrypto/media/8f1a9894/logo.svg", @@ -17,12 +17,27 @@ use cipher::{ }; use core::marker::PhantomData; +#[cfg(feature = "std")] +extern crate std; + #[cfg(feature = "zeroize")] use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; mod backends; mod xsalsa; +#[cfg(feature = "std")] +fn simd_level() -> fearless_simd::Level { + use std::sync::LazyLock; + static LEVEL: LazyLock = LazyLock::new(fearless_simd::Level::new); + *LEVEL +} + +#[cfg(all(feature = "simd", not(feature = "std")))] +fn simd_level() -> fearless_simd::Level { + fearless_simd::Level::baseline() +} + pub use xsalsa::{XSalsa8, XSalsa12, XSalsa20, XSalsaCore, hsalsa}; /// Salsa20/8 stream cipher @@ -124,7 +139,16 @@ impl StreamCipherCore for SalsaCore { rem.try_into().ok() } fn process_with_backend(&mut self, f: impl StreamCipherClosure) { - f.call(&mut backends::soft::Backend(self)); + cfg_if::cfg_if! { + if #[cfg(feature = "simd")] { + let level = simd_level(); + fearless_simd::dispatch!(level, simd => { + backends::simd::process::(simd, self, f); + }); + } else { + f.call(&mut backends::soft::Backend(self)); + } + } } } diff --git a/salsa20/src/xsalsa.rs b/salsa20/src/xsalsa.rs index 9f84cef5..d8b69fcc 100644 --- a/salsa20/src/xsalsa.rs +++ b/salsa20/src/xsalsa.rs @@ -1,5 +1,7 @@ //! XSalsa20 is an extended nonce variant of Salsa20 +#[cfg(feature = "simd")] +use super::simd_level; use super::{CONSTANTS, Key, Nonce, SalsaCore, Unsigned, XNonce}; use cipher::{ BlockSizeUser, IvSizeUser, KeyIvInit, KeySizeUser, StreamCipherClosure, StreamCipherCore, @@ -8,8 +10,6 @@ use cipher::{ consts::{U4, U6, U10, U16, U24, U32, U64}, }; -use crate::backends::soft::quarter_round; - #[cfg(feature = "zeroize")] use cipher::zeroize::ZeroizeOnDrop; @@ -113,27 +113,32 @@ pub fn hsalsa(key: &Key, input: &Array) -> Array .for_each(|(v, chunk)| *v = to_u32(chunk)); state[15] = CONSTANTS[3]; - // 20 rounds consisting of 10 column rounds and 10 diagonal rounds - for _ in 0..R::USIZE { - // column rounds - quarter_round(0, 4, 8, 12, &mut state); - quarter_round(5, 9, 13, 1, &mut state); - quarter_round(10, 14, 2, 6, &mut state); - quarter_round(15, 3, 7, 11, &mut state); - - // diagonal rounds - quarter_round(0, 1, 2, 3, &mut state); - quarter_round(5, 6, 7, 4, &mut state); - quarter_round(10, 11, 8, 9, &mut state); - quarter_round(15, 12, 13, 14, &mut state); + cfg_if::cfg_if! { + if #[cfg(feature = "simd")] { + let level = simd_level(); + let words = fearless_simd::dispatch!(level, simd => { + crate::backends::simd::hsalsa_simd::(simd, &state) + }); + } else { + use crate::backends::soft::quarter_round; + for _ in 0..R::USIZE { + quarter_round(0, 4, 8, 12, &mut state); + quarter_round(5, 9, 13, 1, &mut state); + quarter_round(10, 14, 2, 6, &mut state); + quarter_round(15, 3, 7, 11, &mut state); + quarter_round(0, 1, 2, 3, &mut state); + quarter_round(5, 6, 7, 4, &mut state); + quarter_round(10, 11, 8, 9, &mut state); + quarter_round(15, 12, 13, 14, &mut state); + } + let key_idx: [usize; 8] = [0, 5, 10, 15, 6, 7, 8, 9]; + let words: [u32; 8] = core::array::from_fn(|i| state[key_idx[i]]); + } } let mut output = Array::default(); - let key_idx: [usize; 8] = [0, 5, 10, 15, 6, 7, 8, 9]; - for (i, chunk) in output.chunks_exact_mut(4).enumerate() { - chunk.copy_from_slice(&state[key_idx[i]].to_le_bytes()); + chunk.copy_from_slice(&words[i].to_le_bytes()); } - output }