diff --git a/Cargo.lock b/Cargo.lock index c49e7b75..b6b73500 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,38 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + [[package]] name = "blobby" version = "0.4.0" @@ -18,12 +50,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + [[package]] name = "chacha20" version = "0.10.1" @@ -36,6 +83,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "cipher" version = "0.5.2" @@ -49,6 +107,21 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -67,6 +140,53 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix", + "windows-sys", +] + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.1", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "dudect-bencher" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6174adfb35811a6845001876823965032406784c36e795edb84a74353455e1" +dependencies = [ + "clap", + "ctrlc", + "rand", + "rand_chacha", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + [[package]] name = "hc-256" version = "0.6.0" @@ -75,6 +195,15 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex-literal" version = "1.1.0" @@ -106,6 +235,66 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rabbit" version = "0.5.0" @@ -114,6 +303,27 @@ dependencies = [ "hex-literal", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e6af7f3e25ded52c41df4e0b1af2d047e45896c2f3281792ed68a1c243daedb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + [[package]] name = "rand_core" version = "0.10.1" @@ -134,15 +344,118 @@ version = "0.11.0" dependencies = [ "cfg-if", "cipher", + "cpufeatures", + "dudect-bencher", "hex-literal", ] +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/salsa20/CHANGELOG.md b/salsa20/CHANGELOG.md index efc51c61..410fdf34 100644 --- a/salsa20/CHANGELOG.md +++ b/salsa20/CHANGELOG.md @@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased +### Added +- SSE2 and AVX2 intrinsics backends with runtime detection via `cpufeatures` +- `dudect` constant-time timing tests (`examples/ctbench.rs`) + ## 0.11.0 (2026-03-30) ### Added - SSE2 backend ([#333]) diff --git a/salsa20/Cargo.toml b/salsa20/Cargo.toml index 3f3e549a..06ba2618 100644 --- a/salsa20/Cargo.toml +++ b/salsa20/Cargo.toml @@ -16,8 +16,12 @@ description = "Pure Rust implementation of the Salsa20 stream cipher" cfg-if = "1" cipher = { version = "0.5", features = ["stream-wrapper"] } +[target.'cfg(any(target_arch = "x86_64", target_arch = "x86"))'.dependencies] +cpufeatures = "0.3" + [dev-dependencies] cipher = { version = "0.5", features = ["dev"] } +dudect-bencher = "0.7" hex-literal = "1" [features] diff --git a/salsa20/README.md b/salsa20/README.md index 9b7b38be..903e2a7f 100644 --- a/salsa20/README.md +++ b/salsa20/README.md @@ -11,6 +11,9 @@ Implementation of the [Salsa] family of stream ciphers, including the [XSalsa] variants with an extended 192-bit (24-byte) nonce. +SIMD-accelerated on x86/x86\_64 with SSE2 and AVX2 backends, selected +automatically at runtime via [cpufeatures]. + ## ⚠️ Security Warning: [Hazmat!][hazmat-link] This crate does not ensure ciphertexts are authentic (i.e. by using a MAC to @@ -96,3 +99,4 @@ dual licensed as above, without any additional terms or conditions. [Salsa]: https://en.wikipedia.org/wiki/Salsa20 [XSalsa]: https://cr.yp.to/snuffle/xsalsa-20081128.pdf +[cpufeatures]: https://crates.io/crates/cpufeatures diff --git a/salsa20/examples/ctbench.rs b/salsa20/examples/ctbench.rs new file mode 100644 index 00000000..b8da4f4f --- /dev/null +++ b/salsa20/examples/ctbench.rs @@ -0,0 +1,144 @@ +//! Constant-time verification using dudect statistical timing tests. +//! +//! Run with: cargo run --release --example ctbench + +use dudect_bencher::rand::{Rng, RngExt}; +use dudect_bencher::{BenchRng, Class, CtRunner, ctbench_main}; +use salsa20::cipher::{KeyIvInit, StreamCipher}; + +fn hsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng) { + use salsa20::hsalsa; + + let input = [0x13u8; 16]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(100_000); + for _ in 0..100_000 { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + for (key, class) in &inputs { + let key: salsa20::Key = (*key).into(); + let input: salsa20::cipher::array::Array = input.into(); + runner.run_one(*class, || { + let _ = hsalsa::(&key, &input); + }); + } +} + +fn xsalsa20_key(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let nonce = [0x13u8; 24]; + let fixed_key = [0u8; 32]; + + let mut inputs: Vec<([u8; 32], Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((fixed_key, Class::Left)); + } else { + let mut key = [0u8; 32]; + rng.fill_bytes(&mut key); + inputs.push((key, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (key, class) in &inputs { + buf.fill(0); + let key = *key; + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_data(runner: &mut CtRunner, rng: &mut BenchRng, size: usize, n: usize) { + let key = [0x42u8; 32]; + let nonce = [0x13u8; 24]; + + let mut inputs: Vec<(Vec, Class)> = Vec::with_capacity(n); + for _ in 0..n { + if rng.random::() { + inputs.push((vec![0u8; size], Class::Left)); + } else { + let mut data = vec![0u8; size]; + rng.fill_bytes(&mut data); + inputs.push((data, Class::Right)); + } + } + + let mut buf = vec![0u8; size]; + for (data, class) in &inputs { + buf.copy_from_slice(data); + let buf_ptr = buf.as_mut_ptr(); + let buf_len = buf.len(); + runner.run_one(*class, || { + let buf = unsafe { core::slice::from_raw_parts_mut(buf_ptr, buf_len) }; + let mut cipher = salsa20::XSalsa20::new((&key).into(), (&nonce).into()); + cipher.apply_keystream(buf); + }); + } +} + +fn xsalsa20_key_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 64, 100_000); +} +fn xsalsa20_key_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 256, 50_000); +} +fn xsalsa20_key_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 1024, 10_000); +} +fn xsalsa20_key_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 4096, 5_000); +} +fn xsalsa20_key_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 16384, 2_000); +} +fn xsalsa20_key_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_key(r, rng, 65536, 1_000); +} + +fn xsalsa20_data_64(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 64, 100_000); +} +fn xsalsa20_data_256(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 256, 50_000); +} +fn xsalsa20_data_1024(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 1024, 10_000); +} +fn xsalsa20_data_4096(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 4096, 5_000); +} +fn xsalsa20_data_16384(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 16384, 2_000); +} +fn xsalsa20_data_65536(r: &mut CtRunner, rng: &mut BenchRng) { + xsalsa20_data(r, rng, 65536, 1_000); +} + +ctbench_main!( + hsalsa20_key, + xsalsa20_key_64, + xsalsa20_key_256, + xsalsa20_key_1024, + xsalsa20_key_4096, + xsalsa20_key_16384, + xsalsa20_key_65536, + xsalsa20_data_64, + xsalsa20_data_256, + xsalsa20_data_1024, + xsalsa20_data_4096, + xsalsa20_data_16384, + xsalsa20_data_65536 +); diff --git a/salsa20/src/backends.rs b/salsa20/src/backends.rs index fbc9393c..a1ddcb6a 100644 --- a/salsa20/src/backends.rs +++ b/salsa20/src/backends.rs @@ -1 +1,8 @@ pub(crate) mod soft; + +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + pub(crate) mod avx2; + pub(crate) mod sse2; + } +} diff --git a/salsa20/src/backends/avx2.rs b/salsa20/src/backends/avx2.rs new file mode 100644 index 00000000..5ede5393 --- /dev/null +++ b/salsa20/src/backends/avx2.rs @@ -0,0 +1,228 @@ +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(clippy::cast_possible_wrap)] + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{STATE_WORDS, Unsigned}; +use cipher::{ + BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, + consts::{U4, U64}, +}; +use core::marker::PhantomData; + +/// Number of blocks processed in parallel. +const PAR_BLOCKS: usize = 4; +/// Number of `__m256i` sets (each holds 2 blocks). +const N: usize = PAR_BLOCKS / 2; + +/// Diagonal indices for the Salsa20 state matrix. +const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; + +macro_rules! rotl_epi32 { + ($v:expr, $n:literal) => {{ + let v = $v; + _mm256_or_si256(_mm256_slli_epi32(v, $n), _mm256_srli_epi32(v, 32 - $n)) + }}; +} + +#[inline] +#[target_feature(enable = "avx2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamCipherClosure, +{ + f.call(&mut Backend:: { + state, + _pd: PhantomData, + }); +} + +struct Backend<'a, R: Unsigned> { + state: &'a mut [u32; STATE_WORDS], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut cipher::Block) { + unsafe { + let res = rounds::(self.state); + let lo = extract_lo(&res[0]); + store_block(&lo, block); + let counter = get_counter(self.state); + set_counter(self.state, counter + 1); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + unsafe { + let res = rounds::(self.state); + for i in 0..N { + let lo = extract_lo(&res[i]); + let hi = extract_hi(&res[i]); + store_block(&lo, &mut blocks[2 * i]); + store_block(&hi, &mut blocks[2 * i + 1]); + } + let counter = get_counter(self.state); + set_counter(self.state, counter + PAR_BLOCKS as u64); + } + } +} + +/// Load two blocks' diagonal values into `__m256i` registers. +/// +/// Each `__m256i` holds two blocks: the lower 128 bits contain block 0's +/// diagonal and the upper 128 bits contain block 1's diagonal. +#[inline(always)] +unsafe fn load_diag_pair(state: &[u32; STATE_WORDS], ctr0: u64, ctr1: u64) -> [__m256i; 4] { + let s = state; + [ + // a: [s0, s5, s10, s15] (same for both blocks) + _mm256_set_epi32( + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + s[15] as i32, + s[10] as i32, + s[5] as i32, + s[0] as i32, + ), + // b: [s4, ctr_hi, s14, s3] (ctr_hi differs) + _mm256_set_epi32( + s[3] as i32, + s[14] as i32, + (ctr1 >> 32) as i32, + s[4] as i32, + s[3] as i32, + s[14] as i32, + (ctr0 >> 32) as i32, + s[4] as i32, + ), + // c: [ctr_lo, s13, s2, s7] (ctr_lo differs) + _mm256_set_epi32( + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr1 as i32, + s[7] as i32, + s[2] as i32, + s[13] as i32, + ctr0 as i32, + ), + // d: [s12, s1, s6, s11] (same for both blocks) + _mm256_set_epi32( + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + s[11] as i32, + s[6] as i32, + s[1] as i32, + s[12] as i32, + ), + ] +} + +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m256i; 4]; N] { + let counter = get_counter(state); + let mut res = [[_mm256_setzero_si256(); 4]; N]; + for (i, block) in res.iter_mut().enumerate() { + let base = counter + (i * 2) as u64; + *block = load_diag_pair(state, base, base + 1); + } + let initial = res; + + for _ in 0..R::USIZE * 2 { + for block in res.iter_mut() { + quarter_round(block); + shuffle(block); + } + } + + for i in 0..N { + for j in 0..4 { + res[i][j] = _mm256_add_epi32(res[i][j], initial[i][j]); + } + } + + res +} + +#[inline(always)] +unsafe fn quarter_round(block: &mut [__m256i; 4]) { + let [a, b, c, d] = block; + *b = _mm256_xor_si256(*b, rotl_epi32!(_mm256_add_epi32(*a, *d), 7)); + *c = _mm256_xor_si256(*c, rotl_epi32!(_mm256_add_epi32(*b, *a), 9)); + *d = _mm256_xor_si256(*d, rotl_epi32!(_mm256_add_epi32(*c, *b), 13)); + *a = _mm256_xor_si256(*a, rotl_epi32!(_mm256_add_epi32(*d, *c), 18)); +} + +#[inline(always)] +unsafe fn shuffle(block: &mut [__m256i; 4]) { + let [_, b, c, d] = block; + let tmp = *b; + *b = _mm256_shuffle_epi32(*d, 0b_00_11_10_01); + *c = _mm256_shuffle_epi32(*c, 0b_01_00_11_10); + *d = _mm256_shuffle_epi32(tmp, 0b_10_01_00_11); +} + +#[inline(always)] +unsafe fn extract_lo(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_castsi256_si128(set[0]), + _mm256_castsi256_si128(set[1]), + _mm256_castsi256_si128(set[2]), + _mm256_castsi256_si128(set[3]), + ] +} + +#[inline(always)] +unsafe fn extract_hi(set: &[__m256i; 4]) -> [__m128i; 4] { + [ + _mm256_extracti128_si256(set[0], 1), + _mm256_extracti128_si256(set[1], 1), + _mm256_extracti128_si256(set[2], 1), + _mm256_extracti128_si256(set[3], 1), + ] +} + +/// Convert diagonal SIMD layout back to sequential little-endian bytes. +#[inline(always)] +unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { + let mut vals = [[0u32; 4]; 4]; + for (i, v) in diag.iter().enumerate() { + _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); + } + for (reg, idxs) in vals.iter().zip(IDX.iter()) { + for (&val, &idx) in reg.iter().zip(idxs.iter()) { + block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { + (state[8] as u64) | ((state[9] as u64) << 32) +} + +#[inline(always)] +fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { + state[8] = pos as u32; + state[9] = (pos >> 32) as u32; +} diff --git a/salsa20/src/backends/soft.rs b/salsa20/src/backends/soft.rs index caf2693f..66b52b36 100644 --- a/salsa20/src/backends/soft.rs +++ b/salsa20/src/backends/soft.rs @@ -46,7 +46,7 @@ pub(crate) fn quarter_round( } #[inline(always)] -fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { +pub(crate) fn run_rounds(state: &[u32; STATE_WORDS]) -> [u32; STATE_WORDS] { let mut res = *state; for _ in 0..R::USIZE { diff --git a/salsa20/src/backends/sse2.rs b/salsa20/src/backends/sse2.rs new file mode 100644 index 00000000..2ea9df98 --- /dev/null +++ b/salsa20/src/backends/sse2.rs @@ -0,0 +1,169 @@ +#![allow(unsafe_op_in_unsafe_fn)] +#![allow(clippy::cast_possible_wrap)] + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use crate::{STATE_WORDS, Unsigned}; +use cipher::{ + BlockSizeUser, ParBlocksSizeUser, StreamCipherBackend, StreamCipherClosure, + consts::{U4, U64}, +}; +use core::marker::PhantomData; + +const PAR_BLOCKS: usize = 4; + +/// Diagonal indices for the Salsa20 state matrix. +/// +/// The 4x4 state is packed into 4 SSE registers so that each register +/// holds one diagonal. All four column quarter-rounds then execute +/// as a single SIMD operation, and shuffles rotate between +/// column and row round alignment. +const IDX: [[usize; 4]; 4] = [[0, 5, 10, 15], [4, 9, 14, 3], [8, 13, 2, 7], [12, 1, 6, 11]]; + +macro_rules! rotl_epi32 { + ($v:expr, $n:literal) => {{ + let v = $v; + _mm_or_si128(_mm_slli_epi32(v, $n), _mm_srli_epi32(v, 32 - $n)) + }}; +} + +#[inline] +#[target_feature(enable = "sse2")] +pub(crate) unsafe fn inner(state: &mut [u32; STATE_WORDS], f: F) +where + R: Unsigned, + F: StreamCipherClosure, +{ + f.call(&mut Backend:: { + state, + _pd: PhantomData, + }); +} + +struct Backend<'a, R: Unsigned> { + state: &'a mut [u32; STATE_WORDS], + _pd: PhantomData, +} + +impl BlockSizeUser for Backend<'_, R> { + type BlockSize = U64; +} + +impl ParBlocksSizeUser for Backend<'_, R> { + type ParBlocksSize = U4; +} + +impl StreamCipherBackend for Backend<'_, R> { + #[inline(always)] + fn gen_ks_block(&mut self, block: &mut cipher::Block) { + unsafe { + let res = rounds::(self.state); + store_block(&res[0], block); + let counter = get_counter(self.state); + set_counter(self.state, counter + 1); + } + } + + #[inline(always)] + fn gen_par_ks_blocks(&mut self, blocks: &mut cipher::ParBlocks) { + unsafe { + let res = rounds::(self.state); + for i in 0..PAR_BLOCKS { + store_block(&res[i], &mut blocks[i]); + } + let counter = get_counter(self.state); + set_counter(self.state, counter + PAR_BLOCKS as u64); + } + } +} + +#[inline(always)] +unsafe fn load_diag(state: &[u32; STATE_WORDS]) -> [__m128i; 4] { + let s = |i: usize| state[i] as i32; + [ + _mm_set_epi32(s(IDX[0][3]), s(IDX[0][2]), s(IDX[0][1]), s(IDX[0][0])), + _mm_set_epi32(s(IDX[1][3]), s(IDX[1][2]), s(IDX[1][1]), s(IDX[1][0])), + _mm_set_epi32(s(IDX[2][3]), s(IDX[2][2]), s(IDX[2][1]), s(IDX[2][0])), + _mm_set_epi32(s(IDX[3][3]), s(IDX[3][2]), s(IDX[3][1]), s(IDX[3][0])), + ] +} + +#[inline] +#[target_feature(enable = "sse2")] +unsafe fn rounds(state: &[u32; STATE_WORDS]) -> [[__m128i; 4]; PAR_BLOCKS] { + let counter = get_counter(state); + let mut tmp = *state; + let mut res = [[_mm_setzero_si128(); 4]; PAR_BLOCKS]; + for (i, block) in res.iter_mut().enumerate() { + let ctr = counter + i as u64; + tmp[8] = ctr as u32; + tmp[9] = (ctr >> 32) as u32; + *block = load_diag(&tmp); + } + let initial = res; + + for _ in 0..R::USIZE * 2 { + for block in res.iter_mut() { + quarter_round(block); + shuffle(block); + } + } + + for i in 0..PAR_BLOCKS { + for j in 0..4 { + res[i][j] = _mm_add_epi32(res[i][j], initial[i][j]); + } + } + + res +} + +#[inline(always)] +unsafe fn quarter_round(block: &mut [__m128i; 4]) { + let [a, b, c, d] = block; + *b = _mm_xor_si128(*b, rotl_epi32!(_mm_add_epi32(*a, *d), 7)); + *c = _mm_xor_si128(*c, rotl_epi32!(_mm_add_epi32(*b, *a), 9)); + *d = _mm_xor_si128(*d, rotl_epi32!(_mm_add_epi32(*c, *b), 13)); + *a = _mm_xor_si128(*a, rotl_epi32!(_mm_add_epi32(*d, *c), 18)); +} + +/// Rotate diagonal alignment between column and row rounds. +/// +/// The same shuffle+swap sequence converts column layout to row layout +/// and vice versa, so it is applied identically after every quarter round. +#[inline(always)] +unsafe fn shuffle(block: &mut [__m128i; 4]) { + let [_, b, c, d] = block; + let tmp = *b; + *b = _mm_shuffle_epi32(*d, 0b_00_11_10_01); + *c = _mm_shuffle_epi32(*c, 0b_01_00_11_10); + *d = _mm_shuffle_epi32(tmp, 0b_10_01_00_11); +} + +/// Convert diagonal SIMD layout back to sequential little-endian bytes. +#[inline(always)] +unsafe fn store_block(diag: &[__m128i; 4], block: &mut [u8]) { + let mut vals = [[0u32; 4]; 4]; + for (i, v) in diag.iter().enumerate() { + _mm_storeu_si128(vals[i].as_mut_ptr().cast(), *v); + } + for (reg, idxs) in vals.iter().zip(IDX.iter()) { + for (&val, &idx) in reg.iter().zip(idxs.iter()) { + block[idx * 4..idx * 4 + 4].copy_from_slice(&val.to_le_bytes()); + } + } +} + +#[inline(always)] +fn get_counter(state: &[u32; STATE_WORDS]) -> u64 { + (state[8] as u64) | ((state[9] as u64) << 32) +} + +#[inline(always)] +fn set_counter(state: &mut [u32; STATE_WORDS], pos: u64) { + state[8] = pos as u32; + state[9] = (pos >> 32) as u32; +} diff --git a/salsa20/src/lib.rs b/salsa20/src/lib.rs index e794b593..e6e966e2 100644 --- a/salsa20/src/lib.rs +++ b/salsa20/src/lib.rs @@ -23,6 +23,16 @@ use cipher::zeroize::{Zeroize, ZeroizeOnDrop}; mod backends; mod xsalsa; +cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + cpufeatures::new!(avx2_cpuid, "avx2"); + cpufeatures::new!(sse2_cpuid, "sse2"); + type Tokens = (avx2_cpuid::InitToken, sse2_cpuid::InitToken); + } else { + type Tokens = (); + } +} + pub use xsalsa::{XSalsa8, XSalsa12, XSalsa20, XSalsaCore, hsalsa}; /// Salsa20/8 stream cipher @@ -56,6 +66,9 @@ const CONSTANTS: [u32; 4] = [0x6170_7865, 0x3320_646e, 0x7962_2d32, 0x6b20_6574] pub struct SalsaCore { /// Internal state of the core function state: [u32; STATE_WORDS], + /// CPU target feature tokens + #[allow(dead_code)] + tokens: Tokens, /// Number of rounds to perform rounds: PhantomData, } @@ -66,8 +79,16 @@ impl SalsaCore { /// This method is mainly intended for the `scrypt` crate. /// Other users generally should not use this method. pub fn from_raw_state(state: [u32; STATE_WORDS]) -> Self { + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); + } else { + let tokens = (); + } + } Self { state, + tokens, rounds: PhantomData, } } @@ -110,8 +131,17 @@ impl KeyIvInit for SalsaCore { state[15] = CONSTANTS[3]; + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let tokens = (avx2_cpuid::init(), sse2_cpuid::init()); + } else { + let tokens = (); + } + } + Self { state, + tokens, rounds: PhantomData, } } @@ -124,7 +154,24 @@ impl StreamCipherCore for SalsaCore { rem.try_into().ok() } fn process_with_backend(&mut self, f: impl StreamCipherClosure) { - f.call(&mut backends::soft::Backend(self)); + cfg_if::cfg_if! { + if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] { + let (avx2_token, sse2_token) = self.tokens; + if avx2_token.get() { + unsafe { + backends::avx2::inner::(&mut self.state, f); + } + } else if sse2_token.get() { + unsafe { + backends::sse2::inner::(&mut self.state, f); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } else { + f.call(&mut backends::soft::Backend(self)); + } + } } } diff --git a/salsa20/src/xsalsa.rs b/salsa20/src/xsalsa.rs index 9f84cef5..ae1482fc 100644 --- a/salsa20/src/xsalsa.rs +++ b/salsa20/src/xsalsa.rs @@ -8,8 +8,6 @@ use cipher::{ consts::{U4, U6, U10, U16, U24, U32, U64}, }; -use crate::backends::soft::quarter_round; - #[cfg(feature = "zeroize")] use cipher::zeroize::ZeroizeOnDrop; @@ -113,27 +111,24 @@ pub fn hsalsa(key: &Key, input: &Array) -> Array .for_each(|(v, chunk)| *v = to_u32(chunk)); state[15] = CONSTANTS[3]; - // 20 rounds consisting of 10 column rounds and 10 diagonal rounds + use crate::backends::soft::quarter_round; for _ in 0..R::USIZE { - // column rounds quarter_round(0, 4, 8, 12, &mut state); quarter_round(5, 9, 13, 1, &mut state); quarter_round(10, 14, 2, 6, &mut state); quarter_round(15, 3, 7, 11, &mut state); - - // diagonal rounds quarter_round(0, 1, 2, 3, &mut state); quarter_round(5, 6, 7, 4, &mut state); quarter_round(10, 11, 8, 9, &mut state); quarter_round(15, 12, 13, 14, &mut state); } - let mut output = Array::default(); let key_idx: [usize; 8] = [0, 5, 10, 15, 6, 7, 8, 9]; + let words: [u32; 8] = core::array::from_fn(|i| state[key_idx[i]]); + let mut output = Array::default(); for (i, chunk) in output.chunks_exact_mut(4).enumerate() { - chunk.copy_from_slice(&state[key_idx[i]].to_le_bytes()); + chunk.copy_from_slice(&words[i].to_le_bytes()); } - output }