diff --git a/README.md b/README.md index 0e01bdb..4a28768 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,46 @@ # A -![rust workflow](https://github.com/nettrash/a/actions/workflows/rust.yml/badge.svg) +[![rust workflow](https://github.com/nettrash/a/actions/workflows/rust.yml/badge.svg)](https://github.com/nettrash/a/actions/workflows/rust.yml) +[![crates.io](https://img.shields.io/crates/v/nettrash-a.svg)](https://crates.io/crates/nettrash-a) +[![docs.rs](https://img.shields.io/docsrs/nettrash-a)](https://docs.rs/nettrash-a) +[![license](https://img.shields.io/crates/l/nettrash-a.svg)](https://github.com/nettrash/a/blob/HEAD/LICENSE) -The A Library is a set of algorithms. +Algorithms in pure Rust, with no dependencies: + +- **Sorting** (`a::sort`): 24 sorts by comparison and by key, a default one among them. +- **Integers of any size** (`a::math::numbers`): `Number`, with its arithmetic up to transforms for the longest products, roots, powers modulo a number, greatest common divisors, primes and text in any radix. +- **Elliptic curves** (`a::math::ec`): Weierstrass, Montgomery and Edwards curves over prime fields, secp256k1, P-256, P-384, P-521, Curve25519 and Ed25519 among them. +- **Complex numbers** (`a::math::complex`) and **matrices** (`a::math::matrix`): decompositions, eigenvalues, singular values and functions of matrices of floats and complex numbers, and exact algorithms on matrices of integers. + +Every algorithm has a page with its method, examples and measurements against the reference implementations (GMP, OpenSSL, faer, nalgebra and others); the tables below link to them. The API reference is on [docs.rs](https://docs.rs/nettrash-a). The crate needs Rust 1.88 (edition 2024). Its fastest paths are NEON and assembly kernels on aarch64, and assembly for the products of integers on x86_64; everything else is portable Rust. ## Usage Add the crate with `cargo add nettrash-a`; its library is named `a`: ```rust +use a::math::matrix::Matrix; +use a::math::numbers::Number; use a::sort::sort; fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - sort(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; + sort(&mut v); + assert_eq!(v, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + + let x: Number = "123456789012345678901234567890".parse().unwrap(); + assert_eq!((&x * &x).to_string(), "15241578753238836750495351562536198787501905199875019052100"); + + let a: Matrix = Matrix::from_rows(&[[4.0, 2.0], [2.0, 3.0]]); + let b = Matrix::from_rows(&[[2.0], [1.0]]); + let solution = a.solve(&b).unwrap(); + assert!((solution[(0, 0)] - 0.5).abs() < 1e-15 && solution[(1, 0)].abs() < 1e-15); } ``` -Every algorithm has a page with its description, examples and measurements; the tables below link to them. The API reference is on [docs.rs](https://docs.rs/nettrash-a). - ## Sorting -Every sort takes a mutable slice and leaves it sorted; the extra memory is what a sort uses besides the slice. [`sort`](https://github.com/nettrash/a/blob/main/lib/docs/sorting/default.md) is the default: use it when there is no reason to prefer a particular algorithm. +Every sort takes a mutable slice and leaves it sorted; the extra memory is what a sort uses besides the slice. [`sort`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/default.md) is the default: use it when there is no reason to prefer a particular algorithm. ### Comparison sorts @@ -30,25 +48,25 @@ These sort any `T: Ord`. | Sort | Functions | Stable | Best | Average | Worst | Extra memory | | --- | --- | --- | --- | --- | --- | --- | -| [Default](https://github.com/nettrash/a/blob/main/lib/docs/sorting/default.md) | `sort` | yes | O(n) | O(n log n) | O(n log n) | O(n) | -| [Bubble](https://github.com/nettrash/a/blob/main/lib/docs/sorting/bubble.md) | `bubble` | yes | O(n) | O(n²) | O(n²) | O(1) | -| [Selection](https://github.com/nettrash/a/blob/main/lib/docs/sorting/selection.md) | `selection` | no | O(n) | O(n²) | O(n²) | O(1) | -| [Insertion](https://github.com/nettrash/a/blob/main/lib/docs/sorting/insertion.md) | `insertion` | yes | O(n) | O(n²) | O(n²) | O(1) | -| [Cocktail Shaker](https://github.com/nettrash/a/blob/main/lib/docs/sorting/cocktail_shaker.md) | `cocktail_shaker` | yes | O(n) | O(n²) | O(n²) | O(1) | -| [Gnome](https://github.com/nettrash/a/blob/main/lib/docs/sorting/gnome.md) | `gnome` | yes | O(n) | O(n²) | O(n²) | O(1) | -| [Comb](https://github.com/nettrash/a/blob/main/lib/docs/sorting/comb.md) | `comb` | no | O(n log n) | O(n log n)¹ | O(n²)¹ | O(1) | -| [Merge](https://github.com/nettrash/a/blob/main/lib/docs/sorting/merge.md) | `merge` | yes | O(n) | O(n log n) | O(n log n) | O(n) | -| [Quick](https://github.com/nettrash/a/blob/main/lib/docs/sorting/quick.md) | `quick` | no | O(n) | O(n log n) | O(n log n) | O(log n) | -| [Heap](https://github.com/nettrash/a/blob/main/lib/docs/sorting/heap.md) | `heap` | no | O(n) | O(n log n) | O(n log n) | O(1) | -| [Tim](https://github.com/nettrash/a/blob/main/lib/docs/sorting/tim.md) | `tim` | yes | O(n) | O(n log n) | O(n log n) | O(n) | -| [Intro](https://github.com/nettrash/a/blob/main/lib/docs/sorting/intro.md) | `intro` | no | O(n) | O(n log n) | O(n log n) | O(log n) | -| [Shell](https://github.com/nettrash/a/blob/main/lib/docs/sorting/shell.md) | `shell` | no | O(n) | unknown² | O(n^(4/3))² | O(1) | -| [Drift](https://github.com/nettrash/a/blob/main/lib/docs/sorting/drift.md) | `drift` | yes | O(n) | O(n log n) | O(n log n) | O(n) | -| [Tree](https://github.com/nettrash/a/blob/main/lib/docs/sorting/tree.md) | `tree` | yes | O(n) | O(n log n) | O(n log n) | O(n) | -| [Cycle](https://github.com/nettrash/a/blob/main/lib/docs/sorting/cycle.md) | `cycle` | no | O(n) | O(n²) | O(n²) | O(1) | -| [Bitonic](https://github.com/nettrash/a/blob/main/lib/docs/sorting/bitonic.md) | `bitonic`, `bitonic_parallel` | no | O(n log² n) | O(n log² n) | O(n log² n) | O(log n) | -| [Odd-Even](https://github.com/nettrash/a/blob/main/lib/docs/sorting/odd_even.md) | `odd_even`, `odd_even_parallel` | yes | O(n) | O(n²) | O(n²) | O(1) | -| [Stooge](https://github.com/nettrash/a/blob/main/lib/docs/sorting/stooge.md) | `stooge` | no | O(n^2.71) | O(n^2.71) | O(n^2.71) | O(log n) | +| [Default](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/default.md) | `sort` | yes | O(n) | O(n log n) | O(n log n) | O(n) | +| [Bubble](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/bubble.md) | `bubble` | yes | O(n) | O(n²) | O(n²) | O(1) | +| [Selection](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/selection.md) | `selection` | no | O(n) | O(n²) | O(n²) | O(1) | +| [Insertion](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/insertion.md) | `insertion` | yes | O(n) | O(n²) | O(n²) | O(1) | +| [Cocktail Shaker](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/cocktail_shaker.md) | `cocktail_shaker` | yes | O(n) | O(n²) | O(n²) | O(1) | +| [Gnome](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/gnome.md) | `gnome` | yes | O(n) | O(n²) | O(n²) | O(1) | +| [Comb](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/comb.md) | `comb` | no | O(n log n) | O(n log n)¹ | O(n²)¹ | O(1) | +| [Merge](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/merge.md) | `merge` | yes | O(n) | O(n log n) | O(n log n) | O(n) | +| [Quick](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/quick.md) | `quick` | no | O(n) | O(n log n) | O(n log n) | O(log n) | +| [Heap](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/heap.md) | `heap` | no | O(n) | O(n log n) | O(n log n) | O(1) | +| [Tim](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/tim.md) | `tim` | yes | O(n) | O(n log n) | O(n log n) | O(n) | +| [Intro](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/intro.md) | `intro` | no | O(n) | O(n log n) | O(n log n) | O(log n) | +| [Shell](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/shell.md) | `shell` | no | O(n) | unknown² | O(n^(4/3))² | O(1) | +| [Drift](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/drift.md) | `drift` | yes | O(n) | O(n log n) | O(n log n) | O(n) | +| [Tree](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/tree.md) | `tree` | yes | O(n) | O(n log n) | O(n log n) | O(n) | +| [Cycle](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/cycle.md) | `cycle` | no | O(n) | O(n²) | O(n²) | O(1) | +| [Bitonic](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/bitonic.md) | `bitonic`, `bitonic_parallel` | no | O(n log² n) | O(n log² n) | O(n log² n) | O(log n) | +| [Odd-Even](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/odd_even.md) | `odd_even`, `odd_even_parallel` | yes | O(n) | O(n²) | O(n²) | O(1) | +| [Stooge](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/stooge.md) | `stooge` | no | O(n^2.71) | O(n^2.71) | O(n^2.71) | O(log n) | ¹ About n·log₁.₃ n comparisons in practice; no O(n log n) bound has been proved, and no known input reaches O(n²). @@ -60,31 +78,152 @@ These sort by integer or float keys without comparing elements. `k` is the numbe | Sort | Functions | Keys | Stable | Time | Extra memory | | --- | --- | --- | --- | --- | --- | -| [Counting](https://github.com/nettrash/a/blob/main/lib/docs/sorting/counting.md) | `counting`, `counting_by_key` | integer | `counting_by_key` | O(n + k) | O(k), O(n + k) by key | -| [Radix](https://github.com/nettrash/a/blob/main/lib/docs/sorting/radix.md) | `radix`, `radix_by_key` | integer | `radix_by_key` | O(n·b/r) for keys that differ in `b` bits, digits of `r` ≈ log₂ n bits | O(n) | -| [Bucket](https://github.com/nettrash/a/blob/main/lib/docs/sorting/bucket.md) | `bucket`, `bucket_by_key` | integer, `f32`, `f64` | `bucket_by_key` | O(n) for keys spread evenly, O(n·w) for `w`-bit keys at worst | O(n) | -| [Pigeonhole](https://github.com/nettrash/a/blob/main/lib/docs/sorting/pigeonhole.md) | `pigeonhole`, `pigeonhole_by_key` | integer | no | O(n + k) | O(k), O(n + k) by key | -| [Sleep](https://github.com/nettrash/a/blob/main/lib/docs/sorting/sleep.md) | `sleep` | integer | — | (max − min) × `unit` of sleeping | a thread per element | +| [Counting](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/counting.md) | `counting`, `counting_by_key` | integer | `counting_by_key` | O(n + k) | O(k), O(n + k) by key | +| [Radix](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/radix.md) | `radix`, `radix_by_key` | integer | `radix_by_key` | O(n·b/r) for keys that differ in `b` bits, digits of `r` ≈ log₂ n bits | O(n) | +| [Bucket](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/bucket.md) | `bucket`, `bucket_by_key` | integer, `f32`, `f64` | `bucket_by_key` | O(n) for keys spread evenly, O(n·w) for `w`-bit keys at worst | O(n) | +| [Pigeonhole](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/pigeonhole.md) | `pigeonhole`, `pigeonhole_by_key` | integer | no | O(n + k) | O(k), O(n + k) by key | +| [Sleep](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/sleep.md) | `sleep` | integer | — | (max − min) × `unit` of sleeping | a thread per element | Sleep sort is a joke: a thread per element sleeps in proportion to its value, the result is not always sorted, and `sleep` returns whether it is. +### Against the standard library + +Measured on an Apple M4 Max with new input for every sort; each page has the other inputs, sizes and element types: + +| Sort | Against | Speed | +| --- | --- | --- | +| [`sort`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/default.md) | `slice::sort` | 1.3–3 times faster on 8–64 random integers, 1.03–1.06 times slower from 100 to 100,000 and as fast beyond; 1.7 times faster on a million random strings | +| [`merge`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/merge.md) | `slice::sort` | 1.6–2.4 times faster on up to 64 random integers, 1.08–1.23 times slower from 1000; 3–6 times slower with few distinct keys | +| [`tim`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/tim.md) | `slice::sort` | 1.2–1.45 times slower on random integers from 1000, 1.5–3.7 times faster on nearly sorted input | +| [`quick`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/quick.md) | `slice::sort_unstable` | 1.1–1.2 times slower on random integers from 100 | +| [`intro`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/intro.md) | `slice::sort_unstable` | 1.45–1.75 times slower on random integers | +| [`radix`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/radix.md) | `slice::sort_unstable` | 1.8–5 times faster on random `u32`s from 1000, up to 2.3 times on random `u64`s | +| [`counting`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/counting.md) | `slice::sort_unstable` | 3.2–10 times faster on random `u32`s and `u64`s with about as many distinct keys as elements, from 100 | +| [`bucket`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/bucket.md) | `slice::sort_unstable` | 1.3–3 times faster on random integers and 1.7–3.7 times on uniform floats, from 100 | +| [`pigeonhole`](https://github.com/nettrash/a/blob/HEAD/lib/docs/sorting/pigeonhole.md) | `slice::sort_unstable` | 1.15–2.2 times faster on random `u32` keys in a range of about `n`, from 1000 | + ## Math +### Integers of any size + +`a::math::numbers::Number` is an integer of any size, its sign and the bytes of its absolute value in either byte order. Operators take owned or borrowed operands and build the result in an operand's vector when it has room. + +| Page | Contents | +| --- | --- | +| [Numbers](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/numbers.md) | `Number`: how it is stored, built and read | +| [Traits and conversions](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/traits.md) | Comparison and hashing by value; conversions from and to the primitive integers, to `f64` and `f32`, and from strings; `Sum` and `Product`; arithmetic with primitive integers | +| [Addition](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/add.md) | `+` and `+=` | +| [Subtraction](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/sub.md) | `-` and `-=` | +| [Multiplication](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/mul.md) | `*` and `*=`: schoolbook, Karatsuba, Toom-3, Toom-4 and number-theoretic transforms, with assembly kernels on aarch64 and x86_64 | +| [Division](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/div.md) | `/`, `%`, `div_rem`, Euclid's division, a prepared `Divisor` and exact division: Knuth's method, divide and conquer, and Newton's method for the longest | +| [Shifts](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/shift.md) | `<<` and `>>` by any number of bits, rounding as `i32` does | +| [Roots](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/root.md) | `isqrt`, `isqrt_rem` and `nth_root`: Zimmermann's Karatsuba square root and Newton's method | +| [Factorials](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/factorial.md) | `factorial` and `binomial`, from prime factorisations by balanced product trees | +| [Bitwise operations](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/bits.md) | `&`, `\|`, `^` and `!` in two's complement; `bits`, `count_ones`, `trailing_zeros`, `bit` and `set_bit` | +| [Text](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/radix.md) | `Display`, hexadecimal, octal and binary formatting; `to_str_radix` and `from_str_radix` in any radix from 2 to 36 | +| [Greatest common divisor](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/gcd.md) | `gcd`, `lcm`, `gcd_ext` and `mod_inverse`: Lehmer's method and the subquadratic half gcd | +| [Powers](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/pow.md) | `pow`, and `pow_mod` by Montgomery's multiplication | +| [Primes](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/prime.md) | `is_probable_prime` by Baillie–PSW, exact below 2⁶⁴, and `next_prime` | +| [Squares modulo a prime](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/quadratic.md) | `jacobi`, and `sqrt_mod` by Tonelli and Shanks's method | + +Measured on an Apple M4 Max in cycles per call, over random operands of the length given, little-endian, against GMP 6.3 and num-bigint 0.4; each page has the conditions, more sizes and the other operations. GMP is faster on some short operands; "–" is not measured: + +| Operation | Size | `a` | GMP | num-bigint | +| --- | --- | ---: | ---: | ---: | +| `a + b` | 64 B | 25 | 37 | – | +| | 4 KiB | 469 | 585 | – | +| `a * b` | 64 B | 109 | 107 | 217 | +| | 1 KiB | 9,545 | 11,261 | 29,665 | +| | 64 KiB | 1.12 M | 2.85 M | 17.1 M | +| | 1 MiB | 24.5 M | 86.3 M | 1,058 M | +| `a / b` | 64 B / 32 B | 197 | 174 | 738 | +| | 1 KiB / 512 B | 4,401 | 5,148 | 22,811 | +| | 64 KiB / 32 KiB | 1.30 M | 2.82 M | 15.0 M | +| `isqrt` | 64 B | 426 | 335 | 2,798 | +| | 1 KiB | 4,770 | 4,615 | 101,731 | +| | 64 KiB | 1.46 M | 2.31 M | 136 M | +| `gcd` | 64 B | 5,138 | 6,853 | 26,954 | +| | 1 KiB | 117,118 | 163,209 | 1.15 M | +| | 64 KiB | 55.5 M | 64.4 M | 4,013 M | +| `pow_mod`, the exponent as long as the modulus | 64 B | 119,000 | 128,779 | 271,225 | +| | 256 B | 4.30 M | 5.13 M | 9.09 M | +| | 1 KiB | 233 M | 239 M | 683 M | +| `to_str_radix(10)` | 128 B | 983 | 2,453 | 4,417 | +| | 8 KiB | 504,112 | 607,962 | 2.33 M | +| | 64 KiB | 7.29 M | 12.6 M | 50.3 M | +| `is_probable_prime` of a prime | 256 bits | 68,457 | 159,951 | – | +| | 2048 bits | 14.9 M | 20.9 M | – | + +### Elliptic curves + +| Page | Contents | +| --- | --- | +| [Elliptic curves](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/ec.md) | `a::math::ec`: short Weierstrass, Montgomery and twisted Edwards curves over prime fields of any length; `+`, `-`, `*`, compression, `multi_mul`, prepared points and maps between the forms; secp256k1, P-256, P-384, P-521, Curve25519 and Ed25519 ready made | + +Measured on an Apple M4 Max in cycles, against OpenSSL 3.6 and curve25519-dalek 4.1; `k·P` multiplies a point, `k·G` the generator, by a table where the library has one. OpenSSL's and dalek's products of a point are constant-time, which costs them time this crate doesn't spend: + +| Curve | Against | `k·P` | theirs | `k·G` | theirs | +| --- | --- | ---: | ---: | ---: | ---: | +| secp256k1 | OpenSSL | 40,100 | 555,000 | 7,250 | 559,000 | +| P-256 | OpenSSL | 82,600 | 90,900 | 10,600 | 14,300 | +| P-384 | OpenSSL | 311,000 | 548,000 | 40,200 | 267,000 | +| P-521 | OpenSSL | 358,000 | 496,000 | 52,500 | 269,000 | +| Ed25519 | dalek | 45,300 | 72,000 | 5,350 | 22,100 | +| Curve25519, X25519 | dalek | 55,400 | 78,500 | | | + +### Complex numbers + | Page | Contents | | --- | --- | -| [Numbers](https://github.com/nettrash/a/blob/main/lib/docs/math/numbers.md) | `a::math::numbers::Number`: an integer of any size, stored as its sign and the bytes of its absolute value in either byte order | -| [Traits and conversions](https://github.com/nettrash/a/blob/main/lib/docs/math/traits.md) | `==`, `<`, `Ord` and `Hash` by value whatever the byte order, also with the primitive integers; `Clone`, `Default`, `Debug`, `-a`, `abs`, `signum`; `From` and `TryFrom` the primitive integers, `to_f64` and `to_f32` correctly rounded, `str::parse`; `Sum` and `Product` (as a balanced tree); `+ - * / %` with a primitive integer on either side, in place in one pass | -| [Addition](https://github.com/nettrash/a/blob/main/lib/docs/math/add.md) | `+` and `+=` for `Number`, with owned or borrowed operands: the exact sum, in the byte order of the left operand, built in an operand's vector when that has room | -| [Subtraction](https://github.com/nettrash/a/blob/main/lib/docs/math/sub.md) | `-` and `-=` for `Number`, with owned or borrowed operands: the exact difference, computed as the sum with the right operand's sign turned over, with the same promises and kernels | -| [Multiplication](https://github.com/nettrash/a/blob/main/lib/docs/math/mul.md) | `*` and `*=` for `Number`, with owned or borrowed operands: the exact product, by the schoolbook method, Karatsuba's from 24 limbs, Toom-3 from 120, Toom-4 from 224 (Toom-4×2 for unbalanced operands) and number-theoretic transforms modulo up to eight primes from 360, with assembly kernels on aarch64 and x86_64 and NEON transforms on aarch64; `&a * &a` squares | -| [Division](https://github.com/nettrash/a/blob/main/lib/docs/math/div.md) | `/`, `/=`, `%` and `%=` for `Number`, owned or borrowed, as for `i32`: the quotient truncated toward zero, the remainder with the dividend's sign; `div_rem`, `div_euclid` and `rem_euclid`; a prepared `Divisor` for many divisions by one number; `div_exact` by Hensel's method and `is_multiple_of`. Knuth's method with Möller–Granlund reciprocals, divide and conquer from 40 limbs, and for the quotient alone a shrinking divisor for its last limbs; Newton's method for the longest, from 128 limbs of divisor and 768 of quotient (at least an eighth of the divisor), an inverse by Newton's iteration and the quotient in blocks, with products modulo `B^m − 1` by the transforms | -| [Shifts](https://github.com/nettrash/a/blob/main/lib/docs/math/shift.md) | `<<`, `>>`, `<<=` and `>>=` for `Number` by a `usize` number of bits, owned or borrowed: `a · 2^k` and `a / 2^k` rounded towards negative infinity, as for `i32`, in one pass over the bytes, built in an owned operand's vector when that has room | -| [Roots](https://github.com/nettrash/a/blob/main/lib/docs/math/root.md) | `isqrt`, `isqrt_rem` and `nth_root` for `Number`: the square root rounded down and its remainder, and the root of any degree truncated toward zero, by Zimmermann's Karatsuba square root and Newton's method at doubling precision, in the time of a product | -| [Factorials](https://github.com/nettrash/a/blob/main/lib/docs/math/factorial.md) | `Number::factorial` and `Number::binomial`: `n!` from the prime factors of swinging factorials (Luschny), binomial coefficients as exact quotients of products or from their prime factors (Kummer, Goetgheluck), by balanced product trees | -| [Bitwise operations](https://github.com/nettrash/a/blob/main/lib/docs/math/bits.md) | `&`, `\|`, `^`, their assignments and `!` for `Number`, owned or borrowed, in infinite two's complement as for `i32`, in one pass with the borrows and carry of negative operands fused in, NEON kernels on aarch64; `bits`, `count_ones`, `trailing_zeros`, `bit` and `set_bit` | -| [Text](https://github.com/nettrash/a/blob/main/lib/docs/math/radix.md) | `Display`, `LowerHex`, `UpperHex`, `Octal` and `Binary` for `Number`, and `to_str_radix` and `from_str_radix` in any radix from 2 to 36: the sign and the digits of the magnitude, by divide and conquer for long numbers | -| [Greatest common divisor](https://github.com/nettrash/a/blob/main/lib/docs/math/gcd.md) | `gcd`, `lcm`, `gcd_ext` and `mod_inverse` for `Number`: the greatest common divisor, the least common multiple, GMP's cofactors and the inverse modulo a number, by Lehmer's method with Möller's hgcd2 and, from 300 limbs for the cofactors and 800 for the divisor alone, Möller's subquadratic half gcd on the multiplication | -| [Powers](https://github.com/nettrash/a/blob/main/lib/docs/math/pow.md) | `pow` for `Number` by a `u32` exponent, exactly, as for `i32`, and `pow_mod`, a power by a `Number` exponent modulo a `Number`, Euclid's remainder: Montgomery's multiplication with a sliding window, REDC two limbs a pass and by products from 100 limbs, in registers up to seven limbs; even moduli by the Chinese remainder theorem; not constant-time | -| [Primes](https://github.com/nettrash/a/blob/main/lib/docs/math/prime.md) | `is_probable_prime` and `next_prime` for `Number`: trial division by the small primes, then Baillie–PSW, a strong test to base 2 and a strong Lucas test with Selfridge's parameters in Montgomery's form; exact below 2^64, no known counterexample above; the next prime by sieving intervals, as GMP's `mpz_nextprime` | -| [Squares modulo a prime](https://github.com/nettrash/a/blob/main/lib/docs/math/quadratic.md) | `jacobi` and `sqrt_mod` for `Number`: the Jacobi symbol, by the binary method, and a square root modulo an odd prime, by Tonelli and Shanks's method | -| [Elliptic curves](https://github.com/nettrash/a/blob/main/lib/docs/math/ec.md) | `a::math::ec`: short Weierstrass, Montgomery and twisted Edwards curves over prime fields of any length, on `Number`; `+`, `-`, `*`, compression, `multi_mul` by Straus's and Pippenger's methods, prepared points, invariants and maps between the forms; secp256k1, P-256, P-384, P-521, Curve25519 and Ed25519 ready made. Not constant-time, so not for secret scalars | +| [Complex numbers](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/complex.md) | `a::math::complex`: `Complex` and `Complex`, their arithmetic, `abs`, `arg`, `exp`, `ln`, `sqrt` and powers, with C99's branch cuts; division and square roots that don't overflow before their results do | + +Measured on an Apple M4 Max for `Complex`, in nanoseconds per operation over 1,024 independent ones, against num-complex 0.4.6, whose division by `|b|²` as it is overflows from `|b|` of about 10¹⁵⁴: + +| Operation | `Complex` | num-complex | +| --- | ---: | ---: | +| `a * b` | 0.43 | 0.44 | +| `a / b` | 1.99 | 0.88 | +| `sqrt` | 4.41 | 9.12 | +| `exp` | 5.15 | 5.46 | +| `ln` | 7.54 | 7.40 | +| `powc` | 15.75 | 15.48 | + +### Matrices + +[`a::math::matrix::Matrix`](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/matrix.md) is a dense matrix of any size, column after column, of `f32`, `f64`, `Complex`, `Complex` or `Number`. The algorithms are LAPACK's, arranged so that most of their work is products of blocks, and the products run on NEON kernels; real matrices stay in real arithmetic. Costs are for `n × n` matrices. + +| Operations | Functions | Method | Cost | +| --- | --- | --- | --- | +| Arithmetic | `+`, `-`, `*`, by an element, `hadamard`, `transpose`, `conjugate_transpose`, row and column operations, `trace`, `pow`, `powi` | the product by Goto and van de Geijn's blocking | 2n³ for a product | +| LU | `lu`, `det`, `solve`, `inverse` | partial pivoting, recursive; the inverse as LAPACK's `getri` | 2n³/3, the inverse 2n³ | +| Cholesky | `cholesky` | recursive, blocked | n³/3 | +| QR | `qr`, `q`, `q_full`, least squares | Householder reflections in blocks, the panels halved recursively | 4n³/3 | +| Singular values | `svd`, `singular_values`, `rank`, `null_space`, `norm`, `cond` | a blocked reduction to bidiagonal form, then divide and conquer, or dqds for the values alone | 9.3n³, the values 8n³/3 | +| Hermitian eigenvalues | `hermitian_eigen`, `hermitian_eigenvalues` | a blocked reduction to tridiagonal form, then divide and conquer | 4.7n³, the values 4n³/3 | +| Eigenvalues | `schur`, `eigen`, `eigenvalues`, `characteristic_polynomial` | a blocked reduction to Hessenberg form, then the multishift QR algorithm with aggressive early deflation | about 25n³, the values 10n³ | +| Functions | `exp`, `sqrt`, `log` | scaling and squaring with Padé approximants; the Schur method; inverse scaling and squaring | `exp` a few products, `sqrt` and `log` a Schur decomposition | +| Exactly, on `Number` | `det`, `rank`, `characteristic_polynomial`, `adjugate`, `inverse`, `null_space` | Bareiss's elimination and Berkowitz's method, without fractions | n³ to 3n⁴/2 products of numbers | + +In `f64` on one core of an Apple M4 Max, each of the twelve operations [measured](https://github.com/nettrash/a/blob/HEAD/lib/docs/math/matrix.md#measurements) was at least as fast as faer's and nalgebra's at every size from 8 × 8 to 1024 × 1024; at 1024 × 1024, in milliseconds: + +| `n = 1024` | `a` | faer | nalgebra | +| --- | --- | --- | --- | +| `a * b` | 34.9 | 38.4 | 37.1 | +| `lu()` | 13.8 | 17.2 | 59.9 | +| `inverse()` | 40.1 | 50.2 | 221 | +| `cholesky()` | 7.4 | 8.5 | 27.8 | +| `qr()` | 29.0 | 32.5 | 90.7 | +| `svd()` | 244 | 331 | 5,886 | +| `hermitian_eigen()` | 114 | 146 | 396 | +| `eigen()` | 498 | 760 | – | +| `exp()` | 512 | – | 778 | + +Apple's Accelerate, which runs its products on the M4's matrix unit, is faster than all three at this size; it has no `exp`. + +## Not constant-time + +Nothing here is written to take the same time whatever its operands: `pow_mod` and the elliptic curves in particular branch on the values of their exponents and scalars. Use them on public data, for verification and research, not on secret keys or scalars. + +## License + +[MIT](https://github.com/nettrash/a/blob/HEAD/LICENSE). diff --git a/README.md.orig b/README.md.orig deleted file mode 100644 index a9ccac3..0000000 --- a/README.md.orig +++ /dev/null @@ -1,1429 +0,0 @@ -# A - -![rust workflow](https://github.com/nettrash/a/actions/workflows/rust.yml/badge.svg) - -The A Library is a set of algorithms. - -## Table of contents - -- [A](#a) - - [Table of contents](#table-of-contents) - - [Sorting](#sorting) - - [Bubble](#bubble) - - [Examples](#examples) - - [Efficiency](#efficiency) - - [Selection](#selection) - - [Examples](#examples-1) - - [Efficiency](#efficiency-1) - - [Insertion](#insertion) - - [Examples](#examples-2) - - [Efficiency](#efficiency-2) - - [Cocktail Shaker](#cocktail-shaker) - - [Examples](#examples-3) - - [Efficiency](#efficiency-3) - - [Gnome](#gnome) - - [Examples](#examples-4) - - [Efficiency](#efficiency-4) - - [Comb](#comb) - - [Examples](#examples-5) - - [Efficiency](#efficiency-5) - - [Merge](#merge) - - [Examples](#examples-6) - - [Efficiency](#efficiency-6) - - [Quick](#quick) - - [Examples](#examples-7) - - [Efficiency](#efficiency-7) - - [Heap](#heap) - - [Examples](#examples-8) - - [Efficiency](#efficiency-8) - - [Tim](#tim) - - [Examples](#examples-9) - - [Efficiency](#efficiency-9) - - [Intro](#intro) - - [Examples](#examples-10) - - [Efficiency](#efficiency-10) - - [Shell](#shell) - - [Examples](#examples-11) - - [Efficiency](#efficiency-11) - - [Drift](#drift) - - [Examples](#examples-12) - - [Efficiency](#efficiency-12) - - [Counting](#counting) - - [Examples](#examples-13) - - [Efficiency](#efficiency-13) - - [Radix](#radix) - - [Examples](#examples-14) - - [Efficiency](#efficiency-14) - - [Bucket](#bucket) - - [Examples](#examples-15) - - [Efficiency](#efficiency-15) - - [Pigeonhole](#pigeonhole) - - [Examples](#examples-16) - - [Efficiency](#efficiency-16) - - [Tree](#tree) - - [Examples](#examples-17) - - [Efficiency](#efficiency-17) - - [Cycle](#cycle) - - [Examples](#examples-18) - - [Efficiency](#efficiency-18) - - [Bitonic](#bitonic) - - [Examples](#examples-19) - - [Efficiency](#efficiency-19) - - [Odd-Even](#odd-even) - - [Examples](#examples-20) - - [Efficiency](#efficiency-20) - - [Stooge](#stooge) - - [Examples](#examples-21) - - [Efficiency](#efficiency-21) - - [Sleep](#sleep) - - [Examples](#examples-22) - - [Efficiency](#efficiency-22) - -## Sorting - -### Bubble - -Bubble sort is a simple, stable, in-place sorting algorithm. It repeatedly scans the slice, swapping adjacent elements that are out of order. This implementation uses these optimizations: - -- Early exit: stop when a pass performs no swaps (already sorted). -- Last-swap bound: the next pass only scans up to the last index that swapped (the tail is already sorted). -- Hole shifting: an element travelling `k` slots is read once, the `k` elements it overtakes shift one slot each, and it is written once: `k + 1` moves instead of the `3k` of `k` swaps. -- Branchless passes for small elements (at most 8 bytes, no drop glue): the running maximum is held in a register and every comparison writes the smaller element into the slot behind it, with no branch on the outcome, and two consecutive passes run at once, the second trailing the first, so that their comparisons overlap. Each pass picks branchy or branchless from what the previous one found, so sorted input still costs one branchy pass. The comparisons and their outcomes are exactly those of classic bubble sort; on random `u64` and `u32` input this is 1.2–2.9 times faster, and up to 3.3 times on elements that are each a few slots from their places. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::bubble; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - bubble(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -Sorting strings: - -```rust -use a::sort::bubble; - -fn main() { - let mut fruits = vec!["pear", "apple", "banana", "apple"]; - bubble(&mut fruits); - assert_eq!(fruits, vec!["apple", "apple", "banana", "pear"]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) - - Average: O(n^2) - - Best: O(n) with early-exit when input is already sorted -- Comparisons and swaps (tight bounds for the canonical variant): - - Worst comparisons: n(n−1)/2; worst swaps: n(n−1)/2 - - Best comparisons: n−1; best swaps: 0 -- Space complexity: O(1) extra (in-place) -- Stability: stable (equal elements keep their relative order) -- Adaptivity: good for nearly-sorted data due to early exit and last-swap bound; unsuitable for large, random inputs compared to O(n log n) sorts - -### Selection - -Selection sort is a simple, in-place, unstable sorting algorithm. Each pass selects the extremes of the unsorted part of the slice and moves them to their final positions. This implementation is double-ended and uses three optimizations: - -- Pairwise min-max scan: elements are taken two at a time, compared with each other, and only the smaller one is compared with the running minimum and the larger one with the running maximum, for `⌈3m/2⌉ − 2` comparisons per pass over `m` elements (the fewest possible for finding both extremes) instead of `2m − 2`. -- Duplicate gathering: every comparison is three-way, so all copies of the minimum and maximum keys are placed in the same pass at no extra comparisons; input with `k` distinct keys finishes in at most `⌈k/2⌉` passes, and a range found to be all equal ends the sort. -- Lazy placement: an extreme with no duplicates is only remembered by index and moved once the pass is over, so distinct keys cost at most `n − 1` swaps and already sorted input is never moved. - -The scan picks which element of a pair to compare with each extreme without a branch, since which one is smaller is a coin flip on random input; with the same comparisons, random input with distinct keys sorts 1.7–2.5 times faster from 1,000 elements up. For elements of 9 to 32 bytes (strings, for one), whose comparisons are slow to start, the first pass counts how predictable the pairs are and later passes branch when they are. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::selection; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - selection(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -Many duplicates (three distinct keys, sorted in two passes): - -```rust -use a::sort::selection; - -fn main() { - let mut v = vec![3, 1, 2, 3, 1, 2, 2, 1]; - selection(&mut v); - assert_eq!(v, vec![1, 1, 1, 2, 2, 2, 3, 3]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2), when all keys are distinct - - Average: O(n^2) for distinct keys; O(nk) with `k` distinct keys - - Best: O(n), when all elements are equal -- Comparisons and swaps: - - Worst comparisons: n(3n−2)/8 for even n, 3(n²−1)/8 for odd n (about 25% fewer than the n(n−1)/2 of classic selection sort); swaps with distinct keys: at most n−1 - - Best comparisons: n−1 (all equal); best swaps: 0 (already non-decreasing input) - - With repeated keys: at most 2m swaps per pass over m elements, trading the n−1 swap bound for far fewer comparisons and passes -- Space complexity: O(1) extra (in-place) -- Stability: unstable (equal elements may be reordered) -- Adaptivity: not adaptive to existing order (sorted input with distinct keys costs as much as any other), but adaptive to duplicates: the fewer distinct keys, the fewer passes; unsuitable for large inputs compared to O(n log n) sorts - -### Insertion - -Insertion sort is a simple, stable, in-place sorting algorithm. It grows a sorted prefix from the left, inserting each new element after every prefix element that is not greater than it. This implementation is a pair insertion sort (the variant Java's dual-pivot quicksort uses for short ranges) with three optimizations: - -- Sorted fast path: the smaller element of each new pair is first compared with the last element of the prefix; if it is not smaller, the pair is already in place, so sorted input costs exactly n−1 comparisons. -- Galloping search: an insertion point is found by probing 1, 2, 4, 8, … slots to the left and then binary searching the last gap, so an element that moves `d` slots costs at most `2⌊log₂ d⌋ + 2` comparisons instead of `d`. -- Pair moves: two elements are inserted at once; the smaller one is only searched for left of the larger one's slot, and prefix elements greater than both shift right by two slots in a single move, so they move once instead of twice. Short moves of elements of up to 32 bytes are made in place, four elements at a time, instead of through `memmove`, which costs them more than the copying (1.05–1.3 times faster for integers on short or nearly sorted slices and up to 1.5 times on short reverse-sorted ones, within 1.03 times either way on long random ones). - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::insertion; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - insertion(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -Equal keys keep their original order (cards are compared by rank only): - -```rust -use a::sort::insertion; -use std::cmp::Ordering; - -#[derive(Debug, PartialEq, Eq)] -struct Card(u8, char); - -impl Ord for Card { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl PartialOrd for Card { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -fn main() { - let mut cards = vec![Card(2, 'a'), Card(1, 'x'), Card(2, 'b'), Card(0, 'y')]; - insertion(&mut cards); - assert_eq!(cards, vec![Card(0, 'y'), Card(1, 'x'), Card(2, 'a'), Card(2, 'b')]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) element moves, but only O(n log n) comparisons - - Average: O(n^2) moves, O(n log n) comparisons - - Best: O(n), when the input is already sorted -- Comparisons and moves (`dₓ` = how many slots element `x` moves left, `I = Σ dₓ` = number of inversions): - - Any input: at most n−1 + Σ 2⌈log₂(dₓ+1)⌉ comparisons and at most I + n moves - - Worst comparisons: at most 2n·log₂ n (17,020 for n = 1000, against 499,500 for classic insertion sort) - - Reverse sorted: about (n/2)·log₂ n + n comparisons and n²/4 moves - - Best comparisons: n−1; best moves: 0 - - Against inserting one element at a time with the same search: about a third fewer moves on random input and half on reverse-sorted input -- Space complexity: O(1) extra (in-place) -- Stability: stable (equal elements keep their relative order) -- Adaptivity: very good; cost grows with the logarithm of how far elements move, so nearly-sorted input sorts in close to linear time; unsuitable for large, random inputs compared to O(n log n) sorts because of the O(n^2) moves - -### Cocktail Shaker - -Cocktail shaker sort is a simple, stable, in-place sorting algorithm: bubble sort that alternates direction. A forward pass carries the largest element seen so far towards the end, then a backward pass carries the smallest element seen so far towards the front, so the unsorted range shrinks from both ends. Unlike bubble sort, a small element near the end (a "turtle") reaches its place in a single backward pass instead of moving one slot per pass. This implementation uses three optimizations: - -- Last-move bounds at both ends: a forward pass ends the unsorted range at the last element it moved and a backward pass starts it just after the last element it moved, so sorted runs at either end are never rescanned; a pass that moves nothing ends the sort. -- Hole shifting instead of swapping: an element travelling `k` slots is read once, the `k` elements it overtakes shift one slot each as they are compared with it, and it is written once into its final slot: `k + 1` moves instead of the `3k` of `k` swaps. Shifting while comparing touches each element once and avoids a `memmove` call per (mostly very short) carry, which makes it about twice as fast on random integers as finding the distance first and moving a block afterwards. -- Branchless passes for small elements (at most 8 bytes, no drop glue): the running extreme is held in a register and every comparison writes the other element into the slot behind it, with no branch on the outcome. Each pass picks branchy or branchless from what the previous one found, so sorted input still costs one branchy pass. The comparisons are exactly those of classic cocktail shaker sort; on random `u64` input this is 1.8–2.6 times faster up to a few hundred elements and 1.2 times at 1,000, and 1.5–2.6 times on elements that are each a few slots from their places. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::cocktail_shaker; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - cocktail_shaker(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -A turtle (the 0 at the end reaches the front in one backward pass; bubble sort would need five passes): - -```rust -use a::sort::cocktail_shaker; - -fn main() { - let mut v = vec![1, 2, 3, 4, 5, 0]; - cocktail_shaker(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) - - Average: O(n^2) - - Best: O(n) with early-exit when input is already sorted -- Comparisons and moves (`I` = number of inversions, `C` = number of carries, i.e. elements picked up and moved at least one slot, counted once per pass): - - Worst comparisons: n(n−1)/2 (reverse sorted); worst moves: n(n+1)/2 − 1 - - Any input: at most n(n−1)/2 comparisons and exactly I + C ≤ 2I moves - - Best comparisons: n−1; best moves: 0 - - Random input: about n²/3 comparisons, a third fewer than bubble sort's n²/2 (≈ 332,000 against ≈ 496,000 for n = 1000) - - Turtle `[1, 2, …, n−1, 0]`: 3n−6 comparisons, against n(n−1)/2 for bubble sort (2,994 against 499,500 for n = 1000) -- Space complexity: O(1) extra (in-place) -- Stability: stable (equal elements keep their relative order) -- Adaptivity: good for nearly-sorted data, including elements that are out of place in either direction; unsuitable for large, random inputs compared to O(n log n) sorts - -### Gnome - -Gnome sort is a simple, stable, in-place sorting algorithm. A gnome walks along the slice comparing each element with the one before it: while they are in order it steps forward, and when they are not it carries the element back, one slot at a time, until it meets an element that is not greater than it or reaches the front. The slice behind the gnome is always sorted. This implementation uses two optimizations: - -- Teleportation: classic gnome sort walks forward again over the slots it just carried an element back across, re-comparing pairs it already knows are in order; this gnome returns to where it turned back at once. Every comparison it makes is one classic gnome sort also makes, in the same order, but it saves exactly one per inversion: about half of them on random input. -- Hole shifting instead of swapping: an element travelling `d` slots is read once, the `d` greater elements it passes shift right by one slot each as they are compared with it, and it is written once into its final slot: `d + 1` moves instead of the `3d` of `d` swaps. The walk back is unrolled four times, which makes small elements up to 1.2 times faster (reverse-sorted input up to 1.5 times). - -With both optimizations gnome sort makes the same comparisons and moves as insertion sort with a linear search; the [Insertion](#insertion) sort above uses a galloping search instead and makes far fewer comparisons. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::gnome; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - gnome(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -A turtle (the 0 at the end is carried to the front in a single walk back): - -```rust -use a::sort::gnome; - -fn main() { - let mut v = vec![1, 2, 3, 4, 5, 0]; - gnome(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) - - Average: O(n^2) - - Best: O(n), when the input is already sorted -- Comparisons and moves (`I` = number of inversions, `C` = number of elements that move, `F` = number of elements other than the first that are smaller than everything before them, i.e. carried all the way to the front): - - Any input: exactly n−1 + I − F comparisons and exactly I + C moves - - Worst comparisons: n(n−1)/2 (reverse sorted), against n(n−1) for classic gnome sort; worst moves: n(n+1)/2 − 1 - - Best comparisons: n−1; best moves: 0 - - Random input: about n²/4 comparisons and n²/4 moves, against n²/2 comparisons and n²/4 swaps (3n²/4 moves) for classic gnome sort (≈ 250,800 against ≈ 500,600 comparisons for n = 1000) - - Turtle `[1, 2, …, n−1, 0]`: 2n−3 comparisons, against 3n−4 for classic gnome sort and n(n−1)/2 for bubble sort -- Space complexity: O(1) extra (in-place) -- Stability: stable (equal elements keep their relative order) -- Adaptivity: good for nearly-sorted data; cost grows with the number of inversions; unsuitable for large, random inputs compared to O(n log n) sorts - -### Comb - -Comb sort is a simple, unstable, in-place sorting algorithm: bubble sort with a gap. A pass compares every element with the one `gap` slots after it and swaps the two if they are out of order, so a small element near the end (a "turtle") moves `gap` slots left in one step instead of one. The gap starts at the length of the slice and shrinks by a factor of 1.3 after each pass; once it reaches 1, the slice is nearly sorted and bubble sort finishes it. This implementation uses four optimizations: - -- Rule of 11 (Combsort11): a gap of 9 or 10 is replaced by 11, which leaves fewer turtles for the final bubble sort; on random input of 10,000 elements this saves about 6% of the comparisons and narrows their spread. -- Adaptive finish: the final phase is this crate's [Bubble](#bubble) sort, which scans only up to the last element it moved and moves elements by hole shifting, instead of the repeated full passes of classic comb sort. -- Branchless swaps for small elements: whether a gapped pass swaps a pair is close to a coin flip, which defeats the branch predictor, so for elements of at most 32 bytes both slots are rewritten from addresses chosen by the comparison, and each pass is walked in blocks of `gap` pairs over two non-overlapping slices. On random input this is 3–4.5 times faster for integers and 1.5–1.8 times for strings. Larger elements are swapped only when needed, since rewriting them costs more than a misprediction. -- Carrying in registers at small gaps, for elements of at most 16 bytes: at a gap of 14 or less, each pair overlaps the one `gap` slots before it, so a swap has to reach memory before the next comparison can read it back. Instead, the pass holds the largest element seen so far of every chain `c, c + gap, c + 2·gap, …` in a register, one per chain, and writes back only the smaller of it and the next element: the comparisons of a plain pass, in its order and with the same outcomes and result, with one move per swap plus one per carried element instead of two per swap, and the `gap` chains overlapping. Together with the faster final bubble sort this makes the whole sort 1.15–1.5 times faster for `u64`, 1.4–1.8 times for `u32` and 1.2–1.5 times for pairs of integers than before; `String`, whose comparisons usually read memory, keeps the plain pass. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::comb; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - comb(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -Turtles (the small elements at the end move many slots per step): - -```rust -use a::sort::comb; - -fn main() { - let mut v = vec![5, 6, 7, 8, 9, 0, 1, 2, 3, 4]; - comb(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) in theory (no known input reaches it in practice, but no O(n log n) bound has been proved) - - Average: about n·log₁.₃ n comparisons - - Best: O(n log n); the gapped passes cost the same on every input -- Comparisons (`g₁ > g₂ > … > gₖ ≥ 2` = the gaps, `k ≈ log₁.₃ n − 4`): - - Gapped passes: exactly Σ (n − gᵢ) ≈ n·(log₁.₃ n − 7.3) on every input - - Best (already sorted): Σ (n − gᵢ) + n − 1 (19,706 for n = 1000); best moves: 0 - - Random input: about n·(log₁.₃ n − 4.5) (≈ 21,300 for n = 1000, against ≈ 499,000 for bubble sort and ≈ 22,500 for classic comb sort without the rule of 11 and the adaptive finish) -- Space complexity: O(1) extra (in-place) -- Stability: unstable (a gapped swap can carry an element past an equal one) -- Adaptivity: not adaptive (only the final bubble sort gets cheaper on nearly-sorted input); much faster than the other quadratic sorts here on random input, about 1.5 times faster than `slice::sort_unstable` on 10,000 random `u32` and about as fast on `u64` (1.2 times slower at 1,000,000) - -### Merge - -Merge sort is a stable, `O(n log n)` sorting algorithm that uses a buffer. It splits the slice into two halves, sorts each half recursively, and merges the two sorted halves into one. Every element takes part in one merge per level of recursion and there are about `log₂ n` levels, so the cost hardly depends on the input. This implementation uses a buffer of `n` elements, kept on the stack when it fits in 4 KiB and allocated once otherwise, and seven optimizations: - -- Short runs sorted without branches: halves of at most 8 elements are not split further. A run is first scanned for order, which leaves a sorted run alone after len−1 comparisons and otherwise stops at the first descent, usually after one or two; then its two halves of at most 4 elements are sorted straight into the buffer by fixed sequences of comparisons that select where each element goes instead of branching on the outcome (1, 3 and 5 comparisons for 2, 3 and 4 elements, the last being the standard library's stable four-element sort), and merged back. Short runs took up to 60% of the time when they were insertion sorted; together with the stack buffer below, this makes the whole sort 1.2–2.8 times faster for integers (most for small slices) and 1.02–1.9 times for strings and large elements. -- Short slices of small elements noted for order: for elements of at most 8 bytes, a half of up to 64 elements is first scanned for descents (elements smaller than the one before) with one comparison per element and no branch on the answers. None and it is left alone, one at every element and it is reversed, otherwise runs and stretches of the recursion without a descent are left alone and the rest is sorted as above (random integers 1.1 times faster for about n more comparisons, sorted input 3.3–4 times, reverse-sorted 2.8–8 times). -- Ordered halves are not merged: if the first element of the right half is not smaller than the last element of the left half, one comparison replaces the merge, so already-sorted input costs exactly n−1 comparisons and is never moved. -- Reversed halves are swapped: if the last element of the right half is smaller than the first element of the left half, the halves trade places as blocks with no further comparison, so reverse-sorted input costs about 2.6n comparisons instead of n·log₂ n (about n for elements of at most 8 bytes). -- No level copied out before it is merged: every merge reads its halves where the level below left them, in the slice or in the buffer, and writes the other place, so each level moves every element once instead of twice, and the result is copied back at most once (1.03–1.1 times faster for integers and strings on random input, 1.06–1.45 times for 64-byte elements). -- Bidirectional, branchless merging: each merge proceeds from both ends at once, the front taking the smaller of the two first elements and the back the larger of the two last ones. The two independent chains of comparisons overlap in the processor, and every step copies from an address chosen by the comparison instead of branching on it. On random input this is 1.2–1.6 times faster than the classic merge that copies only the left half out and merges from the front. -- Long merges split in two or four: a merge of 128 elements or more is split at the middle of its result by a binary search (co-ranking) into two independent merges from both ends, four chains the processor overlaps, and from 4096 elements of at most 32 bytes into four; each part stops when a half has at most one element left, so it never compares an element whose copy is written already (1.13–1.4 times faster for integers, 1.12–1.55 times for strings and 1.08–1.2 times for 64-byte elements from 1000 elements on). -- No allocation for small slices: a buffer of up to 4 KiB lives on the stack, so a slice that fits in it (512 `u64`s, say) is sorted without touching the allocator. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::merge; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - merge(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Equal keys keep their original order (cards are compared by rank only): - -```rust -use a::sort::merge; -use std::cmp::Ordering; - -#[derive(Debug, PartialEq, Eq)] -struct Card(u8, char); - -impl Ord for Card { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl PartialOrd for Card { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -fn main() { - let ranks = [3, 1, 3, 0, 1, 3, 0, 2, 1, 0]; - let mut cards: Vec = "abcdefghij".chars().zip(ranks).map(|(c, r)| Card(r, c)).collect(); - merge(&mut cards); - let order: String = cards.iter().map(|c| c.1).collect(); - assert_eq!(order, "dgjbeihacf"); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n) - - Average: O(n log n) - - Best: O(n), when the input is already sorted (or made of ordered or reversed halves at every level) -- Comparisons and moves (a merge of m elements makes at most 2⌊m/2⌋ comparisons after the two checks, fewer when a half runs out early, plus about 3·log₂ m for the binary searches of a split merge, and moves every element once, from where its halves are to the other place): - - Best (already sorted): n−1 comparisons, 0 moves - - Reverse sorted: about 2.6n comparisons (2,590 for n = 1000, against 5,044 for the textbook merge sort; about n for elements of at most 8 bytes, 1,010 for n = 1000) - - Random input: about n·log₂ n − 0.35n comparisons (≈ 9,570 for n = 1000; ≈ 10,290 for elements of at most 8 bytes, the scans for descents included; the textbook merge sort makes ≈ 8,700, since the checks cost a comparison or two per merge and a bidirectional merge does not stop when one half runs out) and about n·log₂(n/8) + 2n moves - - Worst: at most n·log₂ n + O(n) comparisons and 1.5n·⌈log₂(n/8)⌉ + 3n moves -- Space complexity: O(n) extra (one buffer of n elements, on the stack up to 4 KiB, otherwise allocated once) plus O(log n) stack -- Stability: stable (equal elements keep their relative order) -- Adaptivity: adaptive at the scale of runs and halves (sorted and reverse-sorted input are nearly linear). Against `slice::sort`, with new input for every sort: 1.6–2.4 times faster on up to 64 random integers and 1.08–1.23 times slower from 1000 on, 1.07–1.09 times slower on random strings of 1000 to 100,000 (1.65 times faster on a million), 1.1–1.35 times faster on random 64-byte elements from 1000 on and 1.2–2.6 times on nearly sorted integers, but 3–6 times slower with few distinct keys and 1.6–10 times on reverse-sorted input from 1000 elements on, which the standard library's stable sort handles in one pass - -### Quick - -Quick sort is an unstable, in-place, `O(n log n)` sorting algorithm. It picks a pivot, partitions the slice into the elements smaller than the pivot and the rest, puts the pivot between them, and sorts the two parts the same way. This implementation uses seven optimizations: - -- Sorted and reverse-sorted input is recognised: the slice is first scanned for one ascending or strictly descending run; if it covers the whole slice, the slice is left alone or reversed, at n−1 comparisons (usually two or three on other input). -- Pivots from a median of samples: the median of three elements spread over the slice, from 64 elements on the median of three such medians (the "ninther"), and from 4096 elements on a pseudo-median of 27 evenly spaced samples, three times as many for every 16 times the length, which keeps comparisons at about n·log₂ n on random input, against 1.39·n·log₂ n for a random pivot. -- Branchless partitioning for elements of at most 64 bytes: whether an element is smaller than the pivot is a coin flip on random input, so instead of branching, a vacant slot cycles through the slice and every element is moved unconditionally, the comparison only choosing where (Lomuto's scheme, made branchless). That is 1.9–4.3 times faster than Hoare's scheme for integers and 1.4–2.5 times for strings; larger elements, for which two moves per element cost more than a misprediction, use Hoare's scheme, which moves only misplaced pairs (two moves per pair, cycled through a vacant slot, instead of three for a swap). -- Runs of equal elements in one pass: if the pivot equals the pivot of the enclosing partition, the part is split into the elements equal to it, which are done, and the rest, so `k` distinct keys cost O(n log k) and all-equal input is never quadratic. -- Recursion into the smaller part only, looping on the larger one, so the stack never holds more than log₂ n frames. -- Sorting networks for short parts: for elements of at most 16 bytes, parts of at most 8 elements are sorted by optimal sorting networks made of branchless compare-exchanges, unrolled for each length so the elements stay in registers (1.3–1.85 times faster than insertion sort for random integers); larger elements, for which a compare-exchange's three moves cost more, sort parts of at most 16 elements by insertion sort. -- A guaranteed O(n log n): a part still being partitioned after 2·log₂ n levels means the pivots were consistently bad, which an adversary that answers comparisons adaptively can force on any pivot rule; such a part is finished by heap sort (the "introsort" idea). Against McIlroy's adversary, 10,000 elements cost about 480,000 comparisons instead of 8.4 million. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::quick; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - quick(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Many duplicates (each distinct pivot takes all its copies out in one pass): - -```rust -use a::sort::quick; - -fn main() { - let mut v: Vec = (0..1000).map(|i| (i * 7 % 3) as u8).collect(); - quick(&mut v); - assert!(v.windows(2).all(|w| w[0] <= w[1])); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n), thanks to the [Heap](#heap) sort fallback - - Average: O(n log n) - - Best: O(n), when the input is sorted, reverse sorted or all equal -- Comparisons: - - Best (sorted, reverse sorted, all equal): n−1 - - Random input: about n·log₂ n + 0.2n to n·log₂ n + 0.6n (≈ 10,200 for n = 1000, n·log₂ n + 0.6n for n = 100,000 and 1,000,000), up to 0.4n more for elements larger than 16 bytes - - `k` distinct keys: O(n log k), about 1.1–1.45·n·(log₂ k + 1) in practice -- Space complexity: O(log n) stack, no allocation (in-place) -- Stability: unstable (equal elements may be reordered) -- Adaptivity: adaptive to sorted and reverse-sorted input and to repeated keys; on nearly sorted input (a few elements out of place) Hoare's scheme would be 1.5–1.75 times faster for integers, since its branches are then predictable. `slice::sort_unstable`, which shares several of these ideas, is 1.1–1.2 times faster on random integers from 100 elements on (it merges short parts through a buffer, which is only sound for types without interior mutability), but this sort is 1.9 times faster on 20 integers, 1.03–1.14 times faster on random strings and as fast or faster with 256-byte elements - -### Heap - -Heap sort is an unstable, in-place, `O(n log n)` sorting algorithm with constant stack. It arranges the slice into a max-heap, a tree stored in the slice itself in which no element is smaller than its children, so the largest element is at the root; it then repeatedly moves the root to the end of the heap, shrinks the heap by one and restores the heap order. Building the heap takes O(n) comparisons and each of the n removals O(log n), whatever the input. This implementation uses five optimizations: - -- Sorted and reverse-sorted input is recognised: the slice is first scanned for one ascending or strictly descending run; if it covers the whole slice, the slice is left alone or reversed, at n−1 comparisons (usually two or three on other input). Heap sort itself would spend n·log₂ n comparisons on sorted input. -- Bottom-up removal: the textbook sift-down compares the element moved to the root with the larger child at every level, but that element comes from the end of the heap and usually sinks almost to the bottom anyway. Instead, the vacant root moves down to a leaf along the larger children, one comparison per level, and the element moves up from there, usually by a level or two (Wegener's bottom-up heap sort): about n·log₂ n comparisons instead of 1.7–1.85·n·log₂ n. -- Holes instead of swaps: the element being placed is held outside the slice, and each level moves one element into the vacant slot instead of swapping two, one move instead of three. -- Four children per node for large heaps and small elements: a heap with four children per node has half as many levels, at three comparisons per level instead of one (about 1.5·n·log₂ n in all). That pays from 64 KiB of elements on, where every level of the heap costs a cache miss: on random input 1.15–1.35 times faster for integers from 100,000 elements on, 1.1–1.4 times for strings and 1.25–2 times for 64-byte elements; on nearly sorted strings 1.05–1.15 times slower. It also pays for elements of at most 8 bytes at any length, whose comparisons are so cheap that a level costs about the time to load the children, compare them and find the next ones: 1.15–1.25 times faster for integers in cache (`&str`, as small as a `u128` but slower to compare, would be 1.05–1.15 times slower, so 16-byte elements keep two children below 64 KiB). -- Short chains from one level to the next: each level of a removal waits for the comparison before it, which says where the next children are. Every child is loaded straight from the first child's index, and for elements of up to 16 bytes the larger child and its own first child are chosen by selects between values computed beforehand; with four children the winners of the first round are selected as pointers, so the values already loaded are compared instead of being loaded again. This alone makes integers sort 1.25–1.3 times faster on four children per node. Larger elements, which compare by chains of branches that would turn the selects back into branches, keep arithmetic. - -Together, against the textbook heap sort, this is 1.1–1.25 times faster for integers up to 1000 elements and 1.4–1.85 times from 10,000 on, 1.05–1.4 times for strings from 1000 elements on and 1.3–2.2 times for 256-byte elements. Only sawtooth input of 1000–2000 integers, whose short periods make three comparisons per level cost more than the levels saved, would sort 1.02–1.05 times faster on two children per node. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::heap; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - heap(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Rust usage with strings: - -```rust -use a::sort::heap; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - heap(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n) - - Average: O(n log n) - - Best: O(n), when the input is sorted or reverse sorted -- Comparisons: - - Best (sorted, reverse sorted): n−1 - - Random input: about n·log₂ n + 0.6n (≈ 10,500 for n = 1000, against ≈ 16,800 for the textbook heap sort); about 1.5·n·log₂ n with four children per node, from 64 KiB of elements on and for elements of at most 8 bytes (≈ 14,600 for n = 1000) - - Worst: O(n log n), on every input -- Moves: about n·log₂ n, and about 0.5·n·log₂ n with four children per node -- Space complexity: O(1) stack, no allocation (in-place) -- Stability: unstable (equal elements may be reordered) -- Adaptivity: not adaptive, except to sorted and reverse-sorted input. Heap sort jumps around the slice, so `slice::sort_unstable` and this crate's [Quick](#quick) sort, which falls back to heap sort only when its pivots keep failing, are faster: 2–3.4 times for random integers from 64 elements on, 1.6–6 times for strings and 1.1–3.5 times for 256-byte elements. Heap sort's strength is its guarantee: O(n log n) on every input, in place, with constant stack - -### Tim - -Tim sort is a stable, `O(n log n)` merge sort that makes use of the order already in the input; Tim Peters designed it for Python's `list.sort` in 2002. It splits the slice into runs, the longest stretches that are ascending or strictly descending, and reverses the descending ones. Runs shorter than a minimum length (16 to 32 elements, chosen so that the slice splits into a power of two runs or slightly fewer) are extended to it by insertion sort (to twice it for elements of at most 8 bytes), and runs are merged as they are found, in an order kept on a stack, until one is left. Random input has short runs and costs as much as any merge sort, but input made of a few long runs costs little more than finding them. This implementation uses a buffer of `n` elements, kept on the stack when it fits in 4 KiB and otherwise allocated when the first merge needs it, and six optimizations: - -- Galloping: before a merge, two searches find the elements of the left run that are not greater than the first of the right run, and those of the right run that are not smaller than the last of the left run; they are already in place and stay out of the merge. During a merge, once one run has supplied seven elements in a row, the merge gallops: it finds how many more of that run's elements come next by probing 1, 2, 4, … elements ahead and binary searching the last gap, and moves them as a block, `k` elements for about `2·log₂ k` comparisons. The threshold drops while galloping pays off and rises when it does not, so random input hardly ever gallops. -- Powersort merge policy: two runs are merged once the boundary between them lies deeper in a perfectly balanced merge tree over the slice than the boundary after them (Munro and Wild's powersort, used by CPython's Tim sort since Python 3.11). This merges in close to the cheapest order possible and never keeps more than 65 runs on the stack; Peters' original stack rules are 1.04–1.12 times slower on input made of runs of varying lengths and no faster elsewhere. -- Bidirectional, branchless merging: like [Merge](#merge) sort, a merge copies both runs out and merges them back from both ends at once, two independent chains of comparisons that copy from an address chosen by the comparison instead of branching on it; each end counts its own winning streaks and gallops on its own. Tim sort's classic merge, which copies only the shorter run out and merges from one end, is 1.15–1.45 times slower on random input, but moves less when one run is much longer than the other, so merges of runs more than four times as long as each other still use it (1.05–1.55 times faster on sorted input with elements appended). -- Pair insertion: short runs are extended by inserting elements two at a time, scanning from the right; the larger one is carried left first and the smaller one goes on from where it stopped, so the elements both pass move once. Peters' binary insertion makes fewer comparisons, but each probe waits for the previous one and its moves go through `memmove`: it is 1.2–1.9 times slower on random input, strings included. Runs of 16 to 32 elements (CPython uses 32 to 64) are 1.1–1.2 times faster on 100 to 1000 random elements when every short run is extended by insertion. -- Branchless short runs for small elements: each insertion ends in a mispredicted branch, which for elements of at most 8 bytes costs more than everything else, so for them a natural run shorter than the minimum length is extended to twice it (32 to 64 elements, as in CPython; natural runs of the minimum length are kept), and a short run that starts with fewer than four sorted elements, or whose next elements dip below its end, is sorted by [Merge](#merge) sort's branchless sort for short slices, which notes the descents and leaves alone what is in order. Against insertion this is 1.1–1.3 times faster for integers on random input and 1.1–1.2 times with few distinct keys, and up to 1.25 times slower on short slices that are nearly sorted in small steps; the runs of twice the length are another 1.08–1.3 times faster on random input and with few distinct keys and 1.05–1.45 times on input made of runs up to 1000 elements, up to 1.07 times slower on runs of about 45 elements and 1.05–1.75 times on a sawtooth of 40 to 1000 elements. -- No allocation for small slices or single runs: the buffer lives on the stack while it fits in 4 KiB (512 `u64`s, say), and is otherwise allocated when the first merge needs it, so sorted and strictly descending input never allocates. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::tim; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - tim(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -A sorted log with late entries appended is two runs, merged mostly by galloping: - -```rust -use a::sort::tim; - -fn main() { - let mut times: Vec = (0..1000).map(|i| i * 10).collect(); - times.extend([15, 4005, 9999]); - tim(&mut times); - assert!(times.windows(2).all(|w| w[0] <= w[1])); -} -``` - -Equal keys keep their original order (cards are compared by rank only): - -```rust -use a::sort::tim; -use std::cmp::Ordering; - -#[derive(Debug, PartialEq, Eq)] -struct Card(u8, char); - -impl Ord for Card { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl PartialOrd for Card { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -fn main() { - let ranks = [3, 1, 3, 0, 1, 3, 0, 2, 1, 0]; - let mut cards: Vec = "abcdefghij".chars().zip(ranks).map(|(c, r)| Card(r, c)).collect(); - tim(&mut cards); - let order: String = cards.iter().map(|c| c.1).collect(); - assert_eq!(order, "dgjbeihacf"); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n) - - Average: O(n log n) - - Best: O(n), when the input is sorted or strictly descending; O(n + n·H) for input made of runs, `H ≤ log₂ r` being the entropy of the lengths of its `r` runs -- Comparisons: - - Best (sorted, strictly descending): n−1, with no moves (strictly descending input is reversed) - - Nearly sorted: about 1,180 comparisons for n = 1000 with 10 random elements appended and 1,800 with 10 random pairs swapped (2,060 for elements of at most 8 bytes; merge sort: about 1,800 and 3,900) - - Random input: about 1.06·n·log₂ n for elements of at most 8 bytes (≈ 10,580 for n = 1000, against ≈ 10,290 for merge sort) and about (1.05–1.17)·n·log₂ n for larger ones (≈ 11,700, merge sort ≈ 9,570), whose short runs are built by linear insertion: more comparisons than a binary search would make, but less time - - Worst: O(n log n) -- Space complexity: O(n) extra (one buffer of n elements, on the stack up to 4 KiB, otherwise allocated once, and only if a merge is needed) plus a stack of at most 65 runs -- Stability: stable (equal elements keep their relative order) -- Adaptivity: very good. Against this crate's merge sort, which no longer copies a level out before merging it and splits long merges, and so is the faster of the two on input without long runs: 1.1–1.5 times slower on random input from 1000 elements on and up to 1.35 times on short runs of random content, but 1.2–1.5 times faster on nearly sorted integers and strings, 1.45–2.2 times on sorted input with elements appended and 1.7–11 times on reverse-sorted input from 1000 elements on. Against `slice::sort`, which also finds runs: 1.2–1.45 times slower on random integers from 1000 elements on and 1.35–1.45 times on random strings (1.5 times faster on a million), but 1.5–3.7 times faster on nearly sorted input, 1.9–3.4 times on appended elements and 1.6–3.6 times on input made of runs of `√n` elements from 10,000 on, for integers and strings - -### Intro - -Intro sort (introspective sort) is an unstable, in-place, `O(n log n)` sorting algorithm designed by David Musser in 1997: a quick sort that watches its own progress. Quick sort partitions the slice around a pivot and sorts the two parts the same way; with good pivots each level halves the parts, but no cheap pivot rule is good on every input, and bad pivots make it quadratic. Intro sort counts the levels, and a part still being partitioned after `2·⌊log₂ n⌋` levels is handed to [Heap](#heap) sort, which is `O(n log n)` on every input; typical input never reaches the limit and keeps quick sort's speed. Short parts are left to insertion sort. This implementation keeps Hoare's partitioning comparisons and uses five optimizations: - -- Sorted and reverse-sorted input is recognised: the slice is first scanned for one ascending or strictly descending run; if it covers the whole slice, the slice is left alone or reversed, at n−1 comparisons (usually two or three on other input). -- Branchless block partitioning: Hoare's scheme scans from both ends for elements on the wrong side of the pivot and swaps them, and each scan stops at an unpredictable element, a mispredicted branch. Here each end compares a block of 128 elements first (Edelkamp and Weiss's choice; 1.02–1.06 times faster than 64 on random input from 1000 elements on), recording the misplaced ones by offset without branching, and then exchanges as many misplaced pairs as both blocks have (Edelkamp and Weiss's BlockQuicksort), in a cycle of two moves per pair instead of a swap's three. Same comparisons as Hoare's scheme; on random input 1.8–2.7 times faster for integers, 1.7–2.2 times for strings and 1.4–2.1 times for 64-byte elements. The block scans are unrolled by 8, another 1.03–1.08 times on random input and 1.1–1.25 times on nearly sorted input, where the scans are nearly all the work. -- Equal keys split evenly: as in Hoare's and Musser's scheme, an element equal to the pivot stops the scans on both sides, so equal keys are spread over both parts, and input with few distinct keys, even all equal but one, costs O(n log n) without ever reaching the depth limit. -- Pivots away from the ends, from a ninther on long parts: the median of the elements at a quarter, half and three quarters of the part, and from 128 elements on the median of three medians of three (Tukey's ninther), 1.04–1.13 times faster on sawtooth input of 100,000 elements or more. Musser samples the first, middle and last elements; swapping the pivot to the front then moves the smallest element of a sorted stretch into the middle, where the next sample finds it again, so sorted input with a few elements appended keeps getting the second smallest element as its pivot until the depth limit gives up: quartiles are 1.2–4 times faster there. -- Insertion sort while the part is fresh: parts of at most 24 elements (Musser: 16) are insertion sorted as soon as partitioning leaves them, still in the cache, instead of in one pass over the whole slice at the end, which reads the whole slice once more and is 1.05–1.15 times slower for strings. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::intro; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - intro(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Rust usage with strings: - -```rust -use a::sort::intro; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - intro(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n), guaranteed by the heap sort fallback - - Average: O(n log n) - - Best: O(n), when the input is sorted or reverse sorted -- Comparisons: - - Best (sorted, reverse sorted, all equal): n−1 - - Random input: about (1.1–1.19)·n·log₂ n (≈ 11,800 for n = 1000); insertion into parts of up to 24 elements costs more comparisons than partitioning them further would, but less time - - Few distinct keys: about 0.85·n·log₂ n; unlike [Quick](#quick) sort, intro sort does not take all copies of a pivot out at once - - Adversarial input: McIlroy's adversary, which answers comparisons so as to make every pivot bad, drives partitioning alone to 9.1 million comparisons for n = 10,000; with the heap sort fallback the whole sort makes about 470,000 -- Space complexity: O(log n) stack, no allocation (in-place) -- Stability: unstable (equal elements may be reordered) -- Adaptivity: adaptive to sorted and reverse-sorted input, and cheap on nearly sorted input, where Hoare's scheme moves only what is misplaced. Against [Quick](#quick) sort: 1.35–1.55 times slower on random integers from 1000 elements on, 1.06–1.13 times on random strings and 1.5–3.8 times with few distinct keys, but 1.2–1.65 times faster on random 64-byte elements and 1.3–3.6 times faster on nearly sorted input. Against `slice::sort_unstable`: 1.45–1.75 times slower on random integers and within 1.07 times on random strings, but 1.08–1.55 times faster on random 64-byte elements and 1.5–3.7 times on nearly sorted input. Against [Heap](#heap) sort, which it falls back to: 1.3–2.4 times faster on random integers, 2.1–6.7 times on strings and 1.5–4.4 times on 64-byte elements - -### Shell - -Shell sort, published by Donald Shell in 1959, is an unstable, in-place insertion sort with a gap. A pass with gap `h` insertion sorts every chain of elements `h` apart, so an element far from its place travels there in steps of `h` instead of one slot at a time; the passes use a decreasing sequence of gaps ending with 1, a plain insertion sort that the earlier passes leave with little to do. How fast it is depends almost entirely on the gaps. This implementation uses five optimizations: - -- Sorted and reverse-sorted input is recognised: the slice is first scanned for one ascending or strictly descending run; if it covers the whole slice, the slice is left alone or reversed, at n−1 comparisons (usually two or three on other input). -- Hole shifting: an element is read out once, the greater elements before it on its chain shift one place along the chain each, and it is written once into the slot left free, one move per step instead of a swap's three. -- The gaps that are fastest, not the fewest comparisons: Ciura's sequence (1, 4, 10, 23, 57, 132, 301, 701, then times 2.25) makes 1.1–1.27 times fewer comparisons than Sedgewick's 1982 one (1, 8, 23, 77, 281, 1073, …), but nearly every insertion ends in a mispredicted branch, and Sedgewick's gaps grow by a factor of about 4, so they take fewer passes: for elements larger than 8 bytes they are 1.05–1.35 times faster, strings included (except random strings from a million elements on, 1.1 times slower). Elements of at most 8 bytes use Ciura's gaps, which suit the window below. -- A branchless window for small elements: in a pass over a slice far from sorted, how far an element moves along its chain is about as hard to predict as a coin toss (with Ciura's gaps 0 steps 44% of the time, 1 step 26%, 2 steps 14%, 3 steps 8%, 4 or more 8%). For elements of at most 8 bytes, each element is compared with the 4 elements before it on its chain at once, how many it passes is computed as a number, and the 4 slots it can move into are rewritten from copies held in registers, each chosen by two selects on that number; only the 8% that pass all 4 go on with a branch per step. On random input this is 1.65–1.9 times faster for integers. -- The window only where it pays, judged by the insertions themselves: where the slice is nearly sorted or its order follows a pattern, insertion's branches are predictable and the window's extra comparisons and writes are wasted (a pass can be up to 5 times slower with it), and how many elements move does not tell the two apart, but how regularly they move does. So a pass is made in blocks of 512 elements, and each block counts, at no measurable cost, how predictable its insertions were and chooses the window for the next block only if they were not: inserted plainly, the runs of consecutive elements that moved and the steps they took beyond the first (`2·runs + 0.35·steps > 0.42` per element); through the window, how often the number of elements passed differs from the one before (more than 0.42 per element). As branch predictors also learn short patterns, the last 32 elements of each block note their steps too, and a block whose steps repeat with a period of at most 16 is followed by plain insertion. Against taking the window whenever the pass before moved more than a quarter of the elements: 1.2–1.85 times faster on sawtooth input of 1000 to a million integers with periods from 4 to n/8 (but a period of √n from 100,000 elements on, as fast to 1.1 times slower), 2.1–2.45 times on organ-pipe input, 1.05–1.15 times on nearly sorted input, 1.05–1.25 times with 4 distinct keys, and as fast on random input (sorted runs of about √n random elements are 1.03–1.25 times slower below 10,000 elements). - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::shell; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - shell(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Rust usage with strings: - -```rust -use a::sort::shell; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - shell(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^(4/3)) with Sedgewick's gaps; no bound is known for Ciura's, which were found by experiment - - Average: no exact bound is known; measured, 1.6–2.8·n·log₂ n comparisons up to a million elements (no gap sequence is known to make Shell sort O(n log n)) - - Best: O(n), when the input is sorted or reverse sorted -- Comparisons: - - Best (sorted, reverse sorted, all equal): n−1 - - Random input: about (1.6–2)·n·log₂ n with Sedgewick's gaps (≈ 15,900 for n = 1000) and (2.1–2.8)·n·log₂ n with Ciura's gaps and the window (≈ 21,300), whose unnecessary comparisons are cheap for small elements -- Space complexity: O(1) extra (in-place) -- Stability: unstable (an insertion along a chain can carry an element past an equal one between the two) -- Adaptivity: the passes do less on nearly sorted input. Against classic Shell sort with Ciura's gaps: 1.65–1.9 times faster on random integers from 1000 elements on, up to 1.75 times on nearly sorted ones, 1.3–1.4 times with few distinct keys and 1.1–1.2 times on organ-pipe input; on sawtooth input of 1000–100,000 integers with periods from 4 to n/8 from 1.15 times slower (a period of 16 at 1000 elements) to 1.35 times faster, mostly within 1.1 times either way; 1.05–1.4 times faster on strings and 1.03–1.3 times on 64-byte elements. Against [Comb](#comb) sort: 3.55–4 times slower on random integers from 1000 elements on and 1.8–2.65 times on nearly sorted ones or with few distinct keys (1.15 times faster on 100 nearly sorted ones), but 1.1–2.5 times faster on strings that have few distinct keys or follow a pattern, and 1.6 times on 1000 nearly sorted strings (as fast on longer ones). Against [Heap](#heap) sort: 1.45–1.8 times slower on random integers from 1000 elements on, 1.1–1.6 times on nearly sorted ones and 1.1–1.25 times with few distinct keys, about as fast on random strings, and faster on strings that are nearly sorted or patterned - -### Drift - -Drift sort is a stable, `O(n log n)` sorting algorithm designed by Orson Peters and Lukas Bergdoll in 2024; it has been the stable sort of Rust's standard library since 1.81. It drifts between merge sort and quick sort: it scans for natural runs, ascending or strictly descending, but keeps only those of at least about `√n` elements (`min(n/2, 32)` below 4096 elements), and leaves everything else as unsorted chunks. Runs and chunks are combined in powersort's merge order: two adjacent unsorted chunks are simply taken together, while an unsorted chunk that meets a sorted run is first sorted by a stable quick sort and then merged. Random input ends up as one chunk for quick sort, which handles repeated keys in `O(n log k)`; input made of long runs is merged like [Tim](#tim) sort's. This implementation uses a buffer of `n` elements, kept on the stack when it fits in 4 KiB and otherwise allocated when first needed, and nine optimizations (ratios measured with new input for every sort): - -- This crate's best parts: runs are merged by [Tim](#tim) sort's merge (skipping elements already in place, merging from both ends at once without branching, galloping through runs that interleave in blocks, and running two merges at once once galloping has not paid off: 1.25–1.5 times faster on runs of random content); short parts are sorted by [Merge](#merge) sort's branchless short-run sort; and a quick sort still partitioning after `2·log₂ n` levels hands the part to merge sort, so the sort is `O(n log n)` on any input. -- Pivot samples at several scales: [Quick](#quick) sort's ninther and evenly spaced pseudo-median sample at one scale, which periodic input whose period divides the spacing defeats (a sawtooth of 1000 elements cost 2.4 times the standard library's time). The samples lie in three sections of an eighth of the width, centred at the middle and `3/8` of the width to either side, recursively: 9 samples at two scales from 64 elements, 27 at three from 1024, three times as many for every 16 times the length. The sawtooth then costs 3.3–3.7 times less and random input 1–2% fewer comparisons. -- Branchless stable partitioning: each element is compared with the pivot and copied into the buffer, to the front if it goes left and to the back otherwise, from an address the comparison selects, and both parts are copied back, the back one reversed; a partition that moved nothing skips the copies back (1.05–1.3 times faster for integers with 2 distinct keys). Elements larger than 8 bytes are compacted in place as they go left, only the right part passing through the buffer, with the pivot moved into a local as its only live copy and a guard that restores every element if a comparison panics: 1.2–1.3 times faster on random 64-byte elements, 1.2–1.5 times with repeated keys, 1.1–1.2 times on random strings from 10,000 elements. -- Repeated keys in one pass, without copying the pivot: when a part's pivot is no greater than the pivot of the partition that produced the part, every element equal to it is taken out in one pass; the earlier pivot is found in place (the standard library compares a bitwise copy, sound only for types without interior mutability). Where no earlier pivot bounds a part, it is first scanned for an element smaller than the pivot, a comparison or two on most input, which spares a partition that moves nothing: 1.08–1.25 times faster for integers with 2 distinct keys. -- No per-element bookkeeping: the partition loop is split at the pivot and at the earlier pivot instead of checking for them at every element, and for elements of at most 8 bytes it is unrolled by 4. -- Short parts noted for order and merged in place: a part of up to 32 elements of at most 8 bytes is scanned for descents with one comparison per element and no branch on the answers; runs and stretches of the merge tree without one are left alone, the rest is sorted by the short-run sort, and every merge reads its halves where the level below left them and writes the other place, so no level is copied out before it is merged. The merges leave no stale copy behind when a comparison mutates its arguments: a merge of up to 16 elements decides the order before it copies anything, a longer one copies the few elements compared after their copy was written once more (1.03–1.05 times slower on random integers of 100 to 10,000 elements than merges that compare stale copies, which are sound only for types without interior mutability). -- Short slices adapt too: a slice of up to 64 elements of at most 8 bytes gets the same scan for descents: none and it is left alone, one at every element and it is reversed, at most one per 8 elements and the rest is inserted into the sorted prefix, otherwise it is sorted with the descents found (1.2–3 times faster than scanning for a prefix run and sorting the rest on random, few-keys, sorted, reverse-sorted or appended input of 8–64 integers; 1.1–1.25 times slower on 8–28 nearly sorted ones). Larger elements are scanned for a sorted or strictly descending prefix and the rest inserted, without a buffer, up to 28 elements (40 beyond 32 bytes), and up to twice the small sort's length sort each chunk as soon as it is found and merge the chunks, with no chunk scanned twice. -- Runs cut by a chunk boundary are taken back: a natural run whose rest falls short of the length a run must have is probed with one or two comparisons and, if they are in order, scanned backwards into the unsorted chunk before it, the boundary moving with it: 1.05–1.2 times faster on input made of runs of about `√n` elements. -- No allocation for small slices or single runs: the buffer lives on the stack while it fits in 4 KiB and is otherwise allocated when first needed; short slices of large elements are insertion sorted and need none. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::drift; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - drift(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Equal keys keep their original order (cards are compared by rank only): - -```rust -use a::sort::drift; -use std::cmp::Ordering; - -#[derive(Debug, PartialEq, Eq)] -struct Card(u8, char); - -impl Ord for Card { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl PartialOrd for Card { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -fn main() { - let ranks = [3, 1, 3, 0, 1, 3, 0, 2, 1, 0]; - let mut cards: Vec = "abcdefghij".chars().zip(ranks).map(|(c, r)| Card(r, c)).collect(); - drift(&mut cards); - let order: String = cards.iter().map(|c| c.1).collect(); - assert_eq!(order, "dgjbeihacf"); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n), guaranteed by the merge sort fallback of the quick sort - - Average: O(n log n), and O(n log k) with `k` distinct keys - - Best: O(n), when the input is sorted or strictly descending -- Comparisons: - - Best (sorted, strictly descending): n−1, with no moves (strictly descending input is reversed) - - Random input: about (1.06–1.12)·n·log₂ n (≈ 11,200 for n = 1000, the standard library 10,600; elements of at most 8 bytes trade about 4% more comparisons for fewer mispredicted branches) - - Few distinct keys: about 0.22–0.28·n·log₂ n with 2 keys and 0.38–0.57·n·log₂ n with 16 (≈ 2,800 and 5,700 for n = 1000, against 7,050 and 9,390 for [Tim](#tim) sort with elements of at most 8 bytes and 5,290 and 10,380 with larger ones); about 2,050 with 10 random pairs swapped and 7,100 on a sawtooth of 1000 -- Space complexity: O(n) extra (one buffer of n elements, on the stack up to 4 KiB, otherwise allocated once and only when needed; the standard library makes do with about n/2 for large inputs) plus O(log n) stack -- Stability: stable (equal elements keep their relative order) -- Soundness: a comparison that panics, answers inconsistently or mutates its arguments through interior mutability never leaves a stale copy, a duplicate or a hole behind -- Adaptivity: adaptive to runs of `√n` elements or more, even where a chunk boundary cuts them, and to repeated keys. Against `slice::sort`, the standard library's drift sort, with new input for every sort: 1.3–3 times faster on 8–64 random integers, 1.03–1.06 times slower on 100–100,000 and as fast beyond; 1.05–1.07 times slower on random strings up to 1000 elements, 1.03–1.1 times faster from 10,000 and 1.7 times on a million; 1.1 times slower on random 64-byte elements up to 100 and 1.03–1.25 times faster from 1000; 1.05–1.9 times faster with few or `√n` distinct keys from 1000 elements and 1.25–1.65 times on a sawtooth of 1000; and 1.4–8 times faster on nearly sorted, appended, block-wise sorted input or runs of `√n` elements (runs from 10,000 elements on). Against [Tim](#tim) sort: 1.05–1.3 times faster on random integers from 1000 elements on, 1.4–1.65 times on random strings and 1.2–1.3 times on 64-byte elements, 1.6–9.3 times with few distinct keys (3.4–9.3 times for integers) and 1.25–2.55 times on sawtooth input up to 100,000 elements; 1.1–1.4 times slower on nearly sorted integers (1.15–2 times for strings) and 1.05–1.4 times slower on runs of `√n` elements, which Tim sort keeps whole - -### Counting - -Counting sort, described by Harold Seward in 1954, is the first sort of this library that does not compare elements with each other. It finds the smallest and the largest key, counts how many times each of the `k` keys in between occurs, and turns the counts into the sorted order, in `O(n + k)` time whatever the order of the input: faster than any comparison sort when `k` is not much larger than `n`, as with bytes, small integers or values in a known range. Keys are primitive integers, `char`s and `bool`s (the sealed `IntegerKey` trait). It comes in two forms: - -- `counting` sorts the keys themselves. Equal keys cannot be told apart, so the slice is simply rewritten from the counts: so many copies of the smallest key, then so many of the next. -- `counting_by_key` sorts any type by such a key, stably: the counts become the place where each key's elements begin, and every element is moved, in order, to the next place of its key in a buffer, which is then copied back. This is counting sort as radix sort uses it. `key` is called exactly once per element. - -A range of keys too large to count panics, leaving the slice as it was: one of more than 65,536 keys and more than four per element. Any range of a one- or two-byte type is allowed. This implementation uses six optimizations: - -- Sorted and reverse-sorted input is recognised: `counting` scans for one ascending or one non-increasing run, 16 neighbours at a time with no exit inside a block, which the compiler vectorizes, and `counting_by_key` checks for one ascending or one strictly descending run in the pass that reads the keys; a slice that is one run is left alone or reversed, and nothing is counted. -- Eight count tables or segments for small ranges: incrementing a count waits for the increment before it to reach memory, so a few keys repeating make a chain of dependent increments. `counting` counts into eight tables, one element after the other into the next, with at most 2048 keys and 16 elements per key (3–5 times faster with a few distinct keys and on runs of bytes); `counting_by_key` cuts the slice into eight segments with their own places for every key, after those of the segments before them, with at most 256 keys and 16 elements per key, which keeps it stable (1.6–2.3 times faster with four distinct keys, 2.4–2.9 times on runs of bytes). -- Runs of equal keys do not wait on each other with one table or segment either: counts do not depend on the order of the keys, so one table is counted walking four quarters of the slice side by side (for `counting`, with `u16` counts, or when the first values are mostly in order), and `counting_by_key` places four keys at a time, all four places read before any is written back: 2.2–2.6 times faster on nearly sorted input with sixteen elements per key, 1.4–1.7 times on nearly sorted `counting_by_key` input, and `counting` 1.05–1.3 times faster on random `u8` and `u32` values. -- Keys written ahead of their counts: `counting` writes each key 4 times (8 for one- and two-byte types, in one store) whatever its count, and only a count above that takes a branch; branching on every count mispredicts when there are about as many keys as elements and counts of 0, 1 and 2 are all common (2.5–5 times faster there). A range with 32 or more keys per element skips blocks of 8 empty counts at once instead. -- Keys read once: `counting_by_key` keeps the keys in a buffer as it reads them, and counts and places from it (1.05–1.25 times faster than calling `key` again on random keys); the counts and places can then be trusted without checking, and if `key` panics, no element has moved yet. -- No allocation for small sorts: counts (`u16` up to 65,535 elements, `u32` up to `u32::MAX`), keys and the element buffer each live on the stack up to 4 KiB; the places of `counting_by_key` are summed four at a time, one addition per four waiting for the one before it (1.1–1.25 times faster on random keys over four times as many keys as elements). - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::counting; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - counting(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Sorting by a key, stably (equal lengths keep their order): - -```rust -use a::sort::counting_by_key; - -fn main() { - let mut words = vec!["pear", "fig", "banana", "kiwi", "apple", "lime"]; - counting_by_key(&mut words, |w| w.len()); - assert_eq!(words, vec!["fig", "pear", "kiwi", "lime", "apple", "banana"]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst, average and best: O(n + k) for `k` possible keys between the smallest and the largest - - Sorted and reverse-sorted input (strictly descending for `counting_by_key`): O(n), one scan -- Comparisons: none between elements, except the scan for a sorted or reverse-sorted run, which leaves random input after a block or two -- Calls to `key` (`counting_by_key`): exactly n -- Space complexity: O(k) counts for `counting`; O(n + k) for `counting_by_key` (a buffer of n elements, one of n keys and the places); each on the stack up to 4 KiB, otherwise allocated once -- Stability: `counting_by_key` is stable; `counting` cannot tell equal keys apart -- Against the textbook counting sort: `counting` is 3–5.5 times faster on random integers with about as many distinct keys as elements, 1.7–3.1 times with four distinct keys and 9–80 times on sorted, reverse-sorted and all-equal input; `counting_by_key` is 1.05–1.6 times faster on random keys, 1.35–2.2 times with four distinct keys and 6–20 times on sorted and all-equal input, but 1.02–1.15 times slower on reverse-sorted input with repeated keys, random bytes from 10,000 elements on and 1000–10,000 random keys with sixteen times fewer keys than elements, where keeping the keys costs more than it saves (nearly sorted input, which was 1.02–1.15 times slower, now takes 0.6–0.85 times as long as it did) -- Against the standard library, from 100 elements on: `counting` is 3.2–10 times faster than `slice::sort_unstable` on random `u32` and `u64` with about as many distinct keys as elements, 1.9–4 times with four times as many, 2.8–4 times with four distinct keys, 2.2–12 times on nearly sorted input and 9–15 times on random bytes from 1000 elements on; `counting_by_key` is 2.9–7 times faster than `slice::sort_by_key` on random `u32` keys with about as many distinct keys as elements and 1.6–2.3 times with four distinct keys, and as fast on sorted input. On 16 elements both are up to 2 times slower on sorted or nearly sorted input, where the standard library's insertion sort shines - -### Radix - -LSD radix sort, used on punched-card sorters since the 1920s, does not compare elements either. It reads every key as a number of a few digits and sorts the slice by each digit in turn, from the least significant to the most, with a stable counting pass; as each pass keeps the order the earlier ones made among equal digits, the slice ends up sorted by the whole key. For `w`-bit keys and `r`-bit digits that is `⌈w / r⌉` passes of `O(n + 2^r)` each, whatever the order of the input: unlike [Counting](#counting) sort, any range of keys can be sorted. Keys are the same as counting sort's (the sealed `IntegerKey` trait), signed integers read with their sign bit flipped. It comes in two forms: `radix` sorts the keys themselves, and `radix_by_key` sorts any type by such a key, stably, calling `key` exactly once per element. This implementation uses eight optimizations: - -- Sorted and reverse-sorted input is recognised: `radix` scans for one ascending or one non-increasing run, 16 neighbours at a time with no exit inside a block, and `radix_by_key` checks for one ascending or one strictly descending run in the pass that reads the keys; a slice that is one run is left alone or reversed, and nothing is counted. -- Only the bits in which keys differ are sorted, in digits sized by the slice: one pass ORs together every key's difference from the first, and the span from its lowest to its highest set bit is cut into the fewest digits of equal width of at most `log₂ n` bits (at least 8 from 64 elements on, at most 12), the digit size CLRS derives for radix sort. Against byte digits with shared bytes skipped: 1.1–1.25 times faster on random integers from 10,000 elements on and 1.75–3.4 times on integers below `n`. -- Short slices count their digits side by side: every pass costs the sum of its counts however few elements it moves, 256 of them for an 8-bit digit. Below 80 elements per 8-bit digit (640 for 64-bit keys), the counts of up to eight 8-bit digits are kept in rows of eight lanes, one per digit (4 KiB), and turned into places by one vectorized running sum per lane for all digits at once, the first half of the rows summed upwards and the second downwards: fewer passes than `log₂ n`-bit digits take, for about the cost of one sum. This pays when the `s` passes saved against 6-bit digits make `n·s²` at least 144; otherwise, from 24 elements on, 6-bit digits are taken, whose 64 counts are summed while the pass before waits on memory. In lanes, `radix` sorts signed values as their bits, flipping the sign bit once before and once after the passes rather than in every pass. 1.15–1.8 times faster on 16 to 256 random `u64`s (`radix_by_key`: 1.15–1.6 times for 16-byte elements, 1.35–1.5 times for 72-byte ones), 1.1–1.45 times on 32 to 256 random `u32`s. -- One digit is a counting sort: when a single digit holds every bit in which the keys differ, its pass is this crate's [Counting](#counting) sort, with its count tables (`radix`, 2.4–5.4 times faster with four distinct values) or its segments (`radix_by_key`, 1.3–1.8 times faster with four distinct keys). -- Every count in one pass, and no copy back: the counts of all digits are made in one pass, a digit all keys share is skipped, and the passes move the slice into the buffer and back alternately. The counts of a digit become places four at a time, one addition per four waiting for the one before it (1.15–1.2 times faster on 100 random keys). -- Runs of a digit do not wait on each other: the counts walk four quarters of the slice side by side, as they do not depend on the order of the keys (1.6 times faster on nearly sorted input, whose high digits neighbours share), and every pass places four keys at a time, all four counts read before any is written back (1.7–2.2 times faster with four distinct wide keys). -- Large elements are moved once: when `radix_by_key`'s passes would move at least 96 bytes of every element and the slice takes at most 16 MiB, the keys are sorted with the indices of their elements, which are then gathered once (1.25 times faster for 32-byte elements, up to 1.6 times for 72-byte ones). -- No allocation for small sorts: keys, elements and counts (`u16` up to 65,535 elements, `u32` up to `u32::MAX`) each live on the stack up to 4 KiB. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::radix; - -fn main() { - let mut v = vec![i64::MAX, -1, i64::MIN, 0, 1 << 40]; - radix(&mut v); - assert_eq!(v, vec![i64::MIN, -1, 0, 1 << 40, i64::MAX]); -} -``` - -Sorting by a key, stably: - -```rust -use a::sort::radix_by_key; - -fn main() { - let mut v = vec![(3, 'a'), (1, 'b'), (3, 'c'), (0, 'd'), (1, 'e')]; - radix_by_key(&mut v, |&(k, _)| k); - assert_eq!(v, vec![(0, 'd'), (1, 'b'), (1, 'e'), (3, 'a'), (3, 'c')]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst, average and best: O(n·b/r) for keys that differ in `b` bits and digits of `r` bits (`r` ≈ log₂ n, between 8 and 12 from 64 elements on; 6 or 8 for short slices) - - Sorted and reverse-sorted input (strictly descending for `radix_by_key`): O(n), one scan -- Comparisons: none between elements, except the scan for a sorted or reverse-sorted run -- Calls to `key` (`radix_by_key`): exactly n -- Space complexity: O(n) buffers (`radix`: n values; `radix_by_key`: n keys, n elements and, when gathering, 2n keys with indices) plus the counts; each on the stack up to 4 KiB, otherwise allocated once -- Stability: `radix_by_key` is stable; `radix` cannot tell equal values apart -- Against the textbook LSD radix sort (a pass per byte through a buffer copied back each time): `radix` is 1.15–1.8 times faster on random integers from 1000 elements on (1.9–2.3 times at 100 elements and 2.7–3.4 times at 16 against one that allocates its buffer), 2.6–16 times on integers below `n`, 12–35 times with four distinct values and 58–190 times on sorted input; `radix_by_key` is as fast to 1.2 times faster on random `u32` keys of 16-byte elements, 1.45–2.4 times for 72-byte elements and 6–14 times with four distinct keys -- Against the standard library, from 1000 elements on: `radix` is 1.8–5 times faster than `slice::sort_unstable` on random `u32`s, up to 2.3 times on random `u64`s, 3–7.5 times on integers below `n` and 1.1–5 times on nearly sorted input; `radix_by_key` is 1.7–6 times faster than `slice::sort_by_key` on random `u32` keys (16- to 72-byte elements) and up to 2 times on random `u64` keys. On 100 to 256 random `u32`s `radix` is 1.15–1.75 times faster too. Its weaknesses are short slices of wide keys, where every pass costs its counts however few elements it moves (100 random `u64`s sort 1.4 times slower than with `slice::sort_unstable` on fresh input, 2.3 times on the same input over and over; 16 to 32 elements 1.6 (`u32`) to 3.2 (`u64`) times slower; `radix_by_key` on 100 random `u64` keys of 16-byte elements is 1.15 times slower than `slice::sort_by_key`, of 72-byte elements 1.4 times faster), and a few distinct values spread over wide keys, which take every pass while the standard library partitions them away (four distinct random `u32`s take 2–2.9 times as long, `u64`s 2.8–5 times). For keys in a range of about `n` values, [Counting](#counting) sort is 1.07–1.8 times faster - -### Bucket - -Bucket sort, as CLRS describes it, sorts `n` reals spread uniformly over `[0, 1)` by cutting the range into `n` buckets of equal width, moving every element into the bucket its key falls in, sorting each bucket by insertion and reading the buckets out in order: `O(n)` expected time, as each bucket holds about one element. This implementation cuts the range from the smallest to the largest key of each level into one bucket per element, at most 4096 (65,536 for a level of fewer than 131,072 keys that spread over its range), moves the elements into their buckets with a stable counting pass, sorts runs of buckets of at most 16 elements by one insertion sort, and cuts larger buckets again over their own range. Floats are cut by value at the first level; integers, and floats below it, by their bits, which bounds the depth by the width of the key, so that skewed input, which makes the textbook version quadratic, costs at most a few more levels. Keys are primitive integers, `char`s, `bool`s, `f32`s and `f64`s (the sealed `BucketKey` trait), floats sorted as `f64::total_cmp` orders them. It comes in two forms: `bucket` sorts the keys themselves, and `bucket_by_key` sorts any type by such a key, stably, calling `key` exactly once per element. This implementation uses eleven optimizations: - -- Sorted and reverse-sorted input is recognised (non-increasing input too for `bucket_by_key`, whose runs of equal keys are reversed back), and left alone or reversed. -- Keys are read once, as bits that order as they do: signed integers and floats are rewritten in place as unsigned integers in one pass that also finds their range, and back at the end, so every comparison is one of integers. -- Floats crowded at zero are cut by their bits: values over many orders of magnitude crowd near zero when cut by value; when the three buckets there hold more than half the keys, the level is counted again by bits and moved by the cut with the smaller largest bucket (2.6–3 times faster on floats of random exponents). -- Runs of keys in one bucket do not wait on each other: keys are counted and placed four at a time, and a level with at least four keys per bucket counts into four interleaved tables, from 16 into eight (1.8–3 times faster on powers of two, 1.3–2 times with four distinct keys, 1.7 times on nearly sorted input). -- One bucket per key up to 65,536 for keys that spread: cut into 4096 buckets, a level of 5000 to 130,000 keys holds 1 to 32 per bucket, which the insertion sorts or a second level pay for. A level of more than 4096 and fewer than 131,072 keys takes one bucket per key when 32 of its keys at even distances fall in at least 30 different buckets of 4096, unless fewer than 65,536 keys start mostly in order (1.2–2.9 times faster on random and uniform keys and keys below `n` from 10,000 to 130,000 elements; few distinct keys and keys crowding a few buckets keep 4096 buckets). -- Buckets are found once: a level of at most 2048 keys keeps the bucket of each key as it counts it. -- A level of one bucket per key, when there are fewer keys in its range than buckets, is sorted as it is moved (1.5 times faster on keys below `n`). -- Insertion without a branch for keys that move: each chunk of 32 keys is sorted with a branch when few keys moved in the chunk before, and otherwise by writing the larger and the smaller of a key and its neighbour without one (1.2–1.8 times faster on uniform keys). -- Large elements move once: `bucket_by_key` sorts parts of 64 KiB and at most 8192 elements of at least 64 bytes by the indices of their elements, which then move along the cycles of that permutation (1.1–2 times faster), and elements over 16 bytes that keys often pass in a run of small buckets by their positions. -- Keys are read by a loop fit for the size of the elements: a vectorized one up to 16 bytes, and loops that leave at the first key out of order beyond. -- No allocation for small sorts: up to 16 elements are sorted by insertion at once, and buffers and counts (`u16` up to 65,535 elements) of up to 4 KiB each live on the stack; the counts become the starts of the buckets four at a time (1.03–1.07 times faster at 100–1000 elements). - -#### Examples - -Rust usage with integers and floats: - -```rust -use a::sort::bucket; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - bucket(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - - let mut v = vec![0.42, -1.5, 0.0, f64::INFINITY, -0.0, 0.17]; - bucket(&mut v); - assert_eq!(v, vec![-1.5, -0.0, 0.0, 0.17, 0.42, f64::INFINITY]); -} -``` - -Sorting by a key, stably (equal temperatures keep their order): - -```rust -use a::sort::bucket_by_key; - -fn main() { - let mut cities = vec![("Oslo", 4.5), ("Cairo", 31.0), ("Lima", 19.25), ("Kyiv", 4.5)]; - bucket_by_key(&mut cities, |&(_, t)| t); - let names: Vec<&str> = cities.iter().map(|c| c.0).collect(); - assert_eq!(names, vec!["Oslo", "Kyiv", "Lima", "Cairo"]); -} -``` - -#### Efficiency - -- Time complexity: - - Average, for keys spread evenly over their range: O(n), one or two levels of buckets of a few keys - - Worst: O(n·w) for keys of `w` bits: a level divides the range of its keys by its number of buckets, from 17 to 4096 (65,536 below 131,072 keys), so an element passes at most `⌈w / 4⌉` levels (one more for floats), and while its bucket holds more than 4096 keys, a level takes at least 12 bits off - - Sorted and reverse-sorted input (also non-increasing for `bucket_by_key`): O(n), one scan -- Comparisons: none to place elements; those of the insertion sorts within buckets of a few keys, and of the scan for a sorted or reverse-sorted run -- Calls to `key` (`bucket_by_key`): exactly n -- Space complexity: O(n) buffers (`bucket`: n keys; `bucket_by_key`: n keys and n elements, or 2n indices for large elements) plus the counts of a level; each on the stack up to 4 KiB, otherwise allocated -- Stability: `bucket_by_key` is stable; `bucket` cannot tell equal keys apart -- Against the textbook bucket sort (`n` buckets as vectors, each sorted by insertion, concatenated): `bucket` is 6.8–11 times faster on random and uniform `u64`s, 4.9–8 times on uniform `f64`s, 3–4.4 times with four distinct keys and 55–180 times on sorted and reverse-sorted input; `bucket_by_key` is 4.5–8 times faster on uniform `u64` keys of 16-byte elements. On skewed input the textbook version is quadratic: `f64`s of random exponents take it 170 times as long at 10,000 elements, powers of two 7–33 times at 1000 -- Against the standard library, from 100 elements on: `bucket` is 1.3–3 times faster than `slice::sort_unstable` on random and uniform integers, 1.7–3.7 times on uniform floats (against `sort_unstable_by(f64::total_cmp)`), 2–5.9 times on keys below `n` and 1.25–4.8 times on nearly sorted input; `bucket_by_key` is 1.3–3.1 times faster than `slice::sort_by_key` on random `u64` keys of 16-byte elements, 2.3–4.8 times of 72-byte elements and 1.5–3.4 times on random `f64` keys. Its weakness is keys that equal-width buckets do not spread, where the standard library gains from few distinct values: powers of two take 1.3–4.2 times as long, four distinct keys up to 1.55 times from 1000 elements on (`f64`s 1.6–2.6 times), and `f32`s of random exponents (mostly zeros and infinities) up to 1.8 times; 16 elements take about as long. For 32-bit integers on large slices, [Radix](#radix) sort, which does not depend on how the keys are spread, is faster (1.4–1.6 times on random `u32`s from 100,000 elements on), but on random `u64`s it is 1.2–1.35 times slower - -### Pigeonhole - -Pigeonhole sort gives every possible key from the smallest to the largest a hole of its own, puts each element into the hole of its key, and reads the holes out in order: `O(n + k)` time for `k` keys. This implementation keeps the holes in the slice itself: the keys are counted, which gives every hole its place, and each element out of place is then moved straight into the next free place of its hole, the element there taken out in turn, around the cycle until an element lands where the first one was. Every element moves at most once, in place, with no buffer for the elements; it is not stable. Keys are the same as counting sort's (the sealed `IntegerKey` trait), and so is the range rule. It comes in two forms: `pigeonhole` sorts the keys themselves, for which [Counting](#counting) sort, which rewrites equal values from their counts, is 2.2–7 times faster on random values and 1.4–2.25 times on nearly sorted ones, and `pigeonhole_by_key` sorts any type in place by such a key, calling `key` exactly once per element. This implementation uses eight optimizations: - -- Sorted and reverse-sorted input is recognised (non-increasing input too, as the sort is not stable), and left alone or reversed. -- Large slices are placed by swaps: following a cycle, each step waits for the one before it, and for memory once the slice no longer fits the caches. From 32,768 elements on (more for large elements), unless the input is mostly in order, the element at every free place is swapped with the next free place of its own hole instead; the target of a swap depends on its own key only, so the processor overlaps the swaps' waits for memory (1.1–4.3 times faster from 100,000 elements on). Two holes count as one more element towards that length, the counts become the starts of the holes in place, and the holes with free places are listed without a branch (1.35–1.65 times faster from 40,000 elements on, 1.2–1.4 times from 20,000 with four keys per element). -- A few distinct keys are counted in eight tables (at most 256 keys, 16 elements per key) and, with at most 16, placed by swaps at any length, as each step of a cycle would wait on the step before it in the same few holes (2–3 times faster with four distinct keys up to 10,000 elements, 1.5 times beyond). -- Input mostly in order, and ranges of two keys per element or more, walk the places rather than the holes: the cycles start from each place in turn, with no loop per hole whose end mispredicts (3.5–4.5 times faster on nearly sorted input from 100 elements on); when the first keys are mostly in order they are also counted four at a time, so that a run of equal keys waits for memory once per four (twice as fast with sixteen keys per hole). -- Holes move instead of keys: `pigeonhole_by_key` rewrites each key, as it counts it, as its hole in the fewest bytes that hold every hole of the range (1.2–1.8 times faster at 10,000 elements). -- Holes are counted in `u16` up to 65,535 elements, the next free place of each hole needs no clearing, and a range of more than 32 keys per element skips its empty holes eight at a time (1.1–2.2 times faster on a few elements over a wide range). -- Dense ranges walk their holes in a plain loop with no branch on whether a hole is empty, which mispredicts where about a third of them are. -- No allocation for small sorts: keys and counts of up to 4 KiB each live on the stack. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::pigeonhole; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - pigeonhole(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Sorting by a key, in place (elements with equal keys may come in any order): - -```rust -use a::sort::pigeonhole_by_key; - -fn main() { - let mut v = vec![(3, 'a'), (1, 'b'), (0, 'c'), (2, 'd')]; - pigeonhole_by_key(&mut v, |&(k, _)| k); - assert_eq!(v, vec![(0, 'c'), (1, 'b'), (2, 'd'), (3, 'a')]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst, average and best: O(n + k) for `k` possible keys between the smallest and the largest - - Sorted and reverse-sorted input (also non-increasing for `pigeonhole_by_key`): O(n), one scan -- Moves: each element at most once along the cycles, or twice by swaps for large slices and few distinct keys; none for elements already in their holes -- Calls to `key` (`pigeonhole_by_key`): exactly n -- Space complexity: O(k) counts for `pigeonhole`; O(n + k) for `pigeonhole_by_key` (a buffer of n keys, rewritten as holes, and the counts), with no buffer for the elements; each on the stack up to 4 KiB, otherwise allocated once -- Stability: not stable (`pigeonhole` cannot tell equal keys apart anyway) -- Against the textbook pigeonhole sort (a vector per hole, the elements pushed in and read out): 2–6 times faster on random keys and 160–340 times on sorted `u32` input -- Against the standard library, from 100 elements on: `pigeonhole` is 1.15–2.2 times faster than `slice::sort_unstable` on random `u32` keys in a range of about `n` keys from 1000 elements on and 4–7 times on nearly sorted input, between 1.25 times faster and 1.2 times slower on a range of `4n` keys, and as fast to 1.17 times slower with four distinct keys; `pigeonhole_by_key` is 1.05–2.8 times faster than `slice::sort_unstable_by_key` on random `u64` keys of 16-byte elements up to 10,000 elements (1.35–1.45 times beyond), 1.7–5 times for 72-byte elements and 2–16 times on nearly sorted input, but 1.2–2.1 times slower on sorted and reverse-sorted input and 1.1–1.3 times slower with four distinct keys of 16-byte elements. [Counting](#counting) sort's stable `counting_by_key`, which moves the elements through a buffer, is 1.15–3.1 times faster on random keys; `pigeonhole_by_key` is 1.2–2.35 times faster on nearly sorted input of 72- and 128-byte elements, and needs no buffer for them - -### Tree - -Tree sort is a stable, `O(n log n)` sorting algorithm that inserts the elements one by one into a binary search tree, in which the left subtree of every node holds the elements that go before it and the right subtree those that go after, and then reads the tree out in order (an in-order traversal) back into the slice. Binary search trees, and sorting by building one, go back to around 1960 (Windley; Booth and Colin; Hibbard). A plain binary search tree is only as good as the order the elements come in: sorted input builds a list and costs n²/2 comparisons. This implementation keeps the tree an AVL tree (Adelson-Velsky and Landis, 1962), the first self-balancing binary search tree: the two subtrees of every node differ in height by at most one, restored after each insertion by at most one single or double rotation, so the tree stays below 1.44·log₂ n levels and every insertion costs O(log n) whatever the input. An element equal to one already in the tree goes to its right, after it, which makes the sort stable. AVL trees were measured against red-black trees, whose looser balance makes them up to twice as high: within 10% of each other on random input, but red-black trees were 1.15–2.4 times slower on sorted, nearly sorted and sawtooth input and on input with few distinct keys. This implementation uses seven optimizations: - -- One array of nodes instead of an allocation per node: node `i` belongs to the element that started at index `i`, so the nodes form one array with 32-bit indices as links and the balance factors in a separate byte array that only rebalancing reads; searches and the in-order traversal are loops with an explicit stack. Elements are moved twice and only twice, whatever the input. Against the textbook AVL tree sort, which allocates a node per element and inserts recursively, this alone (the crate's first version of this sort) is 1.2–4 times faster on random input and up to 30 times on sorted strings. -- Small elements live in the nodes: elements of 5 to 8 and of 17 to 24 bytes are copied into nodes of 16 and 32 bytes, so a search step loads a node and its key from one place; other elements stay in the slice, where they are compared, and are gathered into a buffer in order at the end (their nodes would waste a quarter or more on padding, and elements of up to 4 bytes or of 16 bytes are addressed by a shift in the slice too). This is 1.03–1.07 times faster on random input of 10,000 elements and more. -- Branchless search steps: a step loads both links of a node with its key and the comparison selects one, instead of a branch that random input mispredicts half the time or a load that waits for the comparison: 1.02–1.11 times faster up to 10,000 elements. -- Searches side by side: a search is a chain of dependent loads and comparisons that the processor cannot overlap with itself, so the places of the next 2 elements (4 from 1024 elements on, 8 from 262,144 on) are searched for at once in the same tree, and the elements are then inserted one after the other, in order. An insertion can only invalidate a later search through the slot it takes or a node it rotates; such a search goes on from the deepest node of its path still in place, usually a level or two above its slot, so the tree is exactly the one that one-by-one insertion builds, at up to 2% more comparisons with few distinct keys. On random input this is 1.1–3.7 times faster than one search at a time (most for strings and large slices), and 1.03–1.6 times with 16 distinct keys, except for 64-byte elements at 1,000,000 (1.08 times slower). Searching again from the root instead of going on is 1.05–1.2 times slower with few distinct keys. -- Insertion beside the last insertion: if the last element inserted went right after or right before the one inserted before it, the next one is first tried right after or right before it in turn. Two comparisons, with the last element and its neighbour in order, say whether it belongs there; the path to that slot is the last element's, kept up to date through the rotations, so no search is needed. Sorted, reverse-sorted and all-equal input costs n−1 comparisons, and an ascending or descending run costs two per element after its first, wherever it lands. This is 1.9–8 times faster on sorted and reverse-sorted input and 1.6–3.6 times on nearly sorted input, at up to 5% on random input. -- Short shifts of the kept path inline: a rotation removes a node from the path to the last insertion, usually a level or two above it; moving the one or two entries below it inline instead of calling `memmove` is 1.1–1.3 times faster on sorted and reverse-sorted input. -- No allocation for small slices: up to 4 KiB of nodes (240 nodes of 8-byte elements) live on the stack, which is 2.1–3.4 times faster for 2 elements, 1.1–1.4 times for 10 and up to 1.27 times for 30; beyond, nodes, balance factors and the gather buffer take one allocation. - -Together, against the textbook tree sort, which builds an unbalanced binary search tree with a node allocated per element: 1.5–4.4 times faster on random input of 100 to 1,000,000 elements, and 1,600–2,500 times on 10,000 sorted elements, where the textbook sort makes n²/2 comparisons. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::tree; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - tree(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Rust usage with strings: - -```rust -use a::sort::tree; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - tree(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -Equal keys keep their original order (cards are compared by rank only): - -```rust -use a::sort::tree; -use std::cmp::Ordering; - -#[derive(Debug, PartialEq, Eq)] -struct Card(u8, char); - -impl Ord for Card { - fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0) - } -} - -impl PartialOrd for Card { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -fn main() { - let ranks = [3, 1, 3, 0, 1, 3, 0, 2, 1, 0]; - let mut cards: Vec = "abcdefghij".chars().zip(ranks).map(|(c, r)| Card(r, c)).collect(); - tree(&mut cards); - let order: String = cards.iter().map(|c| c.1).collect(); - assert_eq!(order, "dgjbeihacf"); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n log n) - - Average: O(n log n) - - Best: O(n), when every element goes right after or right before the one before it (sorted, reverse-sorted or all-equal input) -- Comparisons: - - Best (sorted, reverse sorted, all equal): n−1 - - Runs: at most 2 per element, plus a search for the first element of each run - - Random input: about n·log₂ n − 1.2n (≈ 8,730 for n = 1000, against ≈ 10,000–11,000 for the textbook tree sort, which makes 499,500 on sorted input) - - Worst: O(n log n), since a search takes fewer than 1.44·log₂ n steps, the height of an AVL tree, on any input -- Moves: exactly 2n, whatever the input (into the nodes and back, or gathered into a buffer and copied back) -- Space complexity: O(n) extra: a node of 8 bytes per element (16 or 32 bytes with the element inside) and a balance factor, on the stack up to 4 KiB and allocated once beyond, plus a buffer of n elements for elements that stay in the slice; O(log n) stack -- Stability: stable (equal elements keep their relative order) -- Adaptivity: adaptive to runs, which cost at most two comparisons per element. A binary search tree is a poor way to sort what fits in an array, though: every search step in a large tree is a cache miss and every insertion pays for rebalancing. `slice::sort` is 1.5–5 times faster on random input, 2–17 times with few distinct keys and 6–30 times on sorted input; this crate's [Merge](#merge) and [Tim](#tim) sorts are 1.25–4.5 times faster on random integers and 64-byte elements, and on random strings Merge sort is 1.3–1.8 times faster and Tim sort 1.05–1.4 times. Tree sort moves every element exactly twice, against n·log₂ n times for a merge sort, and is up to 1.9 times faster than `slice::sort` on nearly sorted strings and 1.4–1.5 times on nearly sorted 64-byte elements - -### Cycle - -Cycle sort, described by B. K. Haddon in 1990, is an unstable, in-place sorting algorithm that writes to the slice as few times as any sort can. The permutation that sorts the slice splits into cycles (the element in slot `a` belongs in slot `b`, the one in `b` in `c`, and so on back to `a`), and each cycle is rotated once: the element of the cycle's first slot is taken out, its final slot is found by counting the elements smaller than it, it is written there and the element it displaces is taken in turn, until the element in hand belongs in the first slot. Equal keys share a block of slots, and an element goes to the first slot of its key's block that does not already hold an equal element. Every element is written at most once, straight into its final slot, and an element whose slot already holds its key is not written at all, which is the point where writes are slow or wear the memory out (flash, EEPROM). The price is the counting: `Θ(n²)` comparisons on any input that is not in order already. This implementation writes the same elements to the same slots as the textbook version (in the same order, except on strictly decreasing input) and uses six optimizations: - -- Written cycle starts are passed over: after a cycle, the textbook takes the next slot as the next cycle start and counts for its element even if an earlier cycle wrote it, though such an element is final and the count finds nothing smaller. That is a third of all comparisons on random input (about 0.5n² of 1.5n²). A 4 KiB bitmap on the stack (a window of 32,768 slots) remembers the written slots: 1.23–1.25 times faster on 100 random `u64`s, 1.45–1.5 times from 1000 on and for strings and 64-byte elements, 1.15–1.25 times with four distinct keys from 1000 elements on. On nearly sorted input it gains nothing, and its bookkeeping (about 40 ns a sort) makes 100 `u64`s 1.08 times slower. -- Runs of written starts are passed over at once: a written start is recognised by its bit with a branch, and the whole run of written slots after it is passed over a word at a time (1.03–1.04 times faster on 100 `u64`s than one bit per start; scanning the bits from every start with no branch made each count wait and was up to 1.3 times slower on nearly sorted input). -- Sorted and reverse-sorted input is recognised: a non-decreasing slice is left alone and a strictly decreasing one is reversed, which writes the same elements to the same slots as the cycles would, at n−1 or n comparisons instead of `Θ(n²)` (18–460 times faster from 100 to 3000 elements; usually two comparisons on other input). -- The element in hand: the textbook in safe generic Rust keeps the element being placed in the cycle's first slot and swaps, writing that slot again at every step; here the element is moved out once and the first slot stays vacant until the cycle closes, so each element placed costs exactly one write (a guard puts the element back if a comparison panics). The speed is the same, within 4%. -- Long runs of equal small elements are skipped 16 at a time: for elements of up to 8 bytes, after 32 equal elements one at a time, they are compared 16 at a time in a few vector instructions (1.06 times faster on 1000 `u64`s with four distinct keys, 1.17 times on 3000; at most 1.06 times slower on 200–300 of them). -- Narrow elements are counted in counters as narrow as they are: the counts compare every element with no branch on the answer, so integers are compared several at a time, but into a `usize` the compiler widened the answer of every 1-, 2- and 4-byte element to 64 bits, two elements per vector instruction. Counting them into `u8`, `u16` or `u32` counters, added to the total every 192 elements for `u8` and every 32,768 for the others, compares 16, 8 or 4 at a time: on random input 2.6–3.3 times faster for `u32` from 1000 elements on, 4.1–6.6 times for `u16` and 3.9–5 times for `u8` (1.3–1.65 times on 100 elements), and 2.2–5.2 times with four distinct keys. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::cycle; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - cycle(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Only the elements in the wrong slot are written (here the 0 and the 2): - -```rust -use a::sort::cycle; - -fn main() { - let mut v = vec![2, 1, 0, 3]; - cycle(&mut v); - assert_eq!(v, vec![0, 1, 2, 3]); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) - - Average: O(n^2) - - Best: O(n), when the input is non-decreasing or strictly decreasing -- Comparisons: - - Non-decreasing input: n−1; strictly decreasing: n - - Random distinct keys: about n² (the textbook cycle sort: about 1.5n²); with duplicates, placing an element also compares it with the equal elements already in its key's block - - Worst: Θ(n²) -- Writes: exactly one per element whose slot does not already hold its key, and none for any other element: the fewest any sort can make (at most n; none for sorted input, 2⌊n/2⌋ for strictly decreasing input) -- Space complexity: O(1): a 4 KiB bitmap on the stack, no allocation (in-place) -- Stability: unstable (equal elements may be reordered) -- Against the textbook cycle sort: 1.2–1.3 times faster on 100 random `u64`s, 1.45–1.5 times from 1000 on and on random strings and 64-byte elements, 1.15–1.45 times with four distinct keys from 1000 elements on, 11–7000 times on sorted and reverse-sorted input, and as fast on nearly sorted input from 1000 elements on; on up to 200 nearly sorted `u64`s the textbook is 1.08–1.2 times faster, and on 100 `u64`s with four distinct keys 1.04 times. Elements of at most 4 bytes gain another 2.6–6.6 times from 1000 elements on (see above) -- Against [Selection](#selection) sort (at most n−1 swaps, so up to 2(n−1) writes): 2–2.6 times faster on random `u64`s and 4–5.1 times on nearly sorted ones; but selection sort is 1.1–2.3 times faster on random strings and 64-byte elements, 1.05–1.2 times on nearly sorted ones and 1.6–2.5 times on sawtooth ones, and, as it gathers equal keys in each pass, 2.25–700 times faster with four distinct keys -- Against the standard library: quadratic, so only worth it where a write costs far more than a comparison; `slice::sort_unstable` is 3–6 times faster on 100 random elements, 12–50 times on 1000 and 90–200 times on 10,000 (125 times on 3000 strings; 17–30 times on 5000 random `u32`s, `u16`s or `u8`s), and as fast on sorted and reverse-sorted input - -### Bitonic - -Bitonic sort, published by Kenneth Batcher in 1968, is a sorting network: a fixed sequence of compare-exchanges, each putting the elements at two given positions in order, chosen in advance and never depending on what the comparisons answer. A sequence that first rises and then falls is bitonic; one stage of compare-exchanges, of each element of the first half with the element half a slice further on, splits it into two bitonic halves with nothing in the first greater than anything in the second, and repeating this on the halves merges it. Bitonic sort sorts the two halves in opposite directions, which makes the whole slice bitonic, and merges it: `n·k·(k + 1) / 4` compare-exchanges for `n = 2^k`, in `k·(k + 1) / 2` stages whose compare-exchanges are independent of each other, which is why it suits hardware, vector units, GPUs and threads. Batcher's network is for powers of two; for other lengths this is H. W. Lang's formulation (the first `⌊n/2⌋` elements sorted descending, the rest ascending, and the merge of `n` elements comparing `i` with `i + m` for `m` the largest power of two below `n`), which is the network of the next power of two without the compare-exchanges that would involve missing elements, so nothing is padded and any `T` can be sorted. It is unstable and in place, and not adaptive: every input of a length gets the same compare-exchanges. It comes in two forms: `bitonic` on the calling thread, and `bitonic_parallel`, which makes exactly the same compare-exchanges on several threads. This implementation uses five optimizations, all of which change only how the network is carried out, never which compare-exchanges it makes: - -- Small networks in straight-line code: sorts of at most 16 elements and merges of at most 16 run their network with every compare-exchange written out at constant positions, so small elements stay in registers (2.2–4.2 times faster for integers than recursing down to single elements, 1.6–2 times for strings, 1.35–1.6 times for 64-byte elements; 1.9–2.4, 1.1–1.15 and 1.35–1.55 times faster than a loop over the same networks). -- Two or three stages per pass: a merge of a power-of-two length runs its first two or three stages as one pass over groups of 4 or 8 elements, each read once, put through the 4 or 12 compare-exchanges of those stages and written once, then merges its parts depth first, so the later stages run in cache (three stages make integers 1.04–1.14 times faster, two make 64-byte elements 1.03–1.08 times faster; strings, whose comparisons cost more than the passes, take one). -- Compare-exchanges without branches: elements of at most 8 bytes are exchanged by selecting the values to write back, which lets the compiler run a pass's independent compare-exchanges in vector registers (4.6–5 times faster than branching). Within a merge, whether a compare-exchange swaps comes in long runs, but in the small networks it is a coin toss on random input, so elements of 9 to 24 bytes, such as `String`, are exchanged there by selecting the addresses to copy from and by a branch in the passes (1.25–1.45 times faster for random strings, 1.05–1.1 times for nearly sorted ones; `u128` and pairs gain 1.4–1.9 times on random input but lose up to 1.8 times on nearly sorted input, and arrays, whose comparison is itself a chain of branches, lose up to 1.4 times). Larger elements branch. -- The direction is a type: each part is sorted ascending or descending through a type parameter rather than a flag, so every compare-exchange is compiled for its direction. -- Narrow elements are merged stage by stage: for elements of at most 4 bytes, a merge of a power-of-two length of at most 2048 runs its stages one after the other, each as one loop over the whole merge, which stays in the first-level cache, so that every stage, down to the one pairing neighbours, runs as vector minimums and maximums of 4 to 16 elements at once instead of the one pair at a time of the straight-line merges of 16 (1.25 times faster for `u32`, `u16` and `u8` at 1000 elements, 1.45–1.6 times at 16,384 and 1.3–1.55 times at a million, on every input; for `u64`, two to a vector, it was 1.1–1.25 times slower). - -`bitonic_parallel` uses one thread per 16,384 elements of at most 8 bytes, or per 4,096 larger elements, up to `std::thread::available_parallelism`; shorter slices are sorted by `bitonic`, with no allocation. It cuts the slice into parts along the sort's own halving, two or four per thread, sorts them in a first round, and then takes two rounds per level of merges: the first runs the stages that compare elements of different parts, one range of positions at a time across all parts, the second merges each part on its own. The threads are spawned once and kept for all rounds (spawning them for each round instead was 1.15–1.6 times slower from 32,768 to 131,072 elements, and 1.2 times for a million integers), take the tasks of a round, which touch disjoint elements, one at a time until none is left, and meet at a barrier between rounds. A panic on any thread lets the others finish their current tasks, skips the rest and is propagated to the caller, with every element still in the slice exactly once. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::bitonic; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - bitonic(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); -} -``` - -Rust usage with strings: - -```rust -use a::sort::bitonic; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - bitonic(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -On all cores (the same compare-exchanges, so the same result, equal elements included): - -```rust -use a::sort::bitonic_parallel; - -fn main() { - let mut v: Vec = (0..1_000_000).map(|i| i * 7919 % 1_000_003).collect(); - bitonic_parallel(&mut v); - assert!(v.windows(2).all(|w| w[0] <= w[1])); -} -``` - -#### Efficiency - -- Time complexity: - - Worst, average and best: O(n log² n), the same on every input - - Parallel: `k·(k + 1) / 2` stages of independent compare-exchanges for `2^(k−1) < n ≤ 2^k`, O(log² n) with enough processors -- Comparisons: exactly the network's, on every input: `n·k·(k + 1) / 4` for `n = 2^k`, between `n·(k − 1)² / 4` and `n·k·(k + 1) / 4` for `2^(k−1) < n < 2^k` (26,984 for n = 1000 against about 10,500 for [Heap](#heap) sort, 14,600 for elements of at most 8 bytes; 102,931,040 for n = 1,000,000, about 5.2·n·log₂ n) -- Space complexity: O(log n) stack, no allocation (in-place); `bitonic_parallel` allocates its list of tasks and its threads -- Stability: unstable (equal elements may be reordered) -- Adaptivity: none; sorted input costs as much as random input -- Against the textbook bitonic sort (Lang's recursion, with the direction as a flag and a branch and a swap for each compare-exchange): 10.5–11.8 times faster on random integers and 4.8–9.8 times on other input, 1.75–2.1 times on random strings and 1.3–1.6 times on other strings, 1.4–1.8 times for random 64-byte elements and 1.05–1.5 times on other input -- Against the standard library: as fast as `slice::sort_unstable` on random integers around 1000 elements (and 1.25 times faster than [Quick](#quick) sort), but its `log₂ n` factor shows from there on: `sort_unstable` is 1.35 times faster at 16,384 integers and 1.85 times at a million, 20–70 times faster on sorted input (random `u32`s and `u16`s, merged stage by stage, are sorted 1.1–1.3 times faster than by `sort_unstable` from 1000 to 16,384 elements and 1.25–1.45 times slower at a million), 2.5–5 times faster on random strings and 2.3–4.1 times for random 64-byte elements. Against [Shell](#shell) sort: 2.25–3.6 times faster on random integers from 1000 elements to a million, as fast on random strings, 1.35–1.6 times slower for 64-byte elements -- `bitonic_parallel` on the 16 threads of an Apple M4 Max (12 performance and 4 efficiency cores), against `bitonic`: integers 1.6 times faster at 32,768 elements, 2.5 times at 65,536, 2.9–3.2 times up to 262,144, 7.2 times at a million and 8.8 times at four million; strings 1.8 times faster at 8,192 elements, 3.3 times at 16,384, 4.4–5.7 times at 32,768 and 10–11 times from 262,144 on; 64-byte elements 1.7 times faster at 8,192 elements, 2.9 times at 16,384 and 8–9.7 times from 262,144 on (between 262,144 and a million integers, 8 to 12 threads would be up to 1.25 times faster than 16, as each round waits for its slowest thread). Against the single-threaded `slice::sort_unstable` it is faster from 32,768 elements on: for integers 1.2 times there and 3.9 times at a million, for strings 1.15 and 2.2 times, for 64-byte elements 1.05 and 2.4 times. The stage-by-stage merges make `bitonic_parallel` 1.05–1.25 times faster for `u32` and 1.15–1.35 times for `u8` from 65,536 to a million elements - -### Odd-Even - -Odd-even sort, also called brick sort or odd-even transposition sort, is a simple, stable, in-place sorting algorithm that compares neighbours in alternating phases: an odd phase compare-exchanges the pairs (1, 2), (3, 4), …, and an even phase the pairs (0, 1), (2, 3), …, swapping a pair if its second element is smaller. An odd and an even phase make a round, and the textbook version repeats rounds until one swaps nothing. No two pairs of a phase share an element, so all compare-exchanges of a phase are independent: the algorithm comes from parallel computing, where a row of processors that only talk to their neighbours sorts n elements in n phases (Habermann, 1972). On one processor it is bubble sort with each pass cut into independent pairs; `odd_even_parallel` runs it on several threads. This implementation uses seven optimizations, none of which changes which compare-exchanges run in which phase (every swap is one the textbook version makes): - -- Bounds from the last swaps: a pair that is in order stays in order until one of its elements changes, so the odd phase of a round only compares the pairs from one before the first to one after the last pair the previous even phase swapped, and the even phase one pair more at each end; the textbook compares all n−1 pairs every round. This saves 7% of the comparisons on 1000 random elements, makes nearly sorted input 1.25–1.8 times faster and input with few distinct keys 1.2–1.4 times, and an element out of place at either end (a turtle or a rabbit) costs about 3.5n comparisons instead of about n²/2. -- At most ⌈n/2⌉ rounds: n phases sort any input (the 0-1 principle), so the sort stops then even if the last round swapped. This saves the textbook's final round on input that needs every phase, such as reverse-sorted input, and guarantees that the sort ends with an inconsistent `Ord`, whose answers would otherwise almost never let a round pass without a swap. -- One sweep per round: the even pair (k, k+1) depends only on the odd pairs on either side of it, so one sweep compares an odd pair and then the even pair just behind it — a wavefront making the same compare-exchanges on the same elements as two separate phases, while loading each element once per round instead of twice: 2–2.2 times faster for integers, up to 1.7 times for strings and 1.6 times for 64-byte elements. -- Branchless compare-exchanges for elements of at most 8 bytes: both elements are read and the smaller and the larger written back, chosen by the comparison, so that a chunk of 32 odd and 32 even pairs compiles to vector code for integers and reports its swaps as a bit mask. 1.3–2 times faster than branching on random input or few distinct keys (2–10% slower on nearly sorted input, where branches are predictable); 16-byte elements are 2.4–3.9 times slower without branches, so larger elements keep them. -- Elements of at most 4 bytes split into even and odd positions: a chunk of 32 steps is moved out of the slice into two small arrays, one for the elements at even positions and one for those at odd ones, and back after both phases, with the element where the next chunk starts carried over in a register. Every pair of a phase then sits at the same place of the two arrays (one place apart for the odd phase), and the compiler runs each phase as a few vector minimums and maximums, where in the slice it picked lanes apart and widened every swap bit to 64 bits. The same pairs are compared and the same swaps made (the elements are compared where the chunk holds them, outside the slice): 1.3–1.5 times faster for 4-byte elements from 1000 elements on, 1.7–2.35 times for 2-byte and 3.3–4.3 times for 1-byte ones (1.05–1.5 times on 100); for `u64` it made no difference. -- For larger elements, swaps by plain copies with the even phase two pairs behind the odd one: `mem::swap` compiled to copies the processor could not forward to the next comparison (1.2–1.9 times slower for 16- and 64-byte elements), and an even phase right behind the odd one was 1.07–1.45 times slower for 64-byte elements. -- Parallel form: `odd_even_parallel` cuts the slice into blocks that threads (`std::thread::scope`, no dependencies) claim in turn; on its block a thread runs a super-step of K phases, each leaving out one more pair at each interior end (a trapezoid), and in the next super-step the slice is cut at the midpoints of the previous blocks, so that each thread first completes the triangle of pairs left out around one old boundary and then runs the next trapezoid. Every compare-exchange still runs exactly once, after the two it depends on, so this is still odd-even transposition sort (not block merge-split sort), with one barrier per super-step instead of one per phase. Short slices and input that is sorted within the first rounds never start a thread; a panic in any thread reaches the caller with its payload. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::odd_even; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8]; - odd_even(&mut v); - assert_eq!(v, vec![1, 2, 4, 5, 8]); -} -``` - -A turtle (the 0 at the end moves two slots per round, and each round only compares around it): - -```rust -use a::sort::odd_even; - -fn main() { - let mut v = vec![1, 2, 3, 4, 5, 0]; - odd_even(&mut v); - assert_eq!(v, vec![0, 1, 2, 3, 4, 5]); -} -``` - -On several threads: - -```rust -use a::sort::odd_even_parallel; - -fn main() { - let mut v: Vec = (0..10_000).rev().collect(); - odd_even_parallel(&mut v); - assert!(v.windows(2).all(|w| w[0] <= w[1])); -} -``` - -#### Efficiency - -- Time complexity: - - Worst: O(n^2) - - Average: O(n^2) - - Best: O(n), when the input is already sorted - - Parallel: O(n^2 / p) work per thread on p threads, with one barrier per super-step -- Comparisons and swaps (`I` = number of inversions): - - Any input: exactly I swaps (each removes one inversion), as the textbook version; never more comparisons than the textbook version - - Best (sorted): n−1 comparisons, 0 swaps - - Reverse sorted: n(n−1)/2 comparisons for even n, (n²−1)/2 for odd n, every one a swap; the textbook version makes n−1 more - - Turtle `[1, 2, …, n−1, 0]`: (7n−16)/2 comparisons for even n, (7n−13)/2 for odd n, against (n−1)(⌊n/2⌋+1) for the textbook version (3,492 against 500,499 for n = 1000) - - Random input: about 453,400 comparisons for n = 1000 (textbook ≈ 486,800) and about n²/4 swaps -- Space complexity: O(1) extra (in-place); the parallel form allocates only for its threads and a block plan -- Stability: stable (only neighbours are swapped, and only when the second is strictly smaller) -- Adaptivity: the bounds shrink to the part still changing; good for nearly sorted input whose displaced elements have not far to go, but a far-displaced element still costs a phase per slot -- Against the textbook version (1000 and 5000 elements): 1.15–3.4 times faster for integers (2–2.6 on random), 1.03–1.75 times for strings, 1.0–2.1 times for 64-byte elements; elements of at most 4 bytes gain another 1.3–4.3 times from splitting chunks into even and odd positions -- Against [Bubble](#bubble): 1.6–1.7 times faster on random integers, but on strings and 64-byte elements from 1.1 times faster to 2 times slower (bubble sort shifts an element with one write per slot instead of swapping). Against [Cocktail Shaker](#cocktail-shaker): 1.45 times faster on random integers and 2.1 times on few distinct keys, but 1.2–1.4 times slower on random strings, 1.5–2.2 times on 64-byte elements and 20–50 times on nearly sorted input, which cocktail shaker sort fixes in a few passes -- Against `slice::sort_unstable`: about 20 times slower on 1000 random integers and 85 times on 5000 (on random `u32`s about 11 and 45 times, on `u8`s 8 and 30 times) — prefer `slice::sort` for real workloads -- Parallel form against the sequential one (16-core machine, 12 performance cores, under other load): 4.3–6 times faster on 20,000 integers, 7–8 times on strings and 6.6–10.5 times on 64-byte elements; 1.9–3.8, 4.4–6.5 and 3.5–6.6 times on 8192; 1.3–2.1, 2.7–4 and 2–4.4 times on 4096. It uses threads from 4096 elements of at most 8 bytes and 2048 larger ones, after the first 64 rounds (or 2²⁰ / 2¹⁶ compared pairs) on the calling thread. Splitting chunks into even and odd positions makes it 1.2–1.25 times faster for `u32` and 2.1–2.2 times for `u8` on 8192 and 20,000 elements - -### Stooge - -Stooge sort is a deliberately slow, unstable, in-place sorting algorithm, mostly of educational value: an example of a short, correct algorithm with a surprising running time. It compares the first and the last element and swaps them if they are out of order; then, if the slice has at least three elements, it stooge-sorts the first two thirds (rounded up), the last two thirds, and the first two thirds again. The first call moves every element of the largest third that starts in the first two thirds into the middle, so after the second call all of them sit at the end, in order, and the third call sorts the rest. It is Problem 7-3 of Cormen, Leiserson, Rivest and Stein's *Introduction to Algorithms*, where Professors Howard, Fine and Howard (the Three Stooges) propose it as an "elegant" sorting algorithm, and the reader shows that it takes Θ(n^(log 3 / log 1.5)) ≈ Θ(n^2.71) time, worse than bubble sort. Which elements are compared depends only on the length of the slice, never on the answers: stooge sort is a sorting network, one fixed sequence of compare-exchanges for each length. This implementation makes exactly the textbook's comparisons, in the same order, with the same swaps, and uses three optimizations in executing them: - -- Unrolled bottom levels: all parts at one depth of the recursion have the same length, so the parts four levels above the bottom have 7, 8 or 9 elements throughout one sort; they run as straight-line code, the 121 compare-exchanges below them at constant indices, with no calls and no index checks. The textbook makes one call per compare-exchange, and the calls are its cost: recursing down to two elements without index checks is only 1.05–1.15 times faster than it, while unrolling four levels is 5–8 times faster for `u64`, 1.4–1.65 times for strings and 1.4–2.3 times for 64-byte elements than recursing down to two elements. -- A branch, not a branchless select: only about 1% of the compare-exchanges swap anything (4.5% at 16 random elements, 0.2% with few distinct keys), so the branch is predicted almost always and the compare-exchanges do not wait for one another; a branchless select, which chains each to the one before, was 2 times slower than the textbook for `u64`, 4 times for strings and 1.8 times for 64-byte elements. -- The swap is marked cold, so the compare-exchanges that do not swap are laid out as one straight line: 1.5–1.85 times faster for 64-byte elements, 1.05–1.25 times for strings and 1.1–1.25 times for `u64` from 100 elements on. - -The code is safe Rust, with no `unsafe` block. - -#### Examples - -Rust usage with integers: - -```rust -use a::sort::stooge; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0]; - stooge(&mut v); - assert_eq!(v, vec![0, 1, 2, 4, 5, 8]); -} -``` - -Sorting strings: - -```rust -use a::sort::stooge; - -fn main() { - let mut words = vec!["pear", "apple", "fig"]; - stooge(&mut words); - assert_eq!(words, vec!["apple", "fig", "pear"]); -} -``` - -#### Efficiency - -- Time complexity: Θ(n^(log 3 / log 1.5)) ≈ Θ(n^2.71) on every input (worst, average and best alike) -- Comparisons: exactly C(n) = (3^(D+1) − 1)/2 on every input of n ≥ 2 elements, where D is the number of times m ↦ m − ⌊m/3⌋ takes n to 2 (equivalently C(2) = 1, C(n) = 1 + 3·C(n − ⌊n/3⌋)): 13 for n = 4, 121 for n = 8, 1,093 for n = 16, 265,720 for n = 100 (and for 128: D grows in steps), 64,570,081 for n = 1000, against at most n(n−1)/2 = 499,500 for bubble sort -- Swaps: at most C(n), depending on the input; about 1% of the comparisons on random input; none on already-sorted input -- Space complexity: O(log n) stack, no allocation (in-place); the recursion nests D − 4 levels for n ≥ 10 (12 for n = 1000) -- Stability: unstable (`[1a, 1b, 0]` becomes `[0, 1b, 1a]`) -- Adaptivity: none; the same comparisons on every input of a given length -- Against the textbook stooge sort (recursive, one call per compare-exchange): 9–11 times faster for `u64` from 64 elements on (3–10 times below), 1.7–2.2 times for strings, and 4.3–5 times for 64-byte elements from 32 on (2.5–4.5 times below; 1.4–2 times with few distinct keys, where every comparison reads all 64 bytes) -- Against the standard library: `slice::sort_unstable` is 1.2–4 times faster on 8 elements, 50–130 times on 100 random elements, 900–2400 times on 1000 and about 4000 times on 2000 random `u64`; on sorted input, which std recognises in one pass, 10,000–70,000 times from 500 elements on - -### Sleep - -Sleep sort, posted anonymously to 4chan's programming board in 2011, is a joke: a shell script that started `sleep $n; echo $n` in the background for every argument `n`, so that the small numbers wake, and are printed, first. It sorts in "linear time", lets the operating system's timer queue do all the work, and is wrong as soon as the scheduler is late. `sleep` is that algorithm with real threads and real sleeping: one thread per element sleeps `(value − min) × unit` past a start all threads share, and the values are written back in the order their threads wake. Keys are the same as counting sort's (the sealed `IntegerKey` trait), so negative numbers, `char`s and `bool`s work too, and the smallest values wake right at the start. - -**It is nondeterministic and intended as a curiosity.** A thread scheduled late lets a larger value overtake a smaller one, so the slice may come out unsorted; `sleep` returns whether it did, `true` exactly when the slice ends up in order. Whatever happens, the slice ends up a permutation of what it was: every value is written back exactly once. It panics, leaving the slice as it was, if the clock cannot represent when the largest value would wake (about 292 billion years on Unix-like platforms: the whole range of `i128` at a nanosecond per unit, or of `u64` at a second; a zero unit never panics) or if the system cannot start a thread for every element (16,384 per process on macOS), after sending home the threads already started. A range the clock can represent is slept through, however long: the whole range of `i64` at a nanosecond per unit takes 584 years. This implementation uses four optimizations, for faithfulness and robustness rather than speed (the success rates are for random keys in `0..n` on a 16-core Mac loaded by other benchmarks): - -- One shared start: every thread sleeps until `start + (value − min) × unit`, not for a duration from whenever it happened to be started, so the order and the cost of starting the threads (15 µs each for 1000 of them) do not skew when they wake. Sleeping from each thread's own start, 100 elements at 100 µs per unit and 1000 elements at 1 ms never came out sorted; from the shared start, almost always. -- The clock starts once every thread is running, and a little later: all threads are started first and wait at a gate, a `RwLock` the caller holds for writing, and the start is set 4 µs per thread after the gate opens, as waking the waiting threads takes the system 1.5–3 µs each. With the start at the gate, the smallest values, released last, woke after the next ones: 1000 elements at 100 µs never came out sorted (100 elements: 64%; 1000 at 1 ms: 30%). -- Sleeping in halves: the system may let a sleep run longer than asked, to wake several threads together (here 1 ms took 1.5 ms, 5 ms took 7.5 ms, 50 ms up to 60 ms). Each thread sleeps for half the time left until its deadline, again and again, and for all of it once less than 50 µs is left, so a late sleep still ends before the deadline and the last, short one is late by a few dozen microseconds at most. At 1 ms per unit: from 39% to 100% sorted for 10 elements, from none to all for 100, and from none to 60–95% for 1000. Spinning through the last microseconds instead was no better. -- Small stacks: each thread gets a 16 KiB stack instead of the default 2 MiB, which halves the time to start 5000 threads or more (0.27 s instead of 0.55 s for 5000). - -The values are written back lock-free, each waking thread taking the next place from an atomic counter, and `(value − min) × unit` is computed in 128 bits, saturating at `Duration::MAX`. - -#### Examples - -Rust usage with integers (a unit of a few milliseconds almost always sorts a few values): - -```rust -use a::sort::sleep; -use std::time::Duration; - -fn main() { - let mut v = vec![5, 1, 4, 2, 8, 0, 3, 9, 7, 6]; - let sorted = sleep(&mut v, Duration::from_millis(2)); - if sorted { - assert_eq!(v, vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - } else { - println!("the scheduler was late: {v:?}"); - } -} -``` - -Negative numbers sleep from the smallest one, and the return value always tells the truth: - -```rust -use a::sort::sleep; -use std::time::Duration; - -fn main() { - let mut v = vec![3, -2, 0, -5, 1]; - let sorted = sleep(&mut v, Duration::from_millis(1)); - assert_eq!(sorted, v.is_sorted()); -} -``` - -#### Efficiency - -- Time complexity: `(max − min) × unit` of sleeping, plus starting n threads (about 15 µs each up to 1000 threads, more beyond: 0.27 s for 5000, 1 s for 10,000) and 4 µs per thread before the start, whatever the order of the input: "linear" in the range of the values, not in their number -- Comparisons: none between elements (one pass for the smallest and largest value, one to report whether the result is sorted) -- Threads: exactly n, one per element, all alive at once -- Space complexity: O(n): a 16 KiB stack per thread; the values move through no buffer -- Correctness probability (share of sorts that came out sorted, random keys in `0..n`, over three runs of 20–400 sorts on a loaded 16-core Mac): - - | `unit` | 10 elements | 100 elements | 1000 elements | - |--------|-------------|--------------|---------------| - | 10 µs | 12–18% | 0% | 0% | - | 100 µs | 94–100% | 94–100% | 89–100% | - | 1 ms | 99–100% | 89–100% | 60–90% | - - A unit of 10 µs is below what the timers resolve (a thread wakes 10–15 µs after its deadline at best); a longer sort is longer exposed to the other work of the machine, so 1000 elements at 1 ms (a second) came out sorted less often than at 100 µs (a tenth of that) -- Stability: equal values wake together, in any order, which cannot be seen -- Against the standard library: hopeless, of course. `slice::sort_unstable` sorts 1000 random `u32`s in 3.5 µs and 100 in 0.2 µs; `sleep` takes about 115 ms and 11 ms at 100 µs per unit, 30,000–55,000 times as long, and is not always right diff --git a/lib/Cargo.lock b/lib/Cargo.lock index 959bb0c..0b22347 100644 --- a/lib/Cargo.lock +++ b/lib/Cargo.lock @@ -4,4 +4,4 @@ version = 4 [[package]] name = "nettrash-a" -version = "0.0.27" +version = "0.0.29" diff --git a/lib/Cargo.toml b/lib/Cargo.toml index c6d27c2..b11b219 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "nettrash-a" -version = "0.0.27" +version = "0.0.29" edition = "2024" -description = "A library of algorithms in Rust" +description = "Algorithms in pure Rust with no dependencies: 23 sorting algorithms and a default sort; arbitrary-precision integers with Karatsuba, Toom-Cook and NTT multiplication, Newton division, GCD, roots, modular powers, primes and modular square roots; elliptic curves over prime fields with secp256k1, P-256, P-384, P-521, Curve25519 and Ed25519; complex numbers; matrices of floats, complex numbers or integers with LU, QR, Cholesky, SVD, Schur and eigendecompositions, exp, sqrt and log, and exact integer determinants and inverses. Not constant-time." license = "MIT" repository = "https://github.com/nettrash/a" readme = "../README.md" -keywords = ["algorithms"] -categories = ["algorithms"] +keywords = ["algorithms", "sorting", "bignum", "elliptic-curve", "matrix"] +categories = ["algorithms", "mathematics", "science"] rust-version = "1.88" [lib] diff --git a/lib/docs/math/complex.md b/lib/docs/math/complex.md new file mode 100644 index 0000000..19cc4fa --- /dev/null +++ b/lib/docs/math/complex.md @@ -0,0 +1,64 @@ +[A](../../../README.md) › [Math](../../../README.md#math) + +# Complex numbers + +`Complex` (in `a::math::complex`) is the complex number `re + im·i`, its two parts public fields of the float type `T`, `f32` or `f64`. The trait `Float`, sealed, is what the two have in common, for code generic over both; [matrices](matrix.md) of complex numbers are built on it. + +- `Complex::new(re, im)` builds one, `Complex::from(x)` a real one, and `Complex::from_polar(r, θ)` the one of absolute value `r` and argument `θ`. `Complex::ZERO`, `Complex::ONE` and `Complex::I` are constants. +- They add with `+` and `+=`, subtract with `-` and `-=`, multiply with `*` and `*=` and divide with `/` and `/=`, owned or borrowed, and take a float of their own type as an operand on either side: `z * 2.0`, `2.0 * z`, `1.0 / z`. A real operand multiplies both parts, so `2·∞` stays `∞ + 0i` rather than becoming a NaN through `0·∞`. +- `-z` negates, `conj` conjugates, `recip` is `1 / z`; `==` compares, `Default` is zero, `Sum` and `Product` take iterators of owned or borrowed numbers. `Display` writes `1.5-2i`, each part with the precision given, `{:.2}` making `1.50-2.00i`. +- `abs` is the absolute value, `norm_sqr` its square, cheaper but subject to overflow, `arg` the argument in `(−π, π]`, `to_polar` both. +- `exp`, `ln`, `sqrt`, `powi`, `powf` and `powc` are the principal values: `ln` has its imaginary part in `(−π, π]` and `sqrt` its real part not negative. Both have their branch cut along the negative real axis, where the sign of a zero imaginary part picks the side, as in C99's ``: `sqrt(−4 + 0i) = 2i` and `sqrt(−4 − 0i) = −2i`. `0⁰` is 1. +- `is_finite` and `is_nan` look at both parts. + +## Examples + +```rust +use a::math::complex::Complex; + +let z = Complex::new(3.0, 4.0); +assert_eq!(z.abs(), 5.0); +assert_eq!(z * z.conj(), Complex::new(25.0, 0.0)); +assert_eq!(z / Complex::new(0.0, 2.0), Complex::new(2.0, -1.5)); +assert_eq!(z.sqrt(), Complex::new(2.0, 1.0)); +assert_eq!(Complex::new(1.0, 1.0).powi(8), Complex::new(16.0, 0.0)); +assert_eq!(format!("{}", z - 2.0 * Complex::I), "3+2i"); + +// The branch cut: the sign of zero picks the side. +assert_eq!(Complex::new(-4.0, 0.0).sqrt(), Complex::new(0.0, 2.0)); +assert_eq!(Complex::new(-4.0, -0.0).sqrt(), Complex::new(0.0, -2.0)); +assert_eq!(Complex::new(-1.0, 0.0).ln(), Complex::new(0.0, std::f64::consts::PI)); + +// Neither the squares of `abs` nor the quotient's intermediate values overflow. +let big = Complex::new(1e300_f64, 1e300); +assert_eq!(big / big, Complex::ONE); +assert!(big.abs().is_finite()); +``` + +The method: + +- A quotient is Smith's: the larger part `p` of the divisor divides the smaller `q`, `r = q/p`, and `(a + b·r)/(p + q·r)` and `(b − a·r)/(p + q·r)`, with the parts of the dividend in the order that the case asks, so that nothing overflows where the quotient doesn't. Which part is the larger is chosen by selects, not a branch, which random divisors would mispredict half the time, and so is `r = 0` for a zero divisor, which leaves the parts divided by zero, as floats are. At the ends of the range, as LAPACK's `ladiv` (Baudin and Smith), all four parts are scaled by one power of two, which leaves the quotient as it was: halved when one of them is at least half the largest float, as the sums are at most twice the largest part, so that `(1e308 + 1e308·i) / (1e308 + 1e308·i)` is 1, not a NaN from `∞ / ∞`; and scaled up by `8/ε²` when the parts of either operand are all below `4·MIN/ε`, so that `(u + u·i) / (u + 2u·i)` for the smallest subnormal `u` is `0.6 − 0.2i`. Where `r` itself falls below the normal range, the products with it are taken as `(y·q)/p`, so that `1e300 / (1e100 + 1e−250·i)` keeps its imaginary part `−1e−150`. All of it took a quotient from 1.1 ns to 2.0; separate factors for the two operands took 2.3. +- `abs` is `hypot`. `sqrt` is Kahan's: `t = √((|re| + |z|)/2)`, and the root `t + i·im/(2t)` for `re ≥ 0`, `|im|/(2t) ± i·t` otherwise, which cancels nothing, with the operand scaled by a power of 4 when its parts are near the subnormals or near overflow, and C99's values for infinities and NaNs. +- `ln` takes its real part as `ln(1 + (|z|² − 1))/2` near the unit circle, `(|z| − 1)(|z| + 1) + small²` with the larger part `|z|` of the two, so that `ln |1 + 10⁻¹⁰i| = 5·10⁻²¹` keeps its digits where `ln |z|` would round to 0; `ln |z|` elsewhere, but `ln(big) + ln(1 + (small/big)²)/2` where `|z|` overflows or is subnormal, so that `ln(MAX + MAX·i)` is finite. +- `exp` keeps a real number real, with the sign of its zero imaginary part; `powi` multiplies by repeated squaring, of the reciprocal for a negative power; `powf` and `powc` are `exp(w·ln z)`. + +Measured on an Apple M4 Max for `Complex`, over 1,024 independent operations on random parts in `[−2, 2)`, each result stored, against num-complex 0.4.6 on the same values (the median of 21 interleaved runs, in nanoseconds per operation): + +| Operation | `Complex` | num-complex | +| --- | --- | --- | +| `a * b` | 0.43 | 0.44 | +| `a / b` | 1.99 | 0.88 | +| `abs` | 1.22 | 1.23 | +| `sqrt` | 4.41 | 9.12 | +| `exp` | 5.15 | 5.46 | +| `ln` | 7.54 | 7.40 | +| `powc` | 15.75 | 15.48 | + +- num-complex divides by `|b|²` as it is, so that its quotients overflow from `|b|` of about 10¹⁵⁴ and lose their digits below 10⁻¹⁵⁴, where Smith's method with its scaling keeps them, at about 1.1 ns a quotient. +- `sqrt` takes half the time: a `hypot`, a square root and a division, where num-complex goes through the polar form, a `hypot`, an `atan2`, a square root, a sine and a cosine. +- `ln` takes 2 % longer for the digits it keeps near the unit circle, and `powc` through it as much. + +## Efficiency + +- Time complexity: O(1) for each operation +- Space complexity: O(1), two floats diff --git a/lib/docs/math/matrix.md b/lib/docs/math/matrix.md new file mode 100644 index 0000000..d6cdcc5 --- /dev/null +++ b/lib/docs/math/matrix.md @@ -0,0 +1,187 @@ +[A](../../../README.md) › [Math](../../../README.md#math) + +# Matrices + +`Matrix` (in `a::math::matrix`) is a rectangular array of elements of the type `T`, held column after column in one vector, of any size chosen when it is built. `T` is one of five types, the sealed trait `Element`: + +- `f32` and `f64`, and [`Complex` and `Complex`](complex.md): the trait `Scalar`, with every operation below. `T::Real` is the float type of their parts, of norms, singular values and the eigenvalues of Hermitian matrices. +- [`Number`](numbers.md), an integer of any size: the arithmetic, the transposes, the row and column operations, the trace and powers, and, exactly, the determinant, the rank, the characteristic polynomial, the adjugate, the inverse as the adjugate and the determinant, and the null space. + +## Building and reading + +- `Matrix::from_rows(&[[1.0, 2.0], [3.0, 4.0]])` and `from_columns` take rows or columns of anything that is a slice; `from_row_major(rows, cols, vec)` and `from_column_major(rows, cols, vec)` a vector, the second without copying; `from_fn(rows, cols, |i, j| …)` a function of the indices. `zeros(rows, cols)`, `identity(n)` and `from_diagonal(&d)` need the element's 0 and 1. A shape that doesn't fit panics. +- `rows()`, `cols()`, `shape()`, `is_square()` and `is_empty()`; `a[(i, j)]` reads and writes an element and panics outside, `get(i, j)` and `get_mut(i, j)` give `None` there; `column(j)` is a slice, `row(i)` an iterator; `as_slice()`, `as_mut_slice()` and `into_vec()` the elements column after column; `diagonal()`; `map(|x| …)` a matrix of another type, as `a.map(|&x| Complex::from(x))` makes a real matrix complex. +- Matrices compare with `==`, clone, hash when their elements do, and default to `0 × 0`. `Display` writes the rows on lines of their own, each column right-aligned, two spaces apart, with the precision given; `Debug` writes them as lists, `[[1.0, 2.0], [3.0, 4.0]]`. + +## Arithmetic and transformations + +- `+`, `-` and their assignments add and subtract matrices of the same shape, elementwise; `-a` negates; `*` multiplies matrices whose inner dimensions agree, and a matrix by an element on either side, `&a * 2.0` and `2.0 * &a`; `/` divides by an element. Operands may be owned or borrowed; an owned left operand, or the target of an assignment, keeps its vector. Shapes that don't fit panic. +- `a.hadamard(&b)` is the product elementwise `A ∘ B`; `transpose()` is `Aᵀ` and `conjugate_transpose()` is `A*`, the transpose for real elements. +- `swap_rows(i, j)`, `scale_row(i, &k)` and `add_scaled_row(target, source, &k)`, which adds `k` times row `source` to row `target`, are the elementary row operations, in place; `swap_columns`, `scale_column` and `add_scaled_column` the column ones. +- `trace()` is the sum of the diagonal; `pow(n)` is `Aⁿ` for `n: u32` by repeated squaring, `A⁰ = I`; `powi(n)` takes an `i32`, a negative power of the inverse, and `None` when that doesn't exist. + +## Decompositions + +Each is a type of its own that gives its factors as matrices: + +- `lu()`: `P·A = L·U` by Gaussian elimination with partial pivoting, for any shape: `l()`, `u()`, `p()`, `permutation()`, `det()`, `solve(&b)`, `inverse()` and `is_singular()`. +- `qr()`: `A = Q·R` by Householder reflections, for any shape: `q()` with orthonormal columns, `q_full()` square, `r()` with a real diagonal, and `solve(&b)`, the least squares for at least as many rows as columns. +- `cholesky()`: `A = L·L*` for a Hermitian positive definite matrix, from its lower triangle, or `None`: `l()`, `det()`, `solve(&b)`, `inverse()`. +- `svd()`: `A = U·Σ·V*`, thin: `u()`, `singular_values()`, descending, and `v()`. +- `schur()`: `A = Q·T·Q*` with `Q` unitary: `T` upper triangular for complex elements, and for real ones the real Schur form, with a `2 × 2` block `[[a, b], [c, a]]`, `b·c < 0`, for each pair of complex conjugate eigenvalues; `q()`, `t()` and `eigenvalues()`. +- `eigen()`: the eigenvalues and the eigenvectors, complex, each of length 1, with `A·V = V·Λ`; for a Hermitian matrix `hermitian_eigen()` gives real eigenvalues in ascending order and orthonormal eigenvectors of the matrix's own type, with `A = V·Λ·V*`. + +## Properties + +- `det()`, `inverse()` (`None` when singular) and `solve(&b)` from the LU decomposition. +- `rank()`, the number of singular values above `max(m, n)·ε·σ₁` as NumPy's `matrix_rank`, and `null_space()`, an orthonormal basis of `{x : A·x = 0}` as columns. +- `eigenvalues()` and `hermitian_eigenvalues()` without the eigenvectors, and `singular_values()` without `U` and `V`, for less work. +- `characteristic_polynomial()`, the coefficients of `det(x·I − A)` from `xⁿ` down. +- `norm(Norm::…)`: `One`, the largest column sum of absolute values; `Infinity`, the largest row sum; `Frobenius`; `Max`, the largest absolute value; `Two`, the largest singular value; `Nuclear`, their sum. `cond(Norm::…)` is `‖A‖·‖A⁻¹‖`, infinite for a singular matrix; in the 2-norm `σ₁/σₙ`, for any shape. + +## Functions of matrices + +- `exp()` is `e^A`, `sqrt()` the principal square root, whose eigenvalues have positive real parts, and `log()` the principal logarithm, whose eigenvalues have imaginary parts in `(−π, π]`. `sqrt` and `log` give `None` for a real matrix whose square root or logarithm is not real, with a negative real eigenvalue, and for a singular matrix that has none: a real matrix can be made complex first. +- `pow` and `powi` are the integer powers. + +A matrix with a NaN or an infinity gives NaNs, or `None` from `sqrt` and `log`. + +## Exactly, on numbers + +For `Matrix`, `det()` is Bareiss's fraction-free elimination, `rank()` its number of pivots, `characteristic_polynomial()` Berkowitz's method without a division, `adjugate()` `adj(A)` with `A·adj(A) = det(A)·I` also for singular matrices, `inverse()` the pair `(adj(A), det(A))`, `None` for a zero determinant, and `null_space()` a basis of integer vectors without common factors. + +## Algorithms + +The floating-point algorithms are LAPACK's, arranged so that most of their work is products of blocks, and computed in the elements' own precision; real matrices stay in real arithmetic, also when their eigenvalues are complex. + +- Products, `C = β·C + α·op(A)·op(B)` with `op` the matrix, its transpose or its conjugate transpose: Goto and van de Geijn's method as BLIS arranges it. Blocks of `op(B)` of 512 rows and 1020 columns (2040 for `f32`) and of `op(A)` of 128 rows are packed into panels that stay in the caches, transposed or conjugated as they are packed, and multiplied by kernels of 8 × 6 elements in registers for `f64` and 8 × 12 for `f32`, with narrower ones for a block's last columns. Complex products run on the real kernels by Van Zee's 1m method, at their speed. Products of up to 64³ whose operands are close together in memory skip the packing, on tiles of registers; products of up to 32 rows copy `op(A)` and read `op(B)` where it is. The kernels are NEON on little-endian aarch64, and portable ones of the same shapes elsewhere. +- Triangular solves: the triangle halved down to 16 rows, so that most of the work is the products of the blocks off the diagonal. A leaf copies its triangle to the stack scaled by the reciprocals of its diagonal, so that the substitution is fused multiply-adds alone, with NEON kernels for `f32` and `f64`; a triangle whose diagonal has a zero, a subnormal or a non-finite element, and columns whose results are not all finite, are solved by plain substitution with divisions, so that infinities and NaNs reach only the elements they would. +- `lu()`: partial pivoting, the pivot the first element of the largest `|re| + |im|` in its column, as LAPACK chooses it, and a division rather than a product by its reciprocal when that would overflow, as LAPACK's `getf2` does. Up to 16 × 16 on the stack in unrolled loops, the pivots chosen and the rows swapped by selects rather than branches; up to 24 columns a column at a time; larger matrices by Toledo's recursion, the left half, then the right half updated by it by a triangular solve and a product, down to panels of 16 columns, the row swaps of each half applied to the other a column at a time. +- `inverse()`: up to 12 × 12 on the stack in unrolled loops; larger matrices as LAPACK's `getri` does, `U⁻¹`, then `X·L = U⁻¹` by one triangular solve, then the columns swapped back, `2n³` operations in all. +- `cholesky()`: up to 7 × 7 (4 × 4 for complex elements) unrolled; up to 32 × 32 (16 × 16) column by column, each column in registers for `f32` and `f64`; larger matrices halved, `L11`, then `L21 = A21·L11⁻*` by a triangular solve, then `A22 − L21·L21*` on its lower triangle alone, in blocks of 64 columns. +- `qr()`: Householder reflections. A matrix of up to 4096 elements one reflection at a time (`geqr2`), each applied to the columns after it four at a time; a larger one in blocks of 8 to 32 columns chosen by its shape, each block's panel halved down to 8 columns as `geqrt3` does, so that the panel's own work is products too, its reflections gathered as `I − V·T·V*` (the compact WY form) and applied to the columns after it by three products. The factors `T` are kept: `q()`, `q_full()` and `solve(&b)` apply them by blocks, and the whole `Q` of a matrix of at least 4 times as many rows as columns is formed as one block, in half the work. +- `hermitian_eigen()`: a reduction to a real symmetric tridiagonal matrix by reflections in panels of 32 columns, whose trailing updates are products of rank 64 (`hetrd`, `latrd`); the tridiagonal matrix's eigenvectors by Cuppen's divide and conquer (`stedc`), with LAPACK's deflations, the secular equations solved as `laed4` does and Gu and Eisenstat's recomputed vectors, so that the eigenvectors are orthogonal, and the implicit QL algorithm with Wilkinson's shifts below order 25; the reflections applied back in blocks of 32 by products. Up to 80 to 160 rows by the type, where that machinery costs more than it saves, the reduction goes one column at a time, its reflections are multiplied out into `Q`, two at a time, and the implicit QL algorithm's rotations are applied to `Q`. `hermitian_eigenvalues()`: the same reduction, then Pal, Walker and Kahan's QL and QR without square roots (`sterf`). +- `svd()`: a reduction to a real bidiagonal matrix by reflections from both sides in panels of 16 columns, whose trailing updates are two products (`gebrd`, `labrd`), after a QR decomposition for a matrix of at least 2.5 times as many rows as columns; the bidiagonal matrix's vectors by divide and conquer (`bdsdc`), with the implicit QR algorithm below order 25; the reflections applied back in blocks of 32. `singular_values()`: the same reduction, from 1.75 times as many rows as columns after a QR decomposition, then dqds (`lasq1`–`lasq6`), which computes the singular values of the bidiagonal matrix to high relative accuracy. Up to 88 to 112 columns by the type (to 96 to 192 for the values alone), the reduction goes one column at a time, each right reflection applied together with the next left one, `U` and `V` are multiplied out, and the bidiagonal matrix takes Golub and Kahan's implicit QR algorithm with its rotations applied to them. A wide matrix is decomposed as its conjugate transpose. +- `schur()`, `eigenvalues()` and `eigen()`: a reduction to Hessenberg form in panels of 32 columns (`gehrd`, `lahr2`), then the QR algorithm. From order 120 (90 for complex elements with the vectors), Braman, Byers and Mathias's sweeps of many small bulges at once, with aggressive early deflation (`laqr0`–`laqr5`), the bulges' reflections gathered and applied to the rest of the matrix by products; below that, the double-shift QR algorithm for real matrices and the single-shift one for complex matrices (`lahqr`), and the real 2 × 2 blocks standardized as `lanv2` does. The eigenvectors by back substitution on the triangular or quasi-triangular form, 64 at a time, with the work off the diagonal as products, and a vector that would overflow solved again alone with LAPACK's scaling (`trevc3`); then `Q·X`. +- `hermitian_eigen()`, `svd()` and their value-only forms first scale a matrix whose largest element is near overflow or underflow, as LAPACK's drivers do. +- `rank()`, `null_space()`, `norm(Norm::Two)`, `norm(Norm::Nuclear)` and `cond(Norm::Two)` from the singular values (and `V` for the null space); `cond` in the other norms from the inverse; `characteristic_polynomial()` by La Budde's recurrence on the Hessenberg form, about `n³` operations. +- `exp()`: scaling and squaring, Higham's `expm` (2005): `A / 2^s` with a Padé approximant of degree 3, 5, 7, 9 or 13 chosen by its 1-norm (3, 5 or 7 in single precision), squared `s` times. `sqrt()`: the Schur method (Björck and Hammarling), `R² = T` solved element by element for the triangular `R`, then `Q·R·Q*`. `log()`: inverse scaling and squaring, Higham's `logm` (2001): square roots of the Schur form until it is within 0.264 of `I`, a Padé approximant of degree 7 in partial fractions by Gauss and Legendre's rule of 7 points, times `2^k`, with Al-Mohy and Higham's accurate diagonal. A real matrix's square root and logarithm are computed on its complex Schur form, and are real when they exist. +- On `Number`: Bareiss's elimination takes about `n³` products of numbers no longer than the determinant; Berkowitz's method about `n⁴/2`; the adjugate is `(−1)ⁿ⁻¹·(Aⁿ⁻¹ + c₁·Aⁿ⁻² + … + cₙ₋₁·I)` by Cayley and Hamilton, from the characteristic polynomial's coefficients `c`, for singular matrices too; the null space comes from the fraction-free echelon form, one vector for each column without a pivot. + +## Measurements + +Measured on an Apple M4 Max, one core, for `f64`, against faer 0.24.4 (`Par::Seq`), nalgebra 0.35.0 and Apple's Accelerate (`VECLIB_MAXIMUM_THREADS=1`: `dgemm`, and LAPACK's `dgetrf`, `dgetri`, `dpotrf`, `dgeqrf`, `dgesdd`, `dsyevd` and `dgeev`), on the same square matrices of random elements uniform in `[−1, 1)`, `A·Aᵀ + n·I` for `cholesky()` and the Hermitian eigenvalues (the median of up to 200 calls in 0.4 s, in µs). The others' counterparts: faer's `partial_piv_lu`, `partial_piv_lu().inverse()`, `llt`, `qr`, `thin_svd`, `singular_values`, `self_adjoint_eigen`, `self_adjoint_eigenvalues`, `eigen` and `eigenvalues`, and nalgebra's `lu`, `try_inverse`, `cholesky`, `qr`, `svd(true, true)`, `singular_values`, `symmetric_eigen` and `exp`; "–" where a library has no such function, or where it was not measured here. + +| Operation | n | `Matrix` | faer | nalgebra | Accelerate | +| --- | --- | --- | --- | --- | --- | +| `a * b` | 8 | 0.04 | 0.08 | 0.08 | 0.08 | +| `a * b` | 32 | 1.08 | 1.33 | 1.21 | 0.67 | +| `a * b` | 128 | 74.0 | 80.7 | 76.2 | 12.2 | +| `a * b` | 512 | 4,345 | 4,792 | 4,627 | 612 | +| `a * b` | 1024 | 34,894 | 38,375 | 37,131 | 4,985 | +| `lu()` | 8 | 0.12 | 0.38 | 0.17 | 0.25 | +| `lu()` | 32 | 1.83 | 2.79 | 2.50 | 2.96 | +| `lu()` | 128 | 48.0 | 67.8 | 96.1 | 38.7 | +| `lu()` | 512 | 1,930 | 2,509 | 7,956 | 828 | +| `lu()` | 1024 | 13,776 | 17,180 | 59,943 | 4,062 | +| `inverse()` | 8 | 0.21 | 0.83 | 0.50 | 0.58 | +| `inverse()` | 32 | 4.25 | 8.83 | 13.3 | 7.54 | +| `inverse()` | 128 | 124 | 174 | 423 | 75.5 | +| `inverse()` | 512 | 5,414 | 7,005 | 29,490 | 1,715 | +| `inverse()` | 1024 | 40,098 | 50,225 | 220,728 | 9,229 | +| `cholesky()` | 8 | 0.08 | 0.21 | 0.08 | 0.04 | +| `cholesky()` | 32 | 0.96 | 1.29 | 1.75 | 0.50 | +| `cholesky()` | 128 | 27.3 | 37.5 | 53.9 | 12.1 | +| `cholesky()` | 512 | 1,056 | 1,217 | 3,560 | 200 | +| `cholesky()` | 1024 | 7,388 | 8,469 | 27,834 | 1,153 | +| `qr()` | 8 | 0.21 | 0.46 | 0.29 | 0.42 | +| `qr()` | 32 | 2.46 | 6.71 | 4.12 | 3.88 | +| `qr()` | 128 | 78.8 | 140 | 169 | 105 | +| `qr()` | 512 | 3,929 | 4,601 | 12,106 | 1,996 | +| `qr()` | 1024 | 28,977 | 32,458 | 90,739 | 10,655 | +| `svd()` | 8 | 2.08 | 4.08 | 2.83 | 5.50 | +| `svd()` | 32 | 33.8 | 75.0 | 54.1 | 76.3 | +| `svd()` | 128 | 1,055 | 1,336 | 2,471 | 1,070 | +| `svd()` | 512 | 35,029 | 48,891 | 663,666 | 20,749 | +| `svd()` | 1024 | 243,916 | 330,782 | 5,885,939 | 103,470 | +| `singular_values()` | 8 | 1.67 | 3.50 | 2.29 | 2.12 | +| `singular_values()` | 32 | 18.6 | 55.0 | 32.2 | 21.9 | +| `singular_values()` | 128 | 460 | 922 | 645 | 555 | +| `singular_values()` | 512 | 15,848 | 28,704 | 30,747 | 10,238 | +| `singular_values()` | 1024 | 109,713 | 183,136 | 219,850 | 50,326 | +| `hermitian_eigen()` | 8 | 1.29 | 2.25 | 1.67 | 2.33 | +| `hermitian_eigen()` | 32 | 17.4 | 35.2 | 25.5 | 40.0 | +| `hermitian_eigen()` | 128 | 562 | 789 | 859 | 619 | +| `hermitian_eigen()` | 512 | 18,201 | 21,946 | 51,940 | 11,930 | +| `hermitian_eigen()` | 1024 | 113,630 | 145,524 | 396,273 | 64,084 | +| `hermitian_eigenvalues()` | 8 | 1.17 | 2.04 | – | – | +| `hermitian_eigenvalues()` | 32 | 12.8 | 25.7 | – | – | +| `hermitian_eigenvalues()` | 128 | 232 | 418 | – | – | +| `hermitian_eigenvalues()` | 512 | 8,041 | 11,708 | – | – | +| `hermitian_eigenvalues()` | 1024 | 46,139 | 72,953 | – | – | +| `eigen()` | 8 | 3.88 | 7.96 | – | 5.79 | +| `eigen()` | 32 | 85.0 | 149 | – | 109 | +| `eigen()` | 128 | 2,813 | 3,663 | – | 4,702 | +| `eigen()` | 512 | 83,189 | 106,737 | – | 108,339 | +| `eigen()` | 1024 | 497,656 | 759,539 | – | 409,343 | +| `eigenvalues()` | 8 | 3.25 | 5.75 | – | – | +| `eigenvalues()` | 32 | 49.3 | 111 | – | – | +| `eigenvalues()` | 128 | 1,599 | 2,290 | – | – | +| `eigenvalues()` | 512 | 47,326 | 71,632 | – | – | +| `eigenvalues()` | 1024 | 276,825 | 531,718 | – | – | +| `exp()` | 8 | 1.08 | – | 3.33 | – | +| `exp()` | 32 | 19.2 | – | 36.8 | – | +| `exp()` | 128 | 969 | – | 1,440 | – | +| `exp()` | 512 | 60,683 | – | 94,904 | – | +| `exp()` | 1024 | 511,593 | – | 777,554 | – | + +- Every operation is faster than faer's and nalgebra's at every size measured, but `cholesky()` of `8 × 8`, as fast as nalgebra's. At `1024 × 1024` faer takes 1.10 times as long for the product, 1.25 for `lu()` and `inverse()`, 1.15 for `cholesky()`, 1.12 for `qr()`, 1.36 for `svd()`, 1.67 for `singular_values()`, 1.28 for `hermitian_eigen()`, 1.58 for `hermitian_eigenvalues()`, 1.53 for `eigen()` and 1.92 for `eigenvalues()`; nalgebra 1.52 for `exp()`. +- The product runs at about 60 GFLOP/s in `f64` and 120 in `f32` from `256 × 256`, what a loop of fused multiply-adds alone sustains on one core of this machine. Accelerate runs its products on the M4's matrix unit (SME), several times faster from `32 × 32`, and is faster than these wherever products are most of the work: `cholesky()` at every size, `lu()` and `inverse()` from `128 × 128`, `qr()`, `svd()`, `singular_values()` and `hermitian_eigen()` from `512 × 512`, and `eigen()` at `1024 × 1024`; at the smaller sizes it is slower. +- For `f32` and complex elements, measured while the algorithms were tuned, the standing is the same with a few exceptions, where faer or nalgebra is faster: for `Complex`, `lu()` of `3 × 3`, `4 × 4`, `32 × 32` and `64 × 64`, `inverse()` of `2 × 2`, `3 × 3`, `32 × 32` and `64 × 64`, and `cholesky()` of `32 × 32` and `128 × 128`; `eigenvalues()` of `128 × 128` complex matrices, by about 10 %; `hermitian_eigen()` of `Complex` from `96 × 96`, by 5 to 8 %; and in `f32`, `inverse()` of `2 × 2` to `4 × 4` and `cholesky()` of `8 × 8`. nalgebra inverts `3 × 3` and `4 × 4` matrices by their cofactors, which also takes less time than the elimination in `f64` (18 ns against 21 for `3 × 3`), but gives no `None` for a pivot that is zero. + +## Examples + +```rust +use a::math::complex::Complex; +use a::math::matrix::{Matrix, Norm}; +use a::math::numbers::Number; + +let a: Matrix = Matrix::from_rows(&[[4.0, 2.0], [2.0, 3.0]]); +let b = Matrix::from_rows(&[[1.0], [2.0]]); + +// A·x = b, by LU; and the same with the inverse. +let x = a.solve(&b).unwrap(); +assert!((&(&a * &x) - &b).norm(Norm::Max) < 1e-15); +assert!((&(&a.inverse().unwrap() * &b) - &x).norm(Norm::Max) < 1e-15); +assert!((a.det() - 8.0).abs() < 1e-15); + +// Symmetric positive definite: Cholesky, and real eigenvalues. +let l = a.cholesky().unwrap().l(); +assert!((&(&l * &l.transpose()) - &a).norm(Norm::Max) < 1e-15); +let e = a.hermitian_eigen(); +assert!((e.values()[0] + e.values()[1] - a.trace()).abs() < 1e-14); + +// A rotation has eigenvalues ±i, so complex ones. +let r: Matrix = Matrix::from_rows(&[[0.0, -1.0], [1.0, 0.0]]); +let mut w = r.eigenvalues(); +w.sort_by(|p, q| p.im.total_cmp(&q.im)); +assert_eq!(w, [Complex::new(0.0, -1.0), Complex::new(0.0, 1.0)]); + +// Its exponential rotates by a radian, and its logarithm is back where it started. +let t: Matrix = Matrix::from_rows(&[[0.0, -1.0], [1.0, 0.0]]); +let rot = t.exp(); +assert!((rot[(0, 0)] - 1f64.cos()).abs() < 1e-15); +assert!((&rot.log().unwrap() - &t).norm(Norm::Max) < 1e-14); + +// Exactly, on numbers. +let n = Matrix::from_rows(&[[2, 1], [5, 3]]).map(|&x| Number::from(x)); +assert_eq!(n.det(), 1); +let (adj, det) = n.inverse().unwrap(); +assert_eq!(&n * &adj, &Matrix::identity(2) * &det); +``` + +## Efficiency + +- Time complexity: `2·m·n·k` operations for an `m × k` by `k × n` product; for `n × n` matrices, about `2n³/3` for `lu()`, `2n³` for `inverse()`, `n³/3` for `cholesky()`, `4n³/3` for `qr()`, `4n³/3` for `hermitian_eigenvalues()` and `4.7n³` with the eigenvectors, `8n³/3` for `singular_values()` and `9.3n³` for `svd()`, and about `10n³` for `eigenvalues()` and `25n³` for `eigen()`, the QR algorithm's sweeps, whose number depends on the matrix, taking most of it; `exp()` up to six products and one solve, and a product for each squaring; `sqrt()` and `log()` a Schur decomposition, then about `n³/3` for each square root of its triangular factor. On numbers, `det()` and `rank()` about `n³` products of numbers as long as the minors, `characteristic_polynomial()` about `n⁴/2` and `adjugate()` `3n⁴/2`. +- Space complexity: a matrix is its elements in one vector, column after column, and its shape; a decomposition keeps its factors (`lu()` the matrix and the permutation, `qr()` the reflections and the triangular factors of their blocks, at most 64 elements per column); working memory of at most a few times the matrix, and a product packs blocks of up to `512 × 1020` elements of `op(B)` (`512 × 2040` for `f32`) and `128 × 512` of `op(A)`. +- Soundness: `unsafe` code is the product's kernels and the leaves of the triangular solves and of the Cholesky decomposition, each with the extent of what it reads and writes checked before it is called; Miri runs the portable kernels, which have the same shapes as the NEON ones. A shape that doesn't fit panics; the iterations of the QR algorithms stop after a bounded number of sweeps. diff --git a/lib/src/docs.rs b/lib/src/docs.rs index 3c8b1c5..efa266b 100644 --- a/lib/src/docs.rs +++ b/lib/src/docs.rs @@ -124,3 +124,9 @@ struct MathQuadratic; #[doc = include_str!("../docs/math/ec.md")] struct MathEc; + +#[doc = include_str!("../docs/math/complex.md")] +struct MathComplex; + +#[doc = include_str!("../docs/math/matrix.md")] +struct MathMatrix; diff --git a/lib/src/math/complex/float.rs b/lib/src/math/complex/float.rs new file mode 100644 index 0000000..dd1e007 --- /dev/null +++ b/lib/src/math/complex/float.rs @@ -0,0 +1,200 @@ +//! The float types of the parts. + +use core::fmt; +use core::iter::{Product, Sum}; +use core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Rem, Sub, SubAssign}; + +mod sealed { + pub trait Sealed {} + impl Sealed for f32 {} + impl Sealed for f64 {} +} + +/// `f32` or `f64`: the type of the parts of a [`Complex`](super::Complex), and of the real numbers +/// a matrix of either gives, such as its norms and singular values. The methods are those of the +/// two types, for code generic over both. The trait is sealed. +pub trait Float: + Copy + + Default + + PartialEq + + PartialOrd + + fmt::Debug + + fmt::Display + + fmt::LowerExp + + Add + + Sub + + Mul + + Div + + Rem + + Neg + + AddAssign + + SubAssign + + MulAssign + + DivAssign + + Sum + + Product + + Send + + Sync + + 'static + + sealed::Sealed +{ + /// 0. + const ZERO: Self; + /// 1. + const ONE: Self; + /// 2. + const TWO: Self; + /// 1/2. + const HALF: Self; + /// π. + const PI: Self; + /// Infinity. + const INFINITY: Self; + /// Not a number. + const NAN: Self; + /// The difference between 1 and the next larger float. + const EPSILON: Self; + /// The smallest positive normal float. + const MIN_POSITIVE: Self; + /// The largest finite float. + const MAX: Self; + /// The number of significant bits, the hidden one included. + const MANTISSA_DIGITS: u32; + + /// The nearest float to `x`. + fn from_f64(x: f64) -> Self; + /// The value as an `f64`, exactly. + fn to_f64(self) -> f64; + /// The absolute value. + fn abs(self) -> Self; + /// The square root. + fn sqrt(self) -> Self; + /// `√(self² + other²)` without an overflow or underflow in the squares. + fn hypot(self, other: Self) -> Self; + /// The angle of the point `(other, self)`, in `[-π, π]`. + fn atan2(self, other: Self) -> Self; + /// The sine and the cosine. + fn sin_cos(self) -> (Self, Self); + /// `e^self`. + fn exp(self) -> Self; + /// The natural logarithm. + fn ln(self) -> Self; + /// `ln(1 + self)`, accurate near zero. + fn ln_1p(self) -> Self; + /// `self` to the integer power `n`. + fn powi(self, n: i32) -> Self; + /// `self · a + b` with one rounding. + fn mul_add(self, a: Self, b: Self) -> Self; + /// The absolute value of `self` with the sign of `sign`. + fn copysign(self, sign: Self) -> Self; + /// The larger of the two, ignoring a NaN. + fn max(self, other: Self) -> Self; + /// The smaller of the two, ignoring a NaN. + fn min(self, other: Self) -> Self; + /// Whether it is neither infinite nor NaN. + fn is_finite(self) -> bool; + /// Whether it is infinite. + fn is_infinite(self) -> bool; + /// Whether it is NaN. + fn is_nan(self) -> bool; + /// Whether its sign bit is set, as for `-0.0`. + fn is_sign_negative(self) -> bool; +} + +macro_rules! float { + ($t:ident) => { + impl Float for $t { + const ZERO: Self = 0.0; + const ONE: Self = 1.0; + const TWO: Self = 2.0; + const HALF: Self = 0.5; + const PI: Self = core::$t::consts::PI; + const INFINITY: Self = $t::INFINITY; + const NAN: Self = $t::NAN; + const EPSILON: Self = $t::EPSILON; + const MIN_POSITIVE: Self = $t::MIN_POSITIVE; + const MAX: Self = $t::MAX; + const MANTISSA_DIGITS: u32 = $t::MANTISSA_DIGITS; + + #[inline] + fn from_f64(x: f64) -> Self { + x as $t + } + #[inline] + fn to_f64(self) -> f64 { + self as f64 + } + #[inline] + fn abs(self) -> Self { + $t::abs(self) + } + #[inline] + fn sqrt(self) -> Self { + $t::sqrt(self) + } + #[inline] + fn hypot(self, other: Self) -> Self { + $t::hypot(self, other) + } + #[inline] + fn atan2(self, other: Self) -> Self { + $t::atan2(self, other) + } + #[inline] + fn sin_cos(self) -> (Self, Self) { + $t::sin_cos(self) + } + #[inline] + fn exp(self) -> Self { + $t::exp(self) + } + #[inline] + fn ln(self) -> Self { + $t::ln(self) + } + #[inline] + fn ln_1p(self) -> Self { + $t::ln_1p(self) + } + #[inline] + fn powi(self, n: i32) -> Self { + $t::powi(self, n) + } + #[inline] + fn mul_add(self, a: Self, b: Self) -> Self { + $t::mul_add(self, a, b) + } + #[inline] + fn copysign(self, sign: Self) -> Self { + $t::copysign(self, sign) + } + #[inline] + fn max(self, other: Self) -> Self { + $t::max(self, other) + } + #[inline] + fn min(self, other: Self) -> Self { + $t::min(self, other) + } + #[inline] + fn is_finite(self) -> bool { + $t::is_finite(self) + } + #[inline] + fn is_infinite(self) -> bool { + $t::is_infinite(self) + } + #[inline] + fn is_nan(self) -> bool { + $t::is_nan(self) + } + #[inline] + fn is_sign_negative(self) -> bool { + $t::is_sign_negative(self) + } + } + }; +} + +float!(f32); +float!(f64); diff --git a/lib/src/math/complex/mod.rs b/lib/src/math/complex/mod.rs new file mode 100644 index 0000000..4420c4f --- /dev/null +++ b/lib/src/math/complex/mod.rs @@ -0,0 +1,273 @@ +//! Complex numbers: a [`Complex`] is `re + im·i` with parts of the [`Float`] type `T`, `f32` or +//! `f64`. +//! +//! They add with `+` and `+=`, subtract with `-` and `-=`, multiply with `*` and `*=` and divide +//! with `/` and `/=`, each operand owned or borrowed, and take a float of their own type as an +//! operand on either side. They negate with `-`, compare with `==`, add up with `Sum` and multiply +//! with `Product`, and are written as `1+2i` by `Display`. +//! +//! [`Complex::abs`] and [`Complex::arg`] give the polar form, [`Complex::from_polar`] builds a +//! number from it, and [`Complex::conj`], [`Complex::norm_sqr`] and [`Complex::recip`] the rest +//! of the arithmetic. [`Complex::exp`], [`Complex::ln`], [`Complex::sqrt`], [`Complex::powi`], +//! [`Complex::powf`] and [`Complex::powc`] are the principal values, with the branch cut of the +//! logarithm and the square root along the negative real axis, where the sign of a zero imaginary +//! part picks the side, as in C99's ``. + +mod float; +mod ops; + +pub use float::Float; + +use core::fmt; + +/// The complex number `re + im·i`. +/// +/// # Examples +/// +/// ``` +/// use a::math::complex::Complex; +/// +/// let z = Complex::new(3.0, 4.0); +/// assert_eq!(z.abs(), 5.0); +/// assert_eq!(z * z.conj(), Complex::new(25.0, 0.0)); +/// assert_eq!(Complex::new(-4.0, 0.0).sqrt(), Complex::new(0.0, 2.0)); +/// assert_eq!(format!("{}", z - 2.0 * Complex::I), "3+2i"); +/// ``` +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[repr(C)] +pub struct Complex { + /// The real part. + pub re: T, + /// The imaginary part. + pub im: T, +} + +impl Complex { + /// The number `re + im·i`. + #[inline] + pub const fn new(re: T, im: T) -> Self { + Complex { re, im } + } +} + +impl Complex { + /// Zero. + pub const ZERO: Self = Complex::new(T::ZERO, T::ZERO); + /// One. + pub const ONE: Self = Complex::new(T::ONE, T::ZERO); + /// The imaginary unit `i`. + pub const I: Self = Complex::new(T::ZERO, T::ONE); + + /// The number with the absolute value `r` and the argument `theta`: `r·e^(i·theta)`. + #[inline] + pub fn from_polar(r: T, theta: T) -> Self { + let (sin, cos) = theta.sin_cos(); + Complex::new(r * cos, r * sin) + } + + /// The absolute value and the argument, as [`abs`](Self::abs) and [`arg`](Self::arg) give + /// them. + #[inline] + pub fn to_polar(self) -> (T, T) { + (self.abs(), self.arg()) + } + + /// The complex conjugate, `re - im·i`. + #[inline] + pub fn conj(self) -> Self { + Complex::new(self.re, -self.im) + } + + /// The absolute value `√(re² + im²)`, without an overflow or underflow in the squares. + #[inline] + pub fn abs(self) -> T { + self.re.hypot(self.im) + } + + /// The square of the absolute value, `re² + im²`: cheaper than `abs`, but it overflows and + /// underflows where the squares do. + #[inline] + pub fn norm_sqr(self) -> T { + self.re * self.re + self.im * self.im + } + + /// The argument, the angle from the positive real axis, in `(-π, π]`; `-π` for a negative + /// real number with a negative zero imaginary part. + #[inline] + pub fn arg(self) -> T { + self.im.atan2(self.re) + } + + /// `1 / self`. + #[inline] + pub fn recip(self) -> Self { + Self::ONE / self + } + + /// Whether both parts are finite. + #[inline] + pub fn is_finite(self) -> bool { + self.re.is_finite() && self.im.is_finite() + } + + /// Whether either part is NaN. + #[inline] + pub fn is_nan(self) -> bool { + self.re.is_nan() || self.im.is_nan() + } + + /// `e^self`: `e^re·(cos im + i·sin im)`. A real number stays real, with the sign of its zero + /// imaginary part. + pub fn exp(self) -> Self { + let r = self.re.exp(); + if self.im == T::ZERO { + return Complex::new(r, self.im); + } + let (sin, cos) = self.im.sin_cos(); + if r.is_infinite() && self.re.is_finite() { + // `e^re` overflows where `e^re·cos` may not, as for `exp(710 + i·π/4)`: the + // trigonometric factor between two halves of the exponential. + let h = (self.re * T::HALF).exp(); + return Complex::new(h * cos * h, h * sin * h); + } + Complex::new(r * cos, r * sin) + } + + /// The principal natural logarithm, `ln |self| + i·arg(self)`, whose imaginary part is in + /// `(-π, π]`. The logarithm of zero is `-∞`. Near the unit circle the real part is + /// `ln(1 + (|self|² - 1)) / 2`, so that it keeps its digits where `ln |self|` is small; where + /// `|self|` overflows, or is subnormal, it is `ln(big) + ln(1 + (small/big)²) / 2` of the + /// larger and the smaller part, so that `ln(MAX + MAX·i)` is finite. + pub fn ln(self) -> Self { + let (a, b) = (self.re.abs(), self.im.abs()); + let (big, small) = if a >= b { (a, b) } else { (b, a) }; + let re = if big > T::HALF && big < T::TWO { + ((big - T::ONE) * (big + T::ONE) + small * small).ln_1p() * T::HALF + } else { + let m = self.abs(); + if m.is_finite() && m >= T::MIN_POSITIVE { + m.ln() + } else { + Self::ln_of_extreme(m, big, small) + } + }; + Complex::new(re, self.arg()) + } + + /// `ln |z|` from its absolute value `m` and its larger and smaller part, where `m` overflows + /// or is subnormal: `ln(big) + ln(1 + (small/big)²) / 2`. Zero, infinities and NaNs as + /// `ln m` gives them. + #[cold] + #[inline(never)] + fn ln_of_extreme(m: T, big: T, small: T) -> T { + if big == T::ZERO || !big.is_finite() || !small.is_finite() { + return m.ln(); + } + let r = small / big; + big.ln() + (r * r).ln_1p() * T::HALF + } + + /// The principal square root, whose real part is not negative, by Kahan's method, which + /// neither overflows nor cancels: for `re ≥ 0`, `t = √((re + |self|) / 2)` and the root + /// `t + i·im / (2t)`, and the other way round for `re < 0`. Infinities and NaNs give what + /// C99's `csqrt` gives. + pub fn sqrt(self) -> Self { + let (re, im) = (self.re, self.im); + if re == T::ZERO && im == T::ZERO { + return Complex::new(T::ZERO, im); + } + if im.is_infinite() { + return Complex::new(T::INFINITY, im); + } + if re.is_infinite() { + let zero_or_nan = if im.is_nan() { + im + } else { + T::ZERO.copysign(im) + }; + return if re > T::ZERO { + Complex::new(re, zero_or_nan) + } else { + Complex::new(zero_or_nan.abs(), T::INFINITY.copysign(im)) + }; + } + if re.is_nan() || im.is_nan() { + return Complex::new(T::NAN, T::NAN); + } + let big = re.abs().max(im.abs()); + if big < T::from_f64(4.0) * T::MIN_POSITIVE { + // Near the subnormals: scaled up by 2^64, exactly, and the root back by 2^32. + let up = T::from_f64(18_446_744_073_709_551_616.0); + return (self * up).sqrt() / T::from_f64(4_294_967_296.0); + } + if big > T::MAX / T::from_f64(4.0) { + return (self * T::from_f64(0.25)).sqrt() * T::TWO; + } + let t = ((re.abs() + self.abs()) * T::HALF).sqrt(); + if re >= T::ZERO { + Complex::new(t, im / (T::TWO * t)) + } else { + Complex::new(im.abs() / (T::TWO * t), t.copysign(im)) + } + } + + /// `self` to the integer power `n`, by repeated squaring: exact as long as the products are. + /// A negative power is the power of the reciprocal, and `0^0` is 1. + pub fn powi(self, n: i32) -> Self { + let mut base = if n < 0 { self.recip() } else { self }; + let mut n = n.unsigned_abs(); + let mut r = Self::ONE; + while n > 0 { + if n & 1 == 1 { + r *= base; + } + n >>= 1; + if n > 0 { + base *= base; + } + } + r + } + + /// The principal value of `self` to the real power `x`: `e^(x·ln self)`, with `0^x = 0` for + /// `x ≠ 0` and `0^0 = 1`. + pub fn powf(self, x: T) -> Self { + if self.re == T::ZERO && self.im == T::ZERO { + return if x == T::ZERO { Self::ONE } else { Self::ZERO }; + } + (self.ln() * x).exp() + } + + /// The principal value of `self` to the complex power `w`: `e^(w·ln self)`, with `0^w = 0` + /// for `w ≠ 0` and `0^0 = 1`. + pub fn powc(self, w: Self) -> Self { + if self.re == T::ZERO && self.im == T::ZERO { + return if w == Self::ZERO { + Self::ONE + } else { + Self::ZERO + }; + } + (self.ln() * w).exp() + } +} + +impl From for Complex { + /// The real number `re + 0·i`. + #[inline] + fn from(re: T) -> Self { + Complex::new(re, T::ZERO) + } +} + +impl fmt::Display for Complex { + /// Writes `re+imi` or `re-imi`, each part as `Display` writes the float, with the precision + /// given: `1.5-2i`, or `1.50-2.00i` with `{:.2}`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let sign = if self.im.is_sign_negative() { '-' } else { '+' }; + match f.precision() { + Some(p) => write!(f, "{:.*}{}{:.*}i", p, self.re, sign, p, self.im.abs()), + None => write!(f, "{}{}{}i", self.re, sign, self.im.abs()), + } + } +} diff --git a/lib/src/math/complex/ops.rs b/lib/src/math/complex/ops.rs new file mode 100644 index 0000000..bcefbaa --- /dev/null +++ b/lib/src/math/complex/ops.rs @@ -0,0 +1,289 @@ +//! The operators of complex numbers. + +use super::{Complex, Float}; +use core::iter::{Product, Sum}; +use core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign}; + +impl Complex { + #[inline] + fn plus(a: Self, b: Self) -> Self { + Complex::new(a.re + b.re, a.im + b.im) + } + + #[inline] + fn plus_real(a: Self, b: T) -> Self { + Complex::new(a.re + b, a.im) + } + + #[inline] + fn real_plus(a: T, b: Self) -> Self { + Complex::new(a + b.re, b.im) + } + + #[inline] + fn minus(a: Self, b: Self) -> Self { + Complex::new(a.re - b.re, a.im - b.im) + } + + #[inline] + fn minus_real(a: Self, b: T) -> Self { + Complex::new(a.re - b, a.im) + } + + #[inline] + fn real_minus(a: T, b: Self) -> Self { + Complex::new(a - b.re, -b.im) + } + + #[inline] + fn times(a: Self, b: Self) -> Self { + Complex::new(a.re * b.re - a.im * b.im, a.re * b.im + a.im * b.re) + } + + #[inline] + fn times_real(a: Self, b: T) -> Self { + Complex::new(a.re * b, a.im * b) + } + + #[inline] + fn real_times(a: T, b: Self) -> Self { + Complex::new(a * b.re, a * b.im) + } + + /// `a / b` by Smith's method: the larger part `p` of `b` divides the smaller `q`, so that + /// nothing overflows where the quotient doesn't. The two cases differ only in which parts + /// play which role and in a sign, chosen by selects rather than a branch, which random + /// operands would mispredict half the time. A zero `b` takes `r = 0`, which leaves `a.re / 0` + /// and `a.im / 0`, infinities or NaNs as for floats, by a select too. + /// + /// At the ends of the range, as LAPACK's `ladiv` (Baudin and Smith), all four parts are + /// scaled by one power of two, which leaves the quotient as it was: halved when one of them + /// is at least half the largest float, as the sums are at most twice the largest part, so + /// that `(1e308 + 1e308·i) / (1e308 + 1e308·i)` is 1, not a NaN from `∞ / ∞`; and scaled + /// up by `8/ε²` when the parts of either operand are all below `4·MIN/ε`, so that Smith's + /// products keep their digits and `(u + u·i) / (u + 2u·i)` for the smallest subnormal `u` + /// is `0.6 - 0.2i`. Where that makes the other operand overflow or vanish, the quotient + /// overflows or underflows anyway. Where `r` itself falls below the normal range, the + /// products with it are taken as `(y·q)/p`, so that `1e300 / (1e100 + 1e-250·i)` keeps its + /// imaginary part `-1e-150`. All of it took a quotient from 1.1 ns to 2.0 on an Apple M4 + /// Max; separate factors for the two operands took 2.3, and the rare cases out of line in a + /// function of their own 4.1, its call alone keeping the loop from overlapping quotients. + #[inline] + fn over(a: Self, b: Self) -> Self { + let zero = T::ZERO; + let (aa, ab, ba, bb) = (a.re.abs(), a.im.abs(), b.re.abs(), b.im.abs()); + let limit = T::MAX * T::HALF; + let tiny = T::MIN_POSITIVE * T::from_f64(4.0) / T::EPSILON; + let big = (aa >= limit) | (ab >= limit) | (ba >= limit) | (bb >= limit); + let small = ((aa <= tiny) & (ab <= tiny)) | ((ba <= tiny) & (bb <= tiny)); + let s = if big { + T::HALF + } else if small { + T::from_f64(8.0) / (T::EPSILON * T::EPSILON) + } else { + T::ONE + }; + let (ar, ai, br, bi) = (a.re * s, a.im * s, b.re * s, b.im * s); + let swap = br.abs() < bi.abs(); + let (p, q) = if swap { (bi, br) } else { (br, bi) }; + let (x, y) = if swap { (ai, ar) } else { (ar, ai) }; + let r = if p == zero { zero } else { q / p }; + let d = p + q * r; + let (yr, xr) = if r.abs() < T::MIN_POSITIVE && q != zero && p != zero { + ((y * q) / p, (x * q) / p) + } else { + (y * r, x * r) + }; + let im = (y - xr) / d; + Complex::new((x + yr) / d, if swap { -im } else { im }) + } + + #[inline] + fn over_real(a: Self, b: T) -> Self { + Complex::new(a.re / b, a.im / b) + } + + #[inline] + fn real_over(a: T, b: Self) -> Self { + Complex::over(Complex::from(a), b) + } +} + +/// An operator between two complex numbers, and between a complex number and a float of its +/// type on the right, owned or borrowed, with its assignments. +macro_rules! operators { + ($op:ident, $method:ident, $assign:ident, $assign_method:ident, $both:ident, $real:ident) => { + impl $op for Complex { + type Output = Self; + #[inline] + fn $method(self, rhs: Self) -> Self { + Complex::$both(self, rhs) + } + } + impl $op<&Complex> for Complex { + type Output = Self; + #[inline] + fn $method(self, rhs: &Self) -> Self { + Complex::$both(self, *rhs) + } + } + impl $op> for &Complex { + type Output = Complex; + #[inline] + fn $method(self, rhs: Complex) -> Complex { + Complex::$both(*self, rhs) + } + } + impl $op<&Complex> for &Complex { + type Output = Complex; + #[inline] + fn $method(self, rhs: &Complex) -> Complex { + Complex::$both(*self, *rhs) + } + } + impl $op for Complex { + type Output = Self; + #[inline] + fn $method(self, rhs: T) -> Self { + Complex::$real(self, rhs) + } + } + impl $op<&T> for Complex { + type Output = Self; + #[inline] + fn $method(self, rhs: &T) -> Self { + Complex::$real(self, *rhs) + } + } + impl $op for &Complex { + type Output = Complex; + #[inline] + fn $method(self, rhs: T) -> Complex { + Complex::$real(*self, rhs) + } + } + impl $op<&T> for &Complex { + type Output = Complex; + #[inline] + fn $method(self, rhs: &T) -> Complex { + Complex::$real(*self, *rhs) + } + } + impl $assign for Complex { + #[inline] + fn $assign_method(&mut self, rhs: Self) { + *self = Complex::$both(*self, rhs); + } + } + impl $assign<&Complex> for Complex { + #[inline] + fn $assign_method(&mut self, rhs: &Self) { + *self = Complex::$both(*self, *rhs); + } + } + impl $assign for Complex { + #[inline] + fn $assign_method(&mut self, rhs: T) { + *self = Complex::$real(*self, rhs); + } + } + impl $assign<&T> for Complex { + #[inline] + fn $assign_method(&mut self, rhs: &T) { + *self = Complex::$real(*self, *rhs); + } + } + }; +} + +operators!(Add, add, AddAssign, add_assign, plus, plus_real); +operators!(Sub, sub, SubAssign, sub_assign, minus, minus_real); +operators!(Mul, mul, MulAssign, mul_assign, times, times_real); +operators!(Div, div, DivAssign, div_assign, over, over_real); + +/// An operator with a float on the left of a complex number of its type, which the orphan rule +/// leaves to one implementation for each float type. +macro_rules! left { + ($t:ident, $op:ident, $method:ident, $left:ident) => { + impl $op> for $t { + type Output = Complex<$t>; + #[inline] + fn $method(self, rhs: Complex<$t>) -> Complex<$t> { + Complex::$left(self, rhs) + } + } + impl $op<&Complex<$t>> for $t { + type Output = Complex<$t>; + #[inline] + fn $method(self, rhs: &Complex<$t>) -> Complex<$t> { + Complex::$left(self, *rhs) + } + } + impl $op> for &$t { + type Output = Complex<$t>; + #[inline] + fn $method(self, rhs: Complex<$t>) -> Complex<$t> { + Complex::$left(*self, rhs) + } + } + impl $op<&Complex<$t>> for &$t { + type Output = Complex<$t>; + #[inline] + fn $method(self, rhs: &Complex<$t>) -> Complex<$t> { + Complex::$left(*self, *rhs) + } + } + }; +} + +macro_rules! lefts { + ($t:ident) => { + left!($t, Add, add, real_plus); + left!($t, Sub, sub, real_minus); + left!($t, Mul, mul, real_times); + left!($t, Div, div, real_over); + }; +} + +lefts!(f32); +lefts!(f64); + +impl Neg for Complex { + type Output = Self; + #[inline] + fn neg(self) -> Self { + Complex::new(-self.re, -self.im) + } +} + +impl Neg for &Complex { + type Output = Complex; + #[inline] + fn neg(self) -> Complex { + -*self + } +} + +impl Sum for Complex { + fn sum>(iter: I) -> Self { + iter.fold(Self::ZERO, |s, z| s + z) + } +} + +impl<'a, T: Float> Sum<&'a Complex> for Complex { + fn sum>(iter: I) -> Self { + iter.fold(Self::ZERO, |s, z| s + z) + } +} + +impl Product for Complex { + fn product>(iter: I) -> Self { + iter.fold(Self::ONE, |p, z| p * z) + } +} + +impl<'a, T: Float> Product<&'a Complex> for Complex { + fn product>(iter: I) -> Self { + iter.fold(Self::ONE, |p, z| p * z) + } +} diff --git a/lib/src/math/matrix/basic.rs b/lib/src/math/matrix/basic.rs new file mode 100644 index 0000000..ba4ff33 --- /dev/null +++ b/lib/src/math/matrix/basic.rs @@ -0,0 +1,184 @@ +//! Transposes, products elementwise, row and column operations, the trace and powers. + +use super::{Element, Matrix}; + +impl Matrix { + /// The transpose `Aᵀ`, whose element in row `i` and column `j` is this one's in row `j` and + /// column `i`. + pub fn transpose(&self) -> Self { + let rows = self.rows; + Matrix::from_fn(self.cols, rows, |i, j| self.data[j + i * rows].clone()) + } +} + +impl Matrix { + /// The conjugate transpose `A*`, the transpose with each element conjugated: the transpose + /// itself for real elements. + pub fn conjugate_transpose(&self) -> Self { + let rows = self.rows; + Matrix::from_fn(self.cols, rows, |i, j| self.data[j + i * rows].conj()) + } + + /// The product elementwise, Hadamard's `A ∘ B`. + /// + /// # Panics + /// + /// Panics if the matrices are not of the same shape. + #[track_caller] + pub fn hadamard(&self, other: &Matrix) -> Matrix { + assert!( + self.shape() == other.shape(), + "the matrices are not of the same shape" + ); + Matrix { + rows: self.rows, + cols: self.cols, + data: self + .data + .iter() + .zip(&other.data) + .map(|(a, b)| T::mul(a, b)) + .collect(), + } + } + + /// Swaps rows `a` and `b`. + /// + /// # Panics + /// + /// Panics if either is outside the matrix. + #[track_caller] + pub fn swap_rows(&mut self, a: usize, b: usize) { + assert!( + a < self.rows && b < self.rows, + "the row is outside the matrix" + ); + if a != b { + for j in 0..self.cols { + self.data.swap(a + j * self.rows, b + j * self.rows); + } + } + } + + /// Multiplies row `i` by `k`. + /// + /// # Panics + /// + /// Panics if the row is outside the matrix. + #[track_caller] + pub fn scale_row(&mut self, i: usize, k: &T) { + assert!(i < self.rows, "the row is outside the matrix"); + for j in 0..self.cols { + T::mul_assign(&mut self.data[i + j * self.rows], k); + } + } + + /// Adds `k` times row `source` to row `target`. + /// + /// # Panics + /// + /// Panics if either row is outside the matrix. + #[track_caller] + pub fn add_scaled_row(&mut self, target: usize, source: usize, k: &T) { + assert!( + target < self.rows && source < self.rows, + "the row is outside the matrix" + ); + for j in 0..self.cols { + let x = T::mul(&self.data[source + j * self.rows], k); + T::add_assign(&mut self.data[target + j * self.rows], &x); + } + } + + /// Swaps columns `a` and `b`. + /// + /// # Panics + /// + /// Panics if either is outside the matrix. + #[track_caller] + pub fn swap_columns(&mut self, a: usize, b: usize) { + assert!( + a < self.cols && b < self.cols, + "the column is outside the matrix" + ); + if a != b { + let (lo, hi) = (a.min(b), a.max(b)); + let (left, right) = self.data.split_at_mut(hi * self.rows); + left[lo * self.rows..(lo + 1) * self.rows].swap_with_slice(&mut right[..self.rows]); + } + } + + /// Multiplies column `j` by `k`. + /// + /// # Panics + /// + /// Panics if the column is outside the matrix. + #[track_caller] + pub fn scale_column(&mut self, j: usize, k: &T) { + for x in self.column_mut(j) { + T::mul_assign(x, k); + } + } + + /// Adds `k` times column `source` to column `target`. + /// + /// # Panics + /// + /// Panics if either column is outside the matrix. + #[track_caller] + pub fn add_scaled_column(&mut self, target: usize, source: usize, k: &T) { + assert!( + target < self.cols && source < self.cols, + "the column is outside the matrix" + ); + let rows = self.rows; + for i in 0..rows { + let x = T::mul(&self.data[i + source * rows], k); + T::add_assign(&mut self.data[i + target * rows], &x); + } + } + + /// The trace `tr(A)`, the sum of the main diagonal. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn trace(&self) -> T { + assert!(self.is_square(), "the matrix is not square"); + let mut t = T::zero(); + for i in 0..self.rows { + T::add_assign(&mut t, &self.data[i + i * self.rows]); + } + t + } + + /// `Aⁿ`, by repeated squaring: about `2 log₂ n` products. `A⁰` is the identity. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn pow(&self, n: u32) -> Self { + assert!(self.is_square(), "the matrix is not square"); + if n == 0 { + return Matrix::identity(self.rows); + } + let mut base = self.clone(); + let mut n = n; + while n & 1 == 0 { + base = &base * &base; + n >>= 1; + } + let mut r = base.clone(); + n >>= 1; + while n > 0 { + base = &base * &base; + if n & 1 == 1 { + r = &r * &base; + } + n >>= 1; + } + r + } +} diff --git a/lib/src/math/matrix/blas.rs b/lib/src/math/matrix/blas.rs new file mode 100644 index 0000000..38c18c9 --- /dev/null +++ b/lib/src/math/matrix/blas.rs @@ -0,0 +1,135 @@ +//! The products and triangular solves the algorithms are built on, on column-major blocks: a +//! block of `rows × cols` starts at the beginning of its slice, and column `j` of it at `j · ld`. + +use super::Scalar; +use core::mem::MaybeUninit; + +mod trsm; + +/// How a block takes part in a product. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Op { + /// As it is. + N, + /// Transposed. + T, + /// Conjugate transposed. + C, +} + +impl Op { + /// Element `(i, j)` of `op(a)`, for `a` with the leading dimension `ld`. + #[inline(always)] + fn at(self, a: &[T], ld: usize, i: usize, j: usize) -> T { + match self { + Op::N => a[i + j * ld], + Op::T => a[j + i * ld], + Op::C => a[j + i * ld].conj(), + } + } +} + +/// `c = β·c + α·op(a)·op(b)`, with `op(a)` of `m × k`, `op(b)` of `k × n` and `c` of `m × n`. +/// When `β` is zero, `c` is not read, so it may hold anything, NaNs included; when `α` is zero or +/// `k` is, neither `a` nor `b` is. Computed by [`super::gemm`]: small products directly, larger +/// ones by packed blocks. +/// +/// # Panics +/// +/// Panics if the leading dimension of a nonempty block is less than its rows, or the block reaches +/// past its slice. +#[allow(clippy::too_many_arguments)] +#[track_caller] +pub(crate) fn gemm( + m: usize, + n: usize, + k: usize, + alpha: T, + a: &[T], + lda: usize, + opa: Op, + b: &[T], + ldb: usize, + opb: Op, + beta: T, + c: &mut [T], + ldc: usize, +) { + T::gemm(m, n, k, alpha, a, lda, opa, b, ldb, opb, beta, c, ldc); +} + +/// Which side of `b` a triangular block multiplies. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Side { + /// `op(a)⁻¹ · b`. + Left, + /// `b · op(a)⁻¹`. + Right, +} + +/// Which triangle of a square block holds it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Uplo { + /// On and above the diagonal. + Upper, + /// On and below the diagonal. + Lower, +} + +/// `b = op(a)⁻¹ · b` or `b = b · op(a)⁻¹`, for the triangle `uplo` of the square block `a`, +/// whose diagonal is taken as ones when `unit` is set; `b` is `m × n`. The triangle is halved +/// until it is small, so that most of the work is the product of the off-diagonal block; see +/// [`trsm`](mod@trsm) for the leaves. +#[allow(clippy::too_many_arguments)] +pub(crate) fn trsm( + side: Side, + uplo: Uplo, + op: Op, + unit: bool, + m: usize, + n: usize, + a: &[T], + lda: usize, + b: &mut [T], + ldb: usize, +) { + trsm::solve(side, uplo, op, unit, m, n, a, lda, b, ldb); +} + +/// [`trsm`](fn@trsm) on the left, `b = op(a)⁻¹ · b`, the solution also written to the `m × n` +/// block `x` of the leading dimension `ldx`: a dense copy of it for products that update the rest +/// of the columns `b` lies in, which they could not read where it is. +#[allow(clippy::too_many_arguments)] +pub(crate) fn trsm_into( + uplo: Uplo, + op: Op, + unit: bool, + m: usize, + n: usize, + a: &[T], + lda: usize, + b: &mut [T], + ldb: usize, + x: &mut [T], + ldx: usize, +) { + trsm::solve_into(uplo, op, unit, m, n, a, lda, b, ldb, x, ldx); +} + +/// Elements of working memory that [`with_scratch`] takes from the stack rather than allocates. +const SCRATCH: usize = 1024; + +/// Runs `f` on `len` zeros of working memory: on the stack when they are few, so that small +/// problems allocate nothing, and allocated otherwise. +pub(crate) fn with_scratch(len: usize, f: impl FnOnce(&mut [T]) -> X) -> X { + if len <= SCRATCH { + let mut s = [const { MaybeUninit::::uninit() }; SCRATCH]; + for v in &mut s[..len] { + v.write(T::zero()); + } + // SAFETY: the first `len` elements were just initialized. + f(unsafe { &mut *(&mut s[..len] as *mut [MaybeUninit] as *mut [T]) }) + } else { + f(&mut vec![T::zero(); len]) + } +} diff --git a/lib/src/math/matrix/blas/trsm.rs b/lib/src/math/matrix/blas/trsm.rs new file mode 100644 index 0000000..119862d --- /dev/null +++ b/lib/src/math/matrix/blas/trsm.rs @@ -0,0 +1,801 @@ +//! The triangular solves of [`trsm`](fn@super::trsm). +//! +//! The triangle is halved until it is at most [`LEAF`] = 16 rows, the first part a multiple of +//! 16, so that most of the work is the products of the off-diagonal blocks. A leaf copies its +//! triangle to the stack first ([`Packed`]), its elements scaled by the reciprocals of the +//! diagonal, so that the substitution is fused multiply-adds with nothing but them on the path +//! from one unknown to the next; the scaling by the diagonal is applied once at the end. For +//! `f64` and `f32` the leaves are NEON kernels (`aarch64`): on the left, two columns of `b` in +//! vector registers at a time, each solved row updating them by a lane of its own vector; on +//! the right, 16 rows at a time, each group of columns summed from those solved before it. The +//! portable leaves, for complex elements, other targets and Miri, work in groups of four rows. +//! +//! A triangle with a zero, a subnormal or a non-finite element on its diagonal, whose reciprocal +//! would not be safe to use, and columns whose results are not all finite, are solved by plain +//! substitution with divisions instead ([`substitute`]), so that infinities and NaNs reach only +//! the elements they would. So are the tiniest triangles, up to [`TINY`], when that divides +//! little: packing them would cost more than it saves. +//! +//! On the left, `op(a)⁻¹·b`, the rows solved first update the rows below them; as they share +//! their columns with these, a product cannot read them where they are, so the leaves also write +//! the solution to a dense buffer that the products read, one for the whole solve. On the right, +//! `b·op(a)⁻¹`, the halves are columns, which lie apart. +//! +//! # Speed +//! +//! Measured on an Apple M4 Max, one core, `f64`, in GFLOP/s (`m²n` on the left, `mn²` on the +//! right), against faer 0.24's triangular solves (`Par::Seq`) and the original leaves, scalar +//! substitution with the solved rows copied at every level: +//! +//! | solve | ours | faer | before | +//! | --- | --- | --- | --- | +//! | `L⁻¹·B`, unit, 16 × 16 by 16 | 18.5 | 14.7 | 4.8 | +//! | `L⁻¹·B`, unit, 64 × 64 by 64 | 35.6 | 31.8 | 11.8 | +//! | `L⁻¹·B`, unit, 512 × 512 by 512 | 51.6 | 39.8 | 35.5 | +//! | `U⁻¹·B`, 256 × 256 by 256 | 47.4 | 39.8 | 20.5 | +//! | `U⁻¹·b`, 64 × 64 by 1 | 6.3 | 4.5 | 4.8 | +//! | `B·L⁻*`, 32 by 16 × 16 | 17.2 | 16.6 | 8.9 | +//! | `B·L⁻*`, 64 by 32 × 32 | 26.5 | 24.8 | 19.0 | +//! | `B·L⁻*`, 256 by 128 × 128 | 43.6 | 37.4 | 40.6 | +//! | `B·L⁻¹`, unit, 64 by 64 × 64 | 33.8 | 28.7 | 28.8 | +//! | `B·L⁻¹`, unit, 512 by 512 × 512 | 50.5 | 46.4 | 50.3 | +//! +//! The right side gained least: its old leaves already ran down the columns of `b`, which the +//! compiler vectorized. Accelerate, on the matrix unit, ran 3 to 7 times faster from 64 up. + +use super::{Op, Side, Uplo, gemm, with_scratch}; +use crate::math::matrix::Scalar; +use core::mem::MaybeUninit; + +// The leaves of `f64` and `f32` in NEON intrinsics on little-endian aarch64, as the products' +// kernels; the portable leaves serve the other types and targets, and Miri. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +mod aarch64; + +/// The most rows (on the left) or columns (on the right) of a triangle solved by substitution. +pub(super) const LEAF: usize = 16; + +/// Triangles up to this order are solved by plain substitution, which costs less than packing +/// them when it divides little: on the left, for one or two columns of `b`, or up to 8 when the +/// diagonal is taken as ones; on the right, when the diagonal is taken as ones. +const TINY: usize = 4; + +/// The rows of `b` a leaf on the right solves at a time. +const ROWS: usize = 16; + +/// `b = op(a)⁻¹·b` or `b = b·op(a)⁻¹`: see [`trsm`](fn@super::trsm). +#[allow(clippy::too_many_arguments)] +pub(super) fn solve( + side: Side, + uplo: Uplo, + op: Op, + unit: bool, + m: usize, + n: usize, + a: &[T], + lda: usize, + b: &mut [T], + ldb: usize, +) { + if m == 0 || n == 0 { + return; + } + let t = Tri::new(a, lda, uplo, op, unit); + match side { + Side::Left if m <= LEAF => left_leaf(&t, m, n, b, ldb, None), + Side::Left => with_scratch(m * n, |x| left(&t, m, n, b, ldb, x, m)), + Side::Right => right(&t, m, n, b, ldb), + } +} + +/// `b = op(a)⁻¹·b`, the solution also written to `x`: see [`trsm_into`](super::trsm_into). +#[allow(clippy::too_many_arguments)] +pub(super) fn solve_into( + uplo: Uplo, + op: Op, + unit: bool, + m: usize, + n: usize, + a: &[T], + lda: usize, + b: &mut [T], + ldb: usize, + x: &mut [T], + ldx: usize, +) { + if m == 0 || n == 0 { + return; + } + let t = Tri::new(a, lda, uplo, op, unit); + left(&t, m, n, b, ldb, x, ldx); +} + +/// A triangular block `op(a)` of a [`trsm`](fn@super::trsm). +struct Tri<'a, T> { + /// The block `a` holding it. + a: &'a [T], + /// The leading dimension of `a`. + lda: usize, + /// How `a` is taken. + op: Op, + /// Whether `op(a)` is lower triangular. + lower: bool, + /// Whether the diagonal is taken as ones. + unit: bool, +} + +impl<'a, T: Scalar> Tri<'a, T> { + /// The triangle `uplo` of `op(a)`. + fn new(a: &'a [T], lda: usize, uplo: Uplo, op: Op, unit: bool) -> Self { + Tri { + a, + lda, + op, + // `op(a)` is lower triangular when exactly one of these holds. + lower: (uplo == Uplo::Lower) == (op == Op::N), + unit, + } + } + + /// Element `(i, j)` of `op(a)`. + #[inline(always)] + fn at(&self, i: usize, j: usize) -> T { + self.op.at(self.a, self.lda, i, j) + } + + /// The block of `op(a)` from row `i` and column `j`, where it is held: in `a` at `(i, j)`, + /// or at `(j, i)` when transposed. + fn block(&self, i: usize, j: usize) -> &'a [T] { + if self.op == Op::N { + &self.a[i + j * self.lda..] + } else { + &self.a[j + i * self.lda..] + } + } + + /// The diagonal block of `op(a)` from `(k, k)`. + fn sub(&self, k: usize) -> Self { + Tri { + a: &self.a[k + k * self.lda..], + ..*self + } + } +} + +impl Clone for Tri<'_, T> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Tri<'_, T> {} + +/// Where the recursion splits `k` rows or columns: the first part a multiple of [`LEAF`], so +/// that the leaves are full. +fn split(k: usize) -> usize { + (k / 2).next_multiple_of(LEAF).min(k - 1) +} + +/// `b = op(a)⁻¹·b` for `m > 0` rows, the solution also written to `x`, of the leading +/// dimension `ldx`, which the products read. +fn left( + t: &Tri, + m: usize, + n: usize, + b: &mut [T], + ldb: usize, + x: &mut [T], + ldx: usize, +) { + if m <= LEAF { + left_leaf(t, m, n, b, ldb, Some((x, ldx))); + return; + } + let k1 = split(m); + let k2 = m - k1; + let (a11, a22) = (t, &t.sub(k1)); + let minus_one = -T::one(); + if t.lower { + left(a11, k1, n, b, ldb, x, ldx); + gemm( + k2, + n, + k1, + minus_one, + t.block(k1, 0), + t.lda, + t.op, + x, + ldx, + Op::N, + T::one(), + &mut b[k1..], + ldb, + ); + left(a22, k2, n, &mut b[k1..], ldb, &mut x[k1..], ldx); + } else { + left(a22, k2, n, &mut b[k1..], ldb, &mut x[k1..], ldx); + gemm( + k1, + n, + k2, + minus_one, + t.block(0, k1), + t.lda, + t.op, + &x[k1..], + ldx, + Op::N, + T::one(), + b, + ldb, + ); + left(a11, k1, n, b, ldb, x, ldx); + } +} + +/// `b = b·op(a)⁻¹` for `n > 0` columns. +fn right(t: &Tri, m: usize, n: usize, b: &mut [T], ldb: usize) { + if n <= LEAF { + right_leaf(t, m, n, b, ldb); + return; + } + let k1 = split(n); + let k2 = n - k1; + let (a11, a22) = (t, &t.sub(k1)); + let minus_one = -T::one(); + let (b1, b2) = b.split_at_mut(k1 * ldb); + if t.lower { + // `x·op(a) = b` from the last columns, which the lower triangle leaves alone. + right(a22, m, k2, b2, ldb); + gemm( + m, + k1, + k2, + minus_one, + b2, + ldb, + Op::N, + t.block(k1, 0), + t.lda, + t.op, + T::one(), + b1, + ldb, + ); + right(a11, m, k1, b1, ldb); + } else { + right(a11, m, k1, b1, ldb); + gemm( + m, + k2, + k1, + minus_one, + b1, + ldb, + Op::N, + t.block(0, k1), + t.lda, + t.op, + T::one(), + b2, + ldb, + ); + right(a22, m, k2, b2, ldb); + } +} + +/// A triangle of order `k <= LEAF` ready for substitution: `l[i + p · LEAF]` is element `(i, p)` +/// of the triangle times `r[p]`, the reciprocal of its diagonal element (1 when the diagonal is +/// taken as ones), and the triangle is lower when `forward` is set. The unknowns `y` of the +/// substitution `y_i = b_i - Σ l_ip·y_p` are then the solution times the diagonal. +/// +/// The scaled elements may overflow where the substitution with divisions would not: the +/// substitutions check that their results are finite, and run with divisions when they are not. +struct Packed { + /// The scaled triangle, by columns, zeros outside it: the first [`columns`](Self::columns) + /// columns are written, as many as the substitutions read. + l: [MaybeUninit; LEAF * LEAF], + /// The columns of `l` written. + columns: usize, + /// The reciprocals of the diagonal. + r: [T; LEAF], + /// Whether the triangle is lower, solved from the first row down. + forward: bool, +} + +impl Packed { + /// An empty triangle, to [`fill`](Self::fill) where it lies, as it is too large to move. + fn new() -> Self { + Packed { + l: [const { MaybeUninit::uninit() }; LEAF * LEAF], + columns: 0, + r: [T::one(); LEAF], + forward: false, + } + } + + /// Packs `op(a)` (`transpose` unset) or its transpose, of order `k`, or `false` when the + /// substitution by reciprocals would not be safe: a diagonal element is zero, not finite or + /// so small that its reciprocal overflows. The columns past `k` that the substitutions + /// read, up to the next power of two from 4, are zeros: only these are written, so that a + /// small triangle costs little. + fn fill(&mut self, t: &Tri, k: usize, transpose: bool) -> bool { + self.columns = k.next_power_of_two().clamp(4, LEAF); + for v in &mut self.l[..self.columns * LEAF] { + v.write(T::zero()); + } + self.forward = t.lower != transpose; + let (a, lda, op) = (t.a, t.lda, t.op); + let conj = op == Op::C; + // Column `p` of the packed triangle is a column of `a` when `op(a)` is `a` and it is not + // transposed, or `op(a)` is transposed and it is; a row of `a` otherwise. + let column = (op == Op::N) != transpose; + for p in 0..k { + if !t.unit { + let d = a[p + p * lda]; + let d = if conj { d.conj() } else { d }; + let q = T::one() / d; + // A reciprocal that is finite and nonzero, of a finite nonzero element: then the + // products by it round as the divisions would, up to an ulp. + if !(d.is_finite() && q.is_finite()) || q == T::zero() { + return false; + } + self.r[p] = q; + } + let rp = self.r[p]; + let rows = if self.forward { p + 1..k } else { 0..p }; + let dst = &mut self.l[p * LEAF..p * LEAF + LEAF]; + if column { + let src = &a[p * lda..]; + for i in rows { + let v = if conj { src[i].conj() } else { src[i] }; + dst[i].write(v * rp); + } + } else { + for i in rows { + let v = a[p + i * lda]; + let v = if conj { v.conj() } else { v }; + dst[i].write(v * rp); + } + } + } + true + } + + /// The written columns of the scaled triangle. + fn l(&self) -> &[T] { + let l = &self.l[..self.columns * LEAF]; + // SAFETY: the first `columns` columns are written by `fill`. + unsafe { &*(l as *const [MaybeUninit] as *const [T]) } + } +} + +/// `b = op(a)⁻¹·b` for `m <= LEAF` rows, by substitution, also written to `x` when given. +fn left_leaf( + t: &Tri, + m: usize, + n: usize, + b: &mut [T], + ldb: usize, + mut x: Option<(&mut [T], usize)>, +) { + if m <= TINY && (n <= 2 || (t.unit && n <= 8)) { + substitute(Side::Left, t, m, n, b, ldb); + if let Some((x, ldx)) = x { + for j in 0..n { + x[j * ldx..j * ldx + m].copy_from_slice(&b[j * ldb..j * ldb + m]); + } + } + return; + } + let mut packed = Packed::new(); + let p = packed.fill(t, m, false).then_some(&packed); + let mut j = 0; + while j < n { + let c = if n - j >= 2 { 2 } else { 1 }; + let done = p.as_ref().is_some_and(|p| { + let x = x.as_mut().map(|(x, l)| (&mut **x, *l)); + #[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] + let x = match left_neon(p, t.unit, m, j, c, b, ldb, x) { + Ok(done) => return done, + Err(x) => x, + }; + match (m.div_ceil(4), c) { + (1, 2) => left_columns::(p, t.unit, m, j, b, ldb, x), + (1, _) => left_columns::(p, t.unit, m, j, b, ldb, x), + (2, 2) => left_columns::(p, t.unit, m, j, b, ldb, x), + (2, _) => left_columns::(p, t.unit, m, j, b, ldb, x), + (3, 2) => left_columns::(p, t.unit, m, j, b, ldb, x), + (3, _) => left_columns::(p, t.unit, m, j, b, ldb, x), + (_, 2) => left_columns::(p, t.unit, m, j, b, ldb, x), + (_, _) => left_columns::(p, t.unit, m, j, b, ldb, x), + } + }); + if !done { + let b = &mut b[j * ldb..]; + substitute(Side::Left, t, m, c, b, ldb); + if let Some((x, ldx)) = x.as_mut() { + for q in 0..c { + let x = &mut x[(j + q) * *ldx..]; + x[..m].copy_from_slice(&b[q * ldb..q * ldb + m]); + } + } + } + j += c; + } +} + +/// Columns `j..j + c` of [`left_leaf`] by the NEON kernels, for `f64` and `f32`: whether they +/// were solved, as [`left_columns`] tells, or `x` back for the other types. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +#[allow(clippy::too_many_arguments)] +fn left_neon<'x, T: Scalar>( + p: &Packed, + unit: bool, + m: usize, + j: usize, + c: usize, + b: &mut [T], + ldb: usize, + x: Option<(&'x mut [T], usize)>, +) -> Result> { + use core::any::TypeId; + let (is_f64, is_f32) = ( + TypeId::of::() == TypeId::of::(), + TypeId::of::() == TypeId::of::(), + ); + if !(is_f64 || is_f32) { + return Err(x); + } + assert!((1..=LEAF).contains(&m) && (1..=2).contains(&c)); + assert!((j + c - 1) * ldb + m <= b.len()); + // The kernels read the columns up to the vectors' rows, at most the next power of two. + assert!(p.l().len() >= LEAF * m.next_power_of_two().clamp(4, LEAF)); + let bp = b[j * ldb..].as_mut_ptr(); + let (xp, ldx) = match x { + Some((x, ldx)) => { + assert!((j + c - 1) * ldx + m <= x.len()); + (x[j * ldx..].as_mut_ptr(), ldx) + } + None => (core::ptr::null_mut(), 0), + }; + let (l, r, f) = (p.l(), &p.r, p.forward); + // SAFETY: `T` is the type each branch casts to, and the columns lie in the slices, as + // checked. + unsafe { + if is_f64 { + let (l, r) = ( + &*(l as *const [T] as *const [f64]), + &*(r as *const [T; LEAF]).cast(), + ); + let (b, x) = (bp.cast::(), xp.cast::()); + Ok(match (m, c) { + (..=2, 1) => aarch64::left_f64::<1, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (..=2, _) => aarch64::left_f64::<1, 2>(l, r, f, unit, m, b, ldb, x, ldx), + (..=4, 1) => aarch64::left_f64::<2, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (..=4, _) => aarch64::left_f64::<2, 2>(l, r, f, unit, m, b, ldb, x, ldx), + (..=8, 1) => aarch64::left_f64::<4, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (..=8, _) => aarch64::left_f64::<4, 2>(l, r, f, unit, m, b, ldb, x, ldx), + (_, 1) => aarch64::left_f64::<8, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (_, _) => aarch64::left_f64::<8, 2>(l, r, f, unit, m, b, ldb, x, ldx), + }) + } else { + let (l, r) = ( + &*(l as *const [T] as *const [f32]), + &*(r as *const [T; LEAF]).cast(), + ); + let (b, x) = (bp.cast::(), xp.cast::()); + Ok(match (m, c) { + (..=4, 1) => aarch64::left_f32::<1, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (..=4, _) => aarch64::left_f32::<1, 2>(l, r, f, unit, m, b, ldb, x, ldx), + (..=8, 1) => aarch64::left_f32::<2, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (..=8, _) => aarch64::left_f32::<2, 2>(l, r, f, unit, m, b, ldb, x, ldx), + (_, 1) => aarch64::left_f32::<4, 1>(l, r, f, unit, m, b, ldb, x, ldx), + (_, _) => aarch64::left_f32::<4, 2>(l, r, f, unit, m, b, ldb, x, ldx), + }) + } + } +} + +/// Columns `j..j + C` of [`left_leaf`] for `m <= 4·G` rows, solved together so that their +/// substitutions overlap, or `false`, with `b` left as it was, when a value is not finite: the +/// substitution then runs with divisions, so that infinities and NaNs reach only the elements +/// they would. +/// +/// The rows go in `G` groups of four, held in registers: each group is solved, then updates the +/// groups after it, a block of four by four. The rows past `m` are zeros, which the zeros of the +/// packed triangle there keep. +#[inline(always)] +fn left_columns( + p: &Packed, + unit: bool, + m: usize, + j: usize, + b: &mut [T], + ldb: usize, + x: Option<(&mut [T], usize)>, +) -> bool { + assert!(m <= 4 * G && G * 4 <= LEAF); + // `y[c][g][k]`: row `4g + k` of column `j + c`. + let mut y = [[[T::zero(); 4]; G]; C]; + for (c, y) in y.iter_mut().enumerate() { + let col = &b[(j + c) * ldb..][..m]; + for (i, v) in col.iter().enumerate() { + y[i / 4][i % 4] = *v; + } + } + // Element `(i, q)` of the scaled triangle. + let pl = p.l(); + let l = |i: usize, q: usize| pl[i + q * LEAF]; + if p.forward { + for g in 0..G { + // The triangle of the group, then its products into the groups below. + for q in 0..4 { + for k in q + 1..4 { + let lk = l(4 * g + k, 4 * g + q); + for y in &mut y { + y[g][k] = y[g][k].mul_sub(lk, y[g][q]); + } + } + } + for h in g + 1..G { + for q in 0..4 { + for k in 0..4 { + let lk = l(4 * h + k, 4 * g + q); + for y in &mut y { + y[h][k] = y[h][k].mul_sub(lk, y[g][q]); + } + } + } + } + } + } else { + for g in (0..G).rev() { + for q in (0..4).rev() { + for k in 0..q { + let lk = l(4 * g + k, 4 * g + q); + for y in &mut y { + y[g][k] = y[g][k].mul_sub(lk, y[g][q]); + } + } + } + for h in 0..g { + for q in 0..4 { + for k in 0..4 { + let lk = l(4 * h + k, 4 * g + q); + for y in &mut y { + y[h][k] = y[h][k].mul_sub(lk, y[g][q]); + } + } + } + } + } + } + if !y.iter().flatten().flatten().all(|v| v.is_finite()) { + return false; + } + if !unit { + for y in &mut y { + for (g, y) in y.iter_mut().enumerate() { + for (k, y) in y.iter_mut().enumerate() { + *y *= p.r[4 * g + k]; + } + } + } + } + for (c, y) in y.iter().enumerate() { + let col = &mut b[(j + c) * ldb..][..m]; + for (i, v) in col.iter_mut().enumerate() { + *v = y[i / 4][i % 4]; + } + } + if let Some((x, ldx)) = x { + for (c, y) in y.iter().enumerate() { + let col = &mut x[(j + c) * ldx..][..m]; + for (i, v) in col.iter_mut().enumerate() { + *v = y[i / 4][i % 4]; + } + } + } + true +} + +/// `b = b·op(a)⁻¹` for `n <= LEAF` columns, by substitution, [`ROWS`] rows at a time. +fn right_leaf(t: &Tri, m: usize, n: usize, b: &mut [T], ldb: usize) { + if n <= TINY && t.unit { + substitute(Side::Right, t, m, n, b, ldb); + return; + } + // `x·op(a) = b` is `op(a)ᵀ·xᵀ = bᵀ`: the substitution runs on the transpose. + let mut p = Packed::new(); + if !p.fill(t, n, true) { + substitute(Side::Right, t, m, n, b, ldb); + return; + } + let mut i = 0; + while i < m { + #[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] + if let Some((rows, done)) = right_neon(&p, t.unit, m - i, n, &mut b[i..], ldb) { + if !done { + substitute(Side::Right, t, rows, n, &mut b[i..], ldb); + } + i += rows; + continue; + } + let (rows, done) = if m - i >= ROWS { + (ROWS, right_rows::(&p, t.unit, n, i, b, ldb)) + } else if m - i >= 4 { + (4, right_rows::(&p, t.unit, n, i, b, ldb)) + } else { + (1, right_rows::(&p, t.unit, n, i, b, ldb)) + }; + if !done { + substitute(Side::Right, t, rows, n, &mut b[i..], ldb); + } + i += rows; + } +} + +/// The first rows of [`right_leaf`] by the NEON kernels, for `f64` and `f32`: how many rows +/// they took and whether they solved them, or `None` for the other types and for fewer rows +/// than a vector holds. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +fn right_neon( + p: &Packed, + unit: bool, + m: usize, + n: usize, + b: &mut [T], + ldb: usize, +) -> Option<(usize, bool)> { + use core::any::TypeId; + let is_f64 = TypeId::of::() == TypeId::of::(); + let is_f32 = TypeId::of::() == TypeId::of::(); + let lanes = if is_f64 { 2 } else { 4 }; + if !(is_f64 || is_f32) || m < lanes { + return None; + } + let rows = if m >= 16 { 16 } else { lanes }; + assert!((1..=LEAF).contains(&n) && (n - 1) * ldb + rows <= b.len()); + assert!(p.l().len() >= LEAF * n.next_power_of_two().clamp(4, LEAF)); + let (l, r, f) = (p.l(), &p.r, p.forward); + let bp = b.as_mut_ptr(); + // SAFETY: `T` is the type each branch casts to, and the rows lie in the slice, as checked. + let done = unsafe { + let r = r as *const [T; LEAF]; + if is_f64 { + let (l, r, b) = ( + &*(l as *const [T] as *const [f64]), + &*r.cast(), + bp.cast::(), + ); + if rows == 16 { + aarch64::right_f64::<8>(l, r, f, unit, n, b, ldb) + } else { + aarch64::right_f64::<1>(l, r, f, unit, n, b, ldb) + } + } else { + let (l, r, b) = ( + &*(l as *const [T] as *const [f32]), + &*r.cast(), + bp.cast::(), + ); + if rows == 16 { + aarch64::right_f32::<4>(l, r, f, unit, n, b, ldb) + } else { + aarch64::right_f32::<1>(l, r, f, unit, n, b, ldb) + } + } + }; + Some((rows, done)) +} + +/// Rows `i..i + R` of [`right_leaf`]: each unknown column is summed in registers from those +/// solved before it. `false`, with `b` left as it was, when a value is not finite. +#[inline(always)] +fn right_rows( + p: &Packed, + unit: bool, + n: usize, + i: usize, + b: &mut [T], + ldb: usize, +) -> bool { + assert!(n <= LEAF); + let pl = p.l(); + let mut y = [[T::zero(); R]; LEAF]; + let solve = |j: usize, y: &mut [[T; R]; LEAF], before: core::ops::Range| { + let s = i + j * ldb; + let mut acc: [T; R] = b[s..s + R].try_into().unwrap(); + for q in before { + let l = pl[j + q * LEAF]; + for (acc, y) in acc.iter_mut().zip(&y[q]) { + *acc = acc.mul_sub(*y, l); + } + } + y[j] = acc; + }; + if p.forward { + for j in 0..n { + solve(j, &mut y, 0..j); + } + } else { + for j in (0..n).rev() { + solve(j, &mut y, j + 1..n); + } + } + if !y[..n].iter().flatten().all(|v| v.is_finite()) { + return false; + } + for (j, y) in y.iter().enumerate().take(n) { + let s = i + j * ldb; + let col = &mut b[s..s + R]; + if unit { + col.copy_from_slice(y); + } else { + for (b, y) in col.iter_mut().zip(y) { + *b = *y * p.r[j]; + } + } + } + true +} + +/// [`trsm`](fn@super::trsm) of a small triangle by substitution with divisions, element by +/// element: the path for triangles whose reciprocals of the diagonal are not safe to use. +fn substitute(side: Side, t: &Tri, m: usize, n: usize, b: &mut [T], ldb: usize) { + let (lower, unit) = (t.lower, t.unit); + let at = |i: usize, j: usize| t.at(i, j); + match side { + Side::Left => { + for j in 0..n { + let col = &mut b[j * ldb..j * ldb + m]; + if lower { + for i in 0..m { + let mut s = col[i]; + for (p, x) in col[..i].iter().enumerate() { + s -= at(i, p) * *x; + } + col[i] = if unit { s } else { s / at(i, i) }; + } + } else { + for i in (0..m).rev() { + let mut s = col[i]; + for (p, x) in col.iter().enumerate().skip(i + 1) { + s -= at(i, p) * *x; + } + col[i] = if unit { s } else { s / at(i, i) }; + } + } + } + } + Side::Right => { + // Column `j` of `x = b · op(a)⁻¹` solves `x · op(a) = b` from the side `op(a)` is + // triangular towards. + let order: Vec = if lower { + (0..n).rev().collect() + } else { + (0..n).collect() + }; + for &j in &order { + for &p in &order { + if p == j { + break; + } + let x = at(p, j); + for i in 0..m { + let v = b[i + p * ldb]; + b[i + j * ldb] -= v * x; + } + } + if !unit { + let d = at(j, j); + for i in 0..m { + b[i + j * ldb] /= d; + } + } + } + } + } +} diff --git a/lib/src/math/matrix/blas/trsm/aarch64.rs b/lib/src/math/matrix/blas/trsm/aarch64.rs new file mode 100644 index 0000000..4bca62c --- /dev/null +++ b/lib/src/math/matrix/blas/trsm/aarch64.rs @@ -0,0 +1,309 @@ +//! The leaves of the triangular solves in NEON intrinsics, for `f64` and `f32`. +//! +//! The compiler's own code for the portable leaves kept the unknowns in memory and their +//! multiply-adds scalar, as the rows they update start wherever the solved row ends; here the +//! columns of `b` are vectors held in registers, two columns at a time, and each solved row +//! updates the vectors from the one holding the next row, by a lane of its own vector (`fmls +//! v.2d, v.2d, v.d[i]`). The rows of that vector already solved take products by the zeros the +//! packed triangle holds there, which leave finite values as they are; a value that is not finite +//! sends the columns back to substitution with divisions. + +use super::LEAF; +use core::arch::aarch64::*; +use core::mem::MaybeUninit; +use core::ptr; + +/// A left leaf of `$t`: columns of `b` solved against the packed triangle, `V` vectors of `$n` +/// lanes a column, `C` columns at a time. +macro_rules! left { + ( + $name:ident, $t:ty, $n:literal, [$($l:literal),*], [$($rl:literal),*], + $ld:ident, $st:ident, $dup:ident, $fms:ident, $mul:ident, $add:ident, $addv:ident + ) => { + /// The columns `b`, `b + ldb`, … (`C` of them) of `m <= $n · V` rows, solved by the + /// packed triangle `l` of [`Packed`](super::Packed) (forward when it is lower) and + /// scaled by `r` unless `unit` is set, then written to `b` and, when `x` is not null, to + /// `x`, `x + ldx`, …; or `false`, with nothing written, when a value is not finite. + /// + /// # Safety + /// + /// `V · $n <= LEAF`, `m >= 1`, and `l` holds the first `$n · V` columns of the triangle; + /// the `m` rows of the `C` columns of `b` must be valid for reads and writes, and of `x` + /// for writes when it is not null. + #[inline(never)] + #[allow(clippy::too_many_arguments)] + pub(super) unsafe fn $name( + l: &[$t], + r: &[$t; LEAF], + forward: bool, + unit: bool, + m: usize, + b: *mut $t, + ldb: usize, + x: *mut $t, + ldx: usize, + ) -> bool { + // SAFETY: the reads and writes of `b` and `x` are of their rows, as the caller + // guarantees, and those of `l`, `r` and the stack inside them. + unsafe { + let lp = l.as_ptr(); + let full = m == $n * V; + let mut y = [[$dup(0.0); V]; C]; + for (c, y) in y.iter_mut().enumerate() { + let col = b.add(c * ldb); + if full { + for (v, y) in y.iter_mut().enumerate() { + *y = $ld(col.add($n * v)); + } + } else { + let mut t = [0.0 as $t; LEAF]; + ptr::copy_nonoverlapping(col, t.as_mut_ptr(), m); + for (v, y) in y.iter_mut().enumerate() { + *y = $ld(t.as_ptr().add($n * v)); + } + } + } + if forward { + for qq in 0..V { + $( + { + // Row `q` updates the vectors from the one holding row `q + 1`. + let q = $n * qq + $l; + let v0 = if $l + 1 < $n { qq } else { qq + 1 }; + let s: [_; C] = core::array::from_fn(|c| y[c][qq]); + for v in v0..V { + let lv = $ld(lp.add(q * LEAF + $n * v)); + for c in 0..C { + y[c][v] = $fms::<$l>(y[c][v], lv, s[c]); + } + } + } + )* + } + } else { + for qq in (0..V).rev() { + $( + { + // Row `q` updates the vectors up to the one holding row `q - 1`. + let q = $n * qq + $rl; + let v1 = if $rl > 0 { qq + 1 } else { qq }; + let s: [_; C] = core::array::from_fn(|c| y[c][qq]); + for v in 0..v1 { + let lv = $ld(lp.add(q * LEAF + $n * v)); + for c in 0..C { + y[c][v] = $fms::<$rl>(y[c][v], lv, s[c]); + } + } + } + )* + } + } + // `0 · y` is zero for every finite `y` and NaN otherwise. + let mut z = $dup(0.0); + for y in &y { + for y in y { + z = $add(z, $mul(*y, $dup(0.0))); + } + } + if $addv(z) != 0.0 { + return false; + } + if !unit { + for y in &mut y { + for (v, y) in y.iter_mut().enumerate() { + *y = $mul(*y, $ld(r.as_ptr().add($n * v))); + } + } + } + for (c, y) in y.iter().enumerate() { + for (dst, ld) in [(b, ldb), (x, ldx)] { + if dst.is_null() { + continue; + } + let col = dst.add(c * ld); + if full { + for (v, y) in y.iter().enumerate() { + $st(col.add($n * v), *y); + } + } else { + let mut t = [0.0 as $t; LEAF]; + for (v, y) in y.iter().enumerate() { + $st(t.as_mut_ptr().add($n * v), *y); + } + ptr::copy_nonoverlapping(t.as_ptr(), col, m); + } + } + } + true + } + } + }; +} + +left!( + left_f64, + f64, + 2, + [0, 1], + [1, 0], + vld1q_f64, + vst1q_f64, + vdupq_n_f64, + vfmsq_laneq_f64, + vmulq_f64, + vaddq_f64, + vaddvq_f64 +); +left!( + left_f32, + f32, + 4, + [0, 1, 2, 3], + [3, 2, 1, 0], + vld1q_f32, + vst1q_f32, + vdupq_n_f32, + vfmsq_laneq_f32, + vmulq_f32, + vaddq_f32, + vaddvq_f32 +); + +/// A right leaf of `$t`: `$n · V` rows of `b`, solved against the packed transpose of the +/// triangle, `$n` columns at a time. +macro_rules! right { + ( + $name:ident, $t:ty, $n:literal, [$($l:literal),*], + $ld:ident, $st:ident, $dup:ident, $fms:ident, $fmsn:ident, $mul:ident, $add:ident, + $addv:ident + ) => { + /// Rows `b`, …, `b + $n · V - 1` of the `n` columns of `b`, solved by the packed + /// transpose `l` of [`Packed`](super::Packed) (forward when it is lower) and scaled by + /// `r` unless `unit` is set; or `false`, with nothing written, when a value is not + /// finite. + /// + /// Each group of `$n` columns is summed in registers from the columns solved before it, + /// a vector of the triangle's elements by lanes, then solved within itself. + /// + /// # Safety + /// + /// `1 <= n <= LEAF`, and `l` holds the first `n` columns of the triangle rounded up to + /// `$n`; the rows of the `n` columns of `b` must be valid for reads and writes. + #[inline(never)] + pub(super) unsafe fn $name( + l: &[$t], + r: &[$t; LEAF], + forward: bool, + unit: bool, + n: usize, + b: *mut $t, + ldb: usize, + ) -> bool { + // SAFETY: the reads and writes of `b` are of its rows and columns, as the caller + // guarantees, and those of `l`, `r` and the stack inside them. + unsafe { + let lp = l.as_ptr(); + let groups = n.div_ceil($n); + // The solved columns, each written before it is read. + let mut y = [[MaybeUninit::uninit(); V]; LEAF]; + for gg in 0..groups { + let g = if forward { gg } else { groups - 1 - gg }; + let j0 = $n * g; + let mut acc = [[$dup(0.0); V]; $n]; + for (w, acc) in acc.iter_mut().enumerate() { + if j0 + w < n { + let col = b.add((j0 + w) * ldb); + for (v, acc) in acc.iter_mut().enumerate() { + *acc = $ld(col.add($n * v)); + } + } + } + let solved = if forward { 0..j0 } else { j0 + $n..$n * groups }; + for q in solved { + // Elements `(j0 .. j0 + $n, q)` of the transpose, one a lane. + let lv = $ld(lp.add(j0 + q * LEAF)); + let yq = &y[q]; + for v in 0..V { + let yv = yq[v].assume_init(); + $(acc[$l][v] = $fms::<$l>(acc[$l][v], yv, lv);)* + } + } + // The triangle of the group. + if forward { + for w in 0..$n { + for u in w + 1..$n { + let e = *lp.add(j0 + u + (j0 + w) * LEAF); + for v in 0..V { + acc[u][v] = $fmsn(acc[u][v], acc[w][v], e); + } + } + } + } else { + for w in (0..$n).rev() { + for u in 0..w { + let e = *lp.add(j0 + u + (j0 + w) * LEAF); + for v in 0..V { + acc[u][v] = $fmsn(acc[u][v], acc[w][v], e); + } + } + } + } + for (w, acc) in acc.iter().enumerate() { + for (v, acc) in acc.iter().enumerate() { + y[j0 + w][v] = MaybeUninit::new(*acc); + } + } + } + // All of the first `n` columns are written now. + let y = &*(&y[..n] as *const [[MaybeUninit<_>; V]] as *const [[_; V]]); + // `0 · y` is zero for every finite `y` and NaN otherwise. + let mut z = $dup(0.0); + for y in y { + for y in y { + z = $add(z, $mul(*y, $dup(0.0))); + } + } + if $addv(z) != 0.0 { + return false; + } + for (j, y) in y.iter().enumerate() { + let s = if unit { 1.0 } else { r[j] }; + for (v, y) in y.iter().enumerate() { + let p = b.add(j * ldb + $n * v); + $st(p, if unit { *y } else { $mul(*y, $dup(s)) }); + } + } + true + } + } + }; +} + +right!( + right_f64, + f64, + 2, + [0, 1], + vld1q_f64, + vst1q_f64, + vdupq_n_f64, + vfmsq_laneq_f64, + vfmsq_n_f64, + vmulq_f64, + vaddq_f64, + vaddvq_f64 +); +right!( + right_f32, + f32, + 4, + [0, 1, 2, 3], + vld1q_f32, + vst1q_f32, + vdupq_n_f32, + vfmsq_laneq_f32, + vfmsq_n_f32, + vmulq_f32, + vaddq_f32, + vaddvq_f32 +); diff --git a/lib/src/math/matrix/cholesky.rs b/lib/src/math/matrix/cholesky.rs new file mode 100644 index 0000000..8baf39b --- /dev/null +++ b/lib/src/math/matrix/cholesky.rs @@ -0,0 +1,354 @@ +//! The Cholesky decomposition of a Hermitian positive definite matrix. +//! +//! Up to 7 × 7, or 4 × 4 for complex elements, the decomposition is unrolled ([`tiny`]); from +//! 8 to 9, or 10 for `f64`, it is the dot products of [`dot`]. Up to 32 × 32, or 16 × 16 for +//! complex elements, a leaf factors it column by column ([`narrow`]): for `f64` and `f32` the +//! NEON leaf (`aarch64`) holds each column in vector registers while the columns before it are +//! subtracted, its pivot summed apart in scalars so that the square root need not wait for it to +//! be taken out of a vector; complex elements take [`dot`]. Larger matrices are halved: `L11`, then +//! `L21 = A21·L11⁻*` by a triangular solve on the right, reading a copy of `L11` in working memory +//! taken once for the whole recursion, then `A22 - L21·L21*` on its lower triangle, in blocks of +//! [`UPDATE`] columns. The upper triangle of `L` is zeroed as its parts are finished. +//! +//! # Speed +//! +//! Measured on an Apple M4 Max, one core, `f64`, in µs, the median of 21 interleaved batches, +//! against faer 0.24 (`llt`, `Par::Seq`) and nalgebra 0.35 (`cholesky`): +//! +//! | n | ours | faer | nalgebra | +//! | --- | --- | --- | --- | +//! | 2 | 0.018 | 0.071 | 0.019 | +//! | 4 | 0.026 | 0.114 | 0.039 | +//! | 8 | 0.104 | 0.211 | 0.124 | +//! | 16 | 0.256 | 0.436 | 0.399 | +//! | 32 | 1.09 | 1.42 | 1.99 | +//! | 64 | 5.37 | 7.86 | 10.9 | +//! | 128 | 31.7 | 33.1 | 57.1 | +//! | 256 | 170 | 187 | 432 | +//! | 512 | 1 197 | 1 403 | 3 735 | +//! | 1024 | 7 778 | 8 753 | 29 377 | +//! +//! Apple's Accelerate, on the matrix unit (SME), ran 2 to 6 times faster from 32 × 32 up +//! (`dpotrf` 220 µs at 512). + +use super::blas::{Op, Side, Uplo, gemm, trsm, with_scratch}; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; + +// The leaves of `f64` and `f32` in NEON intrinsics on little-endian aarch64; the portable leaf +// serves the other types and targets, and Miri. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +mod aarch64; + +/// The Cholesky decomposition `A = L·L*` of a Hermitian (for real elements, symmetric) positive +/// definite matrix, `L` lower triangular with a positive real diagonal. From +/// [`Matrix::cholesky`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a = Matrix::from_rows(&[[4.0, 2.0], [2.0, 5.0]]); +/// let c = a.cholesky().unwrap(); +/// assert_eq!(c.l(), Matrix::from_rows(&[[2.0, 0.0], [1.0, 2.0]])); +/// assert_eq!(c.det(), 16.0); +/// assert!(Matrix::from_rows(&[[1.0, 2.0], [2.0, 1.0]]).cholesky().is_none()); +/// ``` +#[derive(Clone, Debug)] +pub struct Cholesky { + /// `L`, zeros above the diagonal. + l: Matrix, +} + +impl Matrix { + /// The Cholesky decomposition `A = L·L*`, or `None` when the matrix is not positive + /// definite: a pivot is not positive, or not a number. Only the lower triangle is read, the + /// upper one taken to be its conjugate. About `n³/3` operations, nearly all of them in + /// products of blocks. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn cholesky(&self) -> Option> { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let mut l = self.clone(); + match n { + 0 => {} + 1 => tiny::(&mut l.data)?, + 2 => tiny::(&mut l.data)?, + 3 => tiny::(&mut l.data)?, + 4 => tiny::(&mut l.data)?, + // Unrolled past 4, complex elements ran slower than by the loops; real ones ran + // fastest unrolled up to 7, by the dot products of [`dot`] at 8 and 9 (and 10 for + // `f64`), and by the NEON leaves past that. + 5 if !T::IS_COMPLEX => tiny::(&mut l.data)?, + 6 if !T::IS_COMPLEX => tiny::(&mut l.data)?, + 7 if !T::IS_COMPLEX => tiny::(&mut l.data)?, + 8 | 9 => dot(&mut l.data, n, n)?, + 10 if size_of::() == 8 => dot(&mut l.data, n, n)?, + _ if n <= narrow_order::() => narrow(&mut l.data, n, n)?, + _ => { + // The working memory of the recursion: a copy of the largest `L11`. + let n1 = split::(n); + with_scratch(n1 * n1, |work| factor(&mut l.data, n, n, work))?; + } + } + Some(Cholesky { l }) + } +} + +impl Cholesky { + /// `L`, lower triangular with a positive real diagonal. + pub fn l(&self) -> Matrix { + self.l.clone() + } + + /// The determinant of the matrix, the square of the product of the diagonal of `L`: real + /// and positive. + pub fn det(&self) -> T::Real { + let mut d = T::Real::ONE; + for x in self.l.diagonal() { + let r = x.re(); + d *= r * r; + } + d + } + + /// The solution `X` of `A·X = B`. + /// + /// # Panics + /// + /// Panics if `B` has not as many rows as the matrix. + #[track_caller] + pub fn solve(&self, b: &Matrix) -> Matrix { + let n = self.l.rows(); + assert_eq!(b.rows(), n, "the right-hand side has not as many rows"); + let mut x = b.clone(); + let cols = x.cols(); + trsm( + Side::Left, + Uplo::Lower, + Op::N, + false, + n, + cols, + &self.l.data, + n, + &mut x.data, + n, + ); + trsm( + Side::Left, + Uplo::Lower, + Op::C, + false, + n, + cols, + &self.l.data, + n, + &mut x.data, + n, + ); + x + } + + /// The inverse of the matrix. + pub fn inverse(&self) -> Matrix { + self.solve(&Matrix::identity(self.l.rows())) + } +} + +/// Blocks up to this order are factored without recursion: 32 for real elements, whose leaves +/// keep their columns in vector registers, and 16 for complex ones. +const fn narrow_order() -> usize { + if T::IS_COMPLEX { 16 } else { 32 } +} + +/// Where the recursion splits `n` columns: the first part a multiple of [`narrow_order`]. +fn split(n: usize) -> usize { + (n / 2).next_multiple_of(narrow_order::()).min(n - 1) +} + +/// Factors the `n × n` block at the start of `a` in place, its lower triangle: `L11`, then +/// `L21 = A21·L11⁻*`, then `A22 - L21·L21*` the same way, and zeroes the upper triangle. +/// `L11` and `L21` share their columns, so the solve reads a copy of `L11` in `work`, of at +/// least `split(n)²` elements. +fn factor(a: &mut [T], n: usize, ld: usize, work: &mut [T]) -> Option<()> { + if n <= narrow_order::() { + return narrow(a, n, ld); + } + let n1 = split::(n); + let n2 = n - n1; + factor(a, n1, ld, work)?; + let (left, right) = a.split_at_mut(n1 * ld); + let l11 = &mut work[..n1 * n1]; + for j in 0..n1 { + l11[j * n1 + j..(j + 1) * n1].copy_from_slice(&left[j * ld + j..j * ld + n1]); + } + trsm( + Side::Right, + Uplo::Lower, + Op::C, + false, + n2, + n1, + l11, + n1, + &mut left[n1..], + ld, + ); + // `A22 -= L21·L21*` on the lower triangle only, a block of columns at a time from its + // diagonal down: about half the products of the whole block. + let l21 = &left[n1..]; + let a22 = &mut right[n1..]; + let mut j0 = 0; + while j0 < n2 { + let w = UPDATE.min(n2 - j0); + gemm( + n2 - j0, + w, + n1, + -T::one(), + &l21[j0..], + ld, + Op::N, + &l21[j0..], + ld, + Op::C, + T::one(), + &mut a22[j0 + j0 * ld..], + ld, + ); + j0 += w; + } + factor(a22, n2, ld, work)?; + // The upper triangle of `L` is zeros: `A12`, and the triangles of the leaves. + for j in 0..n2 { + right[j * ld..j * ld + n1].fill(T::zero()); + } + Some(()) +} + +/// The width of the column blocks of the update of the trailing block. +const UPDATE: usize = 64; + +/// Factors a small block column by column, then zeroes the upper triangle: by the NEON leaves +/// for `f64` and `f32`, and by [`dot`] for the other types and targets, and Miri. +fn narrow(a: &mut [T], n: usize, ld: usize) -> Option<()> { + #[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] + if let Some(done) = narrow_neon(a, n, ld) { + return done.then_some(()); + } + dot(a, n, ld) +} + +/// [`narrow`] without the NEON leaves: each element of column `j` is a dot product of its row +/// with row `j`, the products not fused, so that the sums' chains wait on additions alone. +fn dot(a: &mut [T], n: usize, ld: usize) -> Option<()> { + for j in 0..n { + let mut d = a[j + j * ld].re(); + for p in 0..j { + d -= a[j + p * ld].abs2(); + } + if d.is_nan() || d <= T::Real::ZERO { + return None; + } + let d = d.sqrt(); + a[j + j * ld] = T::from_real(d); + let r = T::Real::ONE / d; + for i in j + 1..n { + let mut s = a[i + j * ld]; + for p in 0..j { + s -= a[i + p * ld] * a[j + p * ld].conj(); + } + a[i + j * ld] = s.scale(r); + } + } + for j in 1..n { + a[j * ld..j * ld + j].fill(T::zero()); + } + Some(()) +} + +/// [`narrow`] by the NEON leaves, for `f64` and `f32`: whether the block was positive +/// definite, or `None` for the other types. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +fn narrow_neon(a: &mut [T], n: usize, ld: usize) -> Option { + use core::any::TypeId; + let is_f64 = TypeId::of::() == TypeId::of::(); + if !(is_f64 || TypeId::of::() == TypeId::of::()) { + return None; + } + assert!((1..=aarch64::ROWS).contains(&n) && (n - 1) * ld + n <= a.len()); + let len = a.len(); + let a = a.as_mut_ptr(); + // SAFETY: `T` is the type each branch casts to, and the block lies in the slice, as + // checked. + Some(unsafe { + if is_f64 { + let a = a.cast::(); + match n { + 1..=2 => aarch64::leaf_f64::<1, 2>(a, n, ld, len), + 3..=4 => aarch64::leaf_f64::<2, 4>(a, n, ld, len), + 5..=6 => aarch64::leaf_f64::<3, 6>(a, n, ld, len), + 7..=8 => aarch64::leaf_f64::<4, 8>(a, n, ld, len), + 9..=12 => aarch64::leaf_f64::<6, 12>(a, n, ld, len), + 13..=16 => aarch64::leaf_f64::<8, 16>(a, n, ld, len), + 17..=24 => aarch64::leaf_f64::<12, 24>(a, n, ld, len), + _ => aarch64::leaf_f64::<16, 32>(a, n, ld, len), + } + } else { + let a = a.cast::(); + match n { + 1..=4 => aarch64::leaf_f32::<1, 4>(a, n, ld, len), + 5..=8 => aarch64::leaf_f32::<2, 8>(a, n, ld, len), + 9..=12 => aarch64::leaf_f32::<3, 12>(a, n, ld, len), + 13..=16 => aarch64::leaf_f32::<4, 16>(a, n, ld, len), + 17..=24 => aarch64::leaf_f32::<6, 24>(a, n, ld, len), + _ => aarch64::leaf_f32::<8, 32>(a, n, ld, len), + } + } + }) +} + +/// [`narrow`] of an `N × N` matrix, `N` known, so that its loops unroll. +#[inline(always)] +fn tiny(a: &mut [T]) -> Option<()> { + let a = &mut a[..N * N]; + for j in 0..N { + for p in 0..j { + let s = a[j + p * N].conj(); + for i in j..N { + a[i + j * N] = a[i + j * N].mul_sub(a[i + p * N], s); + } + } + let d = pivot(&mut a[j + j * N..(j + 1) * N])?; + let r = T::Real::ONE / d; + for i in j + 1..N { + a[i + j * N] = a[i + j * N].scale(r); + } + } + for j in 1..N { + for i in 0..j { + a[i + j * N] = T::zero(); + } + } + Some(()) +} + +/// The pivot of a column, its first element, updated: its square root, written in its place, +/// or `None` when it is not positive, or not a number. Its imaginary part, the rounding of +/// products of conjugates, is dropped. +#[inline(always)] +fn pivot(col: &mut [T]) -> Option { + let d = col[0].re(); + if d.is_nan() || d <= T::Real::ZERO { + return None; + } + let d = d.sqrt(); + col[0] = T::from_real(d); + Some(d) +} diff --git a/lib/src/math/matrix/cholesky/aarch64.rs b/lib/src/math/matrix/cholesky/aarch64.rs new file mode 100644 index 0000000..0bfcb62 --- /dev/null +++ b/lib/src/math/matrix/cholesky/aarch64.rs @@ -0,0 +1,160 @@ +//! The leaf of the Cholesky decomposition in NEON intrinsics, for `f64` and `f32`. +//! +//! The leaf is copied to the stack, its columns padded to whole vectors, and factored left +//! looking: each column is held in registers while the columns before it are subtracted, whole +//! vectors at a time with the multiplier broadcast, then scaled by the reciprocal of its pivot's +//! square root. The subtractions start at the section of the vectors (a quarter, a half or all +//! of them, as their number allows) that holds the column's diagonal, so that their lengths are +//! known and the column stays in registers. Whole vectors +//! include rows above the diagonal: they take products of elements of the upper triangle, which +//! the lower triangle never reads, and are written back as zeros. +//! +//! The pivot is summed apart in scalars, the same products in the same order as its lane: taking +//! it out of the vector cost a store and a load on the path from one column to the next, which +//! held a 9 × 9 decomposition at 167 ns against 136 without. + +use core::arch::aarch64::*; +use core::mem::MaybeUninit; +use core::ptr; + +/// The most rows of a leaf. +pub(super) const ROWS: usize = 32; + +/// A Cholesky leaf of `$t` in vectors of `$n` lanes. +macro_rules! leaf { + ($name:ident, $t:ty, $n:literal, $ld:ident, $st:ident, $dup:ident, $fmsn:ident, $mul:ident) => { + /// Factors the `n × n` block at `a`, columns `ld` apart, `n <= K = $n · V`, in place: + /// see the module. `false` when a pivot is not positive, or not a number, as soon as it + /// is met. + /// + /// # Safety + /// + /// `K = $n · V <= ROWS`; the `n` rows of the `n` columns of `a` must be valid for reads + /// and writes, and the `len` elements from `a` for reads. + #[inline(never)] + pub(super) unsafe fn $name( + a: *mut $t, + n: usize, + ld: usize, + len: usize, + ) -> bool { + // SAFETY: the reads and writes of `a` are of its block or, for whole vectors, of the + // `len` elements from it, as the caller guarantees, and those of the stack inside + // it. + unsafe { + // The first `n` columns, each written here before it is read; the others are + // never read. + let mut l = [[MaybeUninit::uninit(); V]; K]; + for (j, col) in l.iter_mut().enumerate().take(n) { + let src = a.add(j * ld); + if j * ld + K <= len { + // Whole vectors, the rows past the block's taken as padding. + for (v, x) in col.iter_mut().enumerate() { + *x = MaybeUninit::new($ld(src.add($n * v))); + } + } else { + let mut t = [0.0 as $t; K]; + ptr::copy_nonoverlapping(src, t.as_mut_ptr(), n); + for (v, x) in col.iter_mut().enumerate() { + *x = MaybeUninit::new($ld(t.as_ptr().add($n * v))); + } + } + } + let l = &mut *(&mut l[..n] as *mut [[MaybeUninit<_>; V]] as *mut [[_; V]]); + // The rows from `j` on start in a later section of the vectors as `j` grows: + // the update of each section has a constant length, so that `y` stays in + // registers. + let sections = if V % 4 == 0 { + 4 + } else if V % 2 == 0 { + 2 + } else { + 1 + }; + for j in 0..n { + let mut y = l[j]; + let diagonal = l.as_mut_ptr().cast::<$t>().add(j * K + j); + // The pivot is summed apart, in the order and with the roundings of its lane, + // so that it need not be taken out of the vector. + let mut d = *diagonal; + macro_rules! update { + ($from:expr) => { + for p in 0..j { + let s = *l.as_ptr().cast::<$t>().add(p * K + j); + d = (-s).mul_add(s, d); + for v in $from..V { + y[v] = $fmsn(y[v], l[p][v], s); + } + } + }; + } + match j * sections / K { + 0 => update!(0), + 1 => update!(V / sections), + 2 => update!(2 * V / sections), + _ => update!(3 * V / sections), + } + if d.is_nan() || d <= 0.0 { + return false; + } + let d = d.sqrt(); + let r = $dup(1.0 / d); + for y in &mut y { + *y = $mul(*y, r); + } + l[j] = y; + *diagonal = d; + } + // The columns back, zeros above the diagonal: whole vectors, then the last + // row alone. + let whole = n / $n; + for (j, col) in l.iter().enumerate() { + let dst = a.add(j * ld); + for (v, x) in col.iter().enumerate() { + let mut x = *x; + if $n * v + $n <= j { + x = $dup(0.0); + } else if $n * v < j { + // The vector across the diagonal. + let mut t = [0.0 as $t; $n]; + $st(t.as_mut_ptr(), x); + t[..j - $n * v].fill(0.0); + x = $ld(t.as_ptr()); + } + if v < whole { + $st(dst.add($n * v), x); + } else if v == whole && n > $n * whole { + let mut t = [0.0 as $t; $n]; + $st(t.as_mut_ptr(), x); + for (i, t) in t.iter().enumerate().take(n - $n * whole) { + *dst.add($n * whole + i) = *t; + } + } + } + } + true + } + } + }; +} + +leaf!( + leaf_f64, + f64, + 2, + vld1q_f64, + vst1q_f64, + vdupq_n_f64, + vfmsq_n_f64, + vmulq_f64 +); +leaf!( + leaf_f32, + f32, + 4, + vld1q_f32, + vst1q_f32, + vdupq_n_f32, + vfmsq_n_f32, + vmulq_f32 +); diff --git a/lib/src/math/matrix/eigen.rs b/lib/src/math/matrix/eigen.rs new file mode 100644 index 0000000..08471ca --- /dev/null +++ b/lib/src/math/matrix/eigen.rs @@ -0,0 +1,190 @@ +//! Eigenvalues and eigenvectors: of any square matrix from its Schur form, and of a Hermitian +//! one by a reduction to a real tridiagonal matrix and divide and conquer. + +use super::hermitian::hermitian; +use super::schur::{eigenvectors, schur}; +use super::{Matrix, Scalar}; +use crate::math::complex::{Complex, Float}; + +/// The eigendecomposition `A·V = V·Λ` of a square matrix: its eigenvalues, the diagonal of `Λ`, +/// and its eigenvectors, the columns of `V`, complex in general, each of length 1. When `A` has +/// a full set of independent eigenvectors, `A = V·Λ·V⁻¹`. From [`Matrix::eigen`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::complex::Complex; +/// use a::math::matrix::Matrix; +/// +/// let a: Matrix = Matrix::from_rows(&[[2.0, 1.0], [1.0, 2.0]]); +/// let e = a.eigen(); +/// let (v, w) = (e.vectors(), e.values()); +/// // Each column is an eigenvector: A·v = λ·v. +/// let a = a.map(|&x| Complex::from(x)); +/// for (j, &w) in w.iter().enumerate() { +/// let v = Matrix::from_columns(&[v.column(j)]); +/// let d = &(&a * &v) - &(&v * w); +/// assert!(d.as_slice().iter().all(|x| x.abs() < 1e-14)); +/// } +/// ``` +#[derive(Clone, Debug)] +pub struct Eigen { + /// The eigenvalues. + values: Vec>, + /// The eigenvectors, as columns. + vectors: Matrix>, +} + +impl Eigen { + /// The eigenvalues, in the order of the columns of [`vectors`](Eigen::vectors). + pub fn values(&self) -> &[Complex] { + &self.values + } + + /// The eigenvectors, each a column of length 1 in the 2-norm. + pub fn vectors(&self) -> &Matrix> { + &self.vectors + } +} + +/// The eigendecomposition `A = V·Λ·V*` of a Hermitian (for real elements, symmetric) matrix: +/// real eigenvalues in ascending order, and orthonormal eigenvectors, the columns of `V`, of the +/// matrix's own type. From [`Matrix::hermitian_eigen`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a: Matrix = Matrix::from_rows(&[[2.0, 1.0], [1.0, 2.0]]); +/// let e = a.hermitian_eigen(); +/// assert!((e.values()[0] - 1.0).abs() < 1e-15 && (e.values()[1] - 3.0).abs() < 1e-15); +/// let v = e.vectors(); +/// let back = &(v * &Matrix::from_diagonal(e.values())) * &v.transpose(); +/// assert!((&back - &a).as_slice().iter().all(|x| x.abs() < 1e-14)); +/// ``` +#[derive(Clone, Debug)] +pub struct HermitianEigen { + /// The eigenvalues, ascending. + values: Vec, + /// The eigenvectors, as columns. + vectors: Matrix, +} + +impl HermitianEigen { + /// The eigenvalues, real, in ascending order. + pub fn values(&self) -> &[T::Real] { + &self.values + } + + /// The eigenvectors, orthonormal columns in the order of the eigenvalues. + pub fn vectors(&self) -> &Matrix { + &self.vectors + } +} + +impl Matrix { + /// The eigenvalues and eigenvectors: see [`Eigen`]. From the Schur form `A = Q·T·Q*`, the + /// eigenvectors of `T` by back substitution, in real arithmetic for real elements, and `Q` + /// times them. A matrix with a NaN or an infinity gives NaNs. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn eigen(&self) -> Eigen { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let (t, q, values) = schur(self, true, true); + let vectors = eigenvectors(&t.data, &q.unwrap_or_default().data, n); + Eigen { + values, + vectors: Matrix::from_column_major(n, n, vectors), + } + } + + /// The eigenvalues and orthonormal eigenvectors of a Hermitian (for real elements, + /// symmetric) matrix: see [`HermitianEigen`]. A blocked reduction to a real symmetric + /// tridiagonal matrix by Householder reflections, its eigensystem by Cuppen's divide and + /// conquer, then the reflections applied to it in blocks; up to about a hundred rows (80 to + /// 160 by the type), where that takes longer, the reduction one column at a time, its + /// reflections multiplied out into `Q`, and the implicit QL algorithm's rotations applied + /// to `Q`. Only the lower triangle is read, the upper one taken to be its conjugate. A + /// matrix with a NaN or an infinity gives NaNs. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn hermitian_eigen(&self) -> HermitianEigen { + assert!(self.is_square(), "the matrix is not square"); + let (values, vectors) = hermitian(self, true); + HermitianEigen { + values, + vectors: vectors.unwrap_or_default(), + } + } + + /// The eigenvalues of a Hermitian (for real elements, symmetric) matrix, real and + /// ascending, without the eigenvectors: the same reduction, then the QL and QR algorithm + /// without square roots, from about a hundred rows less than half the time of + /// [`hermitian_eigen`](Matrix::hermitian_eigen). Only the lower triangle is read. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn hermitian_eigenvalues(&self) -> Vec { + assert!(self.is_square(), "the matrix is not square"); + hermitian(self, false).0 + } +} + +/// Turns the real Schur form `t`, held as complex numbers, into a complex upper triangular one, +/// a unitary rotation for each 2 × 2 block, applied to `q` too, as MATLAB's `rsf2csf`. +pub(crate) fn real_to_complex_schur(t: &mut [C], q: &mut [C], n: usize) { + let at = |i: usize, j: usize| i + j * n; + let zero = C::Real::ZERO; + for m in (1..n).rev() { + let sub = t[at(m, m - 1)]; + if sub == C::zero() { + continue; + } + // The block's eigenvalue with a positive imaginary part, less `t[m, m]`, from the block + // divided by its largest part: the rotation depends only on the ratio of `mu` to `c`, + // and the products of the block's own elements could overflow or underflow, as those of + // `[[0, -1e200], [1e200, 0]]` do. + let (a, b, c, d) = (t[at(m - 1, m - 1)], t[at(m - 1, m)], sub, t[at(m, m)]); + let part = |x: C| x.re().abs().max(x.im().abs()); + let s = part(a).max(part(b)).max(part(c)).max(part(d)); + let down = |x: C| C::from_parts(x.re() / s, x.im() / s); + let (a, b, c, d) = (down(a), down(b), down(c), down(d)); + let half = (a - d).scale(C::Real::HALF); + let disc = (half * half + b * c).sqrt(); + let mu = half + disc; + let r = mu.abs().hypot(c.abs()); + if r == zero { + continue; + } + let cs = mu.scale(C::Real::ONE / r); + let sn = c.scale(C::Real::ONE / r); + // `G = [[cs*, sn], [-sn, cs]]` from the left on rows `m - 1, m`, `G*` from the right on + // columns `m - 1, m`. + for j in m - 1..n { + let (x, y) = (t[at(m - 1, j)], t[at(m, j)]); + t[at(m - 1, j)] = cs.conj() * x + sn * y; + t[at(m, j)] = -sn.conj() * x + cs * y; + } + for i in 0..=m { + let (x, y) = (t[at(i, m - 1)], t[at(i, m)]); + t[at(i, m - 1)] = x * cs + y * sn.conj(); + t[at(i, m)] = -(x * sn) + y * cs.conj(); + } + for i in 0..n { + let (x, y) = (q[at(i, m - 1)], q[at(i, m)]); + q[at(i, m - 1)] = x * cs + y * sn.conj(); + q[at(i, m)] = -(x * sn) + y * cs.conj(); + } + t[at(m, m - 1)] = C::zero(); + } +} diff --git a/lib/src/math/matrix/element.rs b/lib/src/math/matrix/element.rs new file mode 100644 index 0000000..64d5e2e --- /dev/null +++ b/lib/src/math/matrix/element.rs @@ -0,0 +1,484 @@ +//! The types a matrix can hold. + +use super::blas::{Op, gemm}; +use crate::math::complex::{Complex, Float}; +use crate::math::numbers::Number; +use core::fmt; +use core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign}; + +/// What the arithmetic of a matrix needs of its elements, sealed. +pub(crate) mod sealed { + use super::*; + + /// The ring operations, by reference so that a [`Number`] need not be copied. + pub trait Element: Clone + PartialEq + fmt::Debug { + /// 0. + fn zero() -> Self; + /// 1. + fn one() -> Self; + /// Whether it is 0. + fn is_zero(&self) -> bool; + /// The complex conjugate; the element itself when it is real. + fn conj(&self) -> Self; + /// `a + b`. + fn add(a: &Self, b: &Self) -> Self; + /// `a - b`. + fn sub(a: &Self, b: &Self) -> Self; + /// `a · b`. + fn mul(a: &Self, b: &Self) -> Self; + /// `-a`. + fn neg(a: &Self) -> Self; + /// `*a += b`. + fn add_assign(a: &mut Self, b: &Self); + /// `*a -= b`. + fn sub_assign(a: &mut Self, b: &Self); + /// `*a *= b`. + fn mul_assign(a: &mut Self, b: &Self); + /// `*c += a · b`. + fn mul_add(c: &mut Self, a: &Self, b: &Self); + + /// `c = a · b` for column-major `a` of `m × k`, `b` of `k × n` and `c` of `m × n`, which + /// holds zeros when it is called. + fn product(c: &mut [Self], a: &[Self], b: &[Self], m: usize, k: usize, n: usize) { + for j in 0..n { + let c = &mut c[j * m..(j + 1) * m]; + for p in 0..k { + let x = &b[p + j * k]; + if x.is_zero() { + continue; + } + for (c, a) in c.iter_mut().zip(&a[p * m..(p + 1) * m]) { + Self::mul_add(c, a, x); + } + } + } + } + } + + /// The field operations of floats and complex numbers, by value. + pub trait Scalar: + Element + + Copy + + Default + + Add + + Sub + + Mul + + Div + + Neg + + AddAssign + + SubAssign + + MulAssign + + DivAssign + + Send + + Sync + + 'static + { + /// The real type of the absolute values, the public trait's `Real`. + type R: Float; + /// The complex type of the same precision, which eigenvalues of real matrices need. + type C: super::Scalar; + /// The real type of the same precision as an element, for the real arithmetic inside + /// complex algorithms. + type Re: super::Scalar; + /// Whether the type is complex. + const IS_COMPLEX: bool; + /// The real part. + fn re(self) -> Self::R; + /// The imaginary part. + fn im(self) -> Self::R; + /// The number with the real part `re`. + fn from_real(re: Self::R) -> Self; + /// The number `re + im·i`, or `re` when the type is real. + fn from_parts(re: Self::R, im: Self::R) -> Self; + /// The absolute value. + fn abs(self) -> Self::R; + /// The square of the absolute value. + fn abs2(self) -> Self::R; + /// `|re| + |im|`, cheaper than the absolute value and within a factor √2 of it. + fn abs1(self) -> Self::R; + /// The product by a real number. + fn scale(self, r: Self::R) -> Self; + /// `self - a·b`, with fused multiply-adds: one for a real number, four for a complex one. + fn mul_sub(self, a: Self, b: Self) -> Self; + /// The principal square root. + fn sqrt(self) -> Self; + /// The complex number. + fn to_complex(self) -> Complex; + /// The element as its complex type. + fn to_c(self) -> Self::C; + /// The element of the complex type `c`, its real part for a real type. + fn from_c(c: Self::C) -> Self; + /// Whether both parts are finite. + fn is_finite(self) -> bool; + /// `c = β·c + α·a·b` by `gemm` on blocks of the real type `R`, which code generic over the + /// element reaches no other way. + #[allow(clippy::too_many_arguments)] + fn gemm_real( + m: usize, + n: usize, + k: usize, + alpha: Self::R, + a: &[Self::R], + lda: usize, + b: &[Self::R], + ldb: usize, + beta: Self::R, + c: &mut [Self::R], + ldc: usize, + ); + /// [`gemm`](super::super::blas::gemm) on the type. + #[allow(clippy::too_many_arguments)] + fn gemm( + m: usize, + n: usize, + k: usize, + alpha: Self, + a: &[Self], + lda: usize, + opa: Op, + b: &[Self], + ldb: usize, + opb: Op, + beta: Self, + c: &mut [Self], + ldc: usize, + ); + } +} + +/// A type a [`Matrix`](super::Matrix) can hold and compute with exactly: `f32`, `f64`, +/// [`Complex`], [`Complex`] and [`Number`]. The trait is sealed. +pub trait Element: sealed::Element {} + +/// An element whose matrices have the numerical algorithms, the decompositions, the eigenvalues +/// and the functions of matrices: `f32`, `f64`, [`Complex`] and [`Complex`]. The trait +/// is sealed. +pub trait Scalar: Element + sealed::Scalar::Real> { + /// The float type of the parts and the absolute values, as of norms and singular values: + /// `f32` for `f32` and `Complex`, `f64` for `f64` and `Complex`. + type Real: Float; +} + +impl Element for Number {} + +impl sealed::Element for Number { + #[inline] + fn zero() -> Self { + Number::default() + } + #[inline] + fn one() -> Self { + Number::from(1u8) + } + #[inline] + fn is_zero(&self) -> bool { + Number::is_zero(self) + } + #[inline] + fn conj(&self) -> Self { + self.clone() + } + #[inline] + fn add(a: &Self, b: &Self) -> Self { + a + b + } + #[inline] + fn sub(a: &Self, b: &Self) -> Self { + a - b + } + #[inline] + fn mul(a: &Self, b: &Self) -> Self { + a * b + } + #[inline] + fn neg(a: &Self) -> Self { + -a + } + #[inline] + fn add_assign(a: &mut Self, b: &Self) { + *a += b; + } + #[inline] + fn sub_assign(a: &mut Self, b: &Self) { + *a -= b; + } + #[inline] + fn mul_assign(a: &mut Self, b: &Self) { + *a *= b; + } + #[inline] + fn mul_add(c: &mut Self, a: &Self, b: &Self) { + if !a.is_zero() { + *c += a * b; + } + } +} + +/// [`sealed::Scalar::gemm`] by the function `$f` of [`gemm`](super::gemm). +macro_rules! gemm { + ($f:ident) => { + #[inline] + #[track_caller] + fn gemm( + m: usize, + n: usize, + k: usize, + alpha: Self, + a: &[Self], + lda: usize, + opa: Op, + b: &[Self], + ldb: usize, + opb: Op, + beta: Self, + c: &mut [Self], + ldc: usize, + ) { + super::gemm::$f(m, n, k, alpha, a, lda, opa, b, ldb, opb, beta, c, ldc); + } + }; +} + +/// [`sealed::Element`] by the operators of a `Copy` type. +macro_rules! copy_element { + ($zero:expr, $one:expr, $conj:expr) => { + #[inline] + fn zero() -> Self { + $zero + } + #[inline] + fn one() -> Self { + $one + } + #[inline] + fn is_zero(&self) -> bool { + *self == $zero + } + #[inline] + fn conj(&self) -> Self { + $conj(*self) + } + #[inline] + fn add(a: &Self, b: &Self) -> Self { + *a + *b + } + #[inline] + fn sub(a: &Self, b: &Self) -> Self { + *a - *b + } + #[inline] + fn mul(a: &Self, b: &Self) -> Self { + *a * *b + } + #[inline] + fn neg(a: &Self) -> Self { + -*a + } + #[inline] + fn add_assign(a: &mut Self, b: &Self) { + *a += *b; + } + #[inline] + fn sub_assign(a: &mut Self, b: &Self) { + *a -= *b; + } + #[inline] + fn mul_assign(a: &mut Self, b: &Self) { + *a *= *b; + } + #[inline] + fn mul_add(c: &mut Self, a: &Self, b: &Self) { + *c += *a * *b; + } + fn product(c: &mut [Self], a: &[Self], b: &[Self], m: usize, k: usize, n: usize) { + gemm(m, n, k, $one, a, m, Op::N, b, k, Op::N, $zero, c, m); + } + }; +} + +macro_rules! real { + ($t:ident) => { + impl Element for $t {} + impl Scalar for $t { + type Real = $t; + } + + impl sealed::Element for $t { + copy_element!(0.0, 1.0, core::convert::identity); + } + + impl sealed::Scalar for $t { + type R = $t; + type C = Complex<$t>; + type Re = $t; + const IS_COMPLEX: bool = false; + #[inline] + fn re(self) -> $t { + self + } + #[inline] + fn im(self) -> $t { + 0.0 + } + #[inline] + fn from_real(re: $t) -> Self { + re + } + #[inline] + fn from_parts(re: $t, _: $t) -> Self { + re + } + #[inline] + fn abs(self) -> $t { + $t::abs(self) + } + #[inline] + fn abs2(self) -> $t { + self * self + } + #[inline] + fn abs1(self) -> $t { + $t::abs(self) + } + #[inline] + fn scale(self, r: $t) -> Self { + self * r + } + #[inline(always)] + fn mul_sub(self, a: Self, b: Self) -> Self { + $t::mul_add(-a, b, self) + } + #[inline] + fn sqrt(self) -> Self { + $t::sqrt(self) + } + #[inline] + fn to_complex(self) -> Complex<$t> { + Complex::new(self, 0.0) + } + #[inline] + fn to_c(self) -> Complex<$t> { + Complex::new(self, 0.0) + } + #[inline] + fn from_c(c: Complex<$t>) -> Self { + c.re + } + #[inline] + fn is_finite(self) -> bool { + $t::is_finite(self) + } + #[inline] + fn gemm_real( + m: usize, + n: usize, + k: usize, + alpha: $t, + a: &[$t], + lda: usize, + b: &[$t], + ldb: usize, + beta: $t, + c: &mut [$t], + ldc: usize, + ) { + gemm(m, n, k, alpha, a, lda, Op::N, b, ldb, Op::N, beta, c, ldc); + } + gemm!(real); + } + + impl Element for Complex<$t> {} + impl Scalar for Complex<$t> { + type Real = $t; + } + + impl sealed::Element for Complex<$t> { + copy_element!(Complex::<$t>::ZERO, Complex::<$t>::ONE, Complex::conj); + } + + impl sealed::Scalar for Complex<$t> { + type R = $t; + type C = Self; + type Re = $t; + const IS_COMPLEX: bool = true; + #[inline] + fn re(self) -> $t { + self.re + } + #[inline] + fn im(self) -> $t { + self.im + } + #[inline] + fn from_real(re: $t) -> Self { + Complex::new(re, 0.0) + } + #[inline] + fn from_parts(re: $t, im: $t) -> Self { + Complex::new(re, im) + } + #[inline] + fn abs(self) -> $t { + Complex::abs(self) + } + #[inline] + fn abs2(self) -> $t { + self.norm_sqr() + } + #[inline] + fn abs1(self) -> $t { + self.re.abs() + self.im.abs() + } + #[inline] + fn scale(self, r: $t) -> Self { + self * r + } + #[inline(always)] + fn mul_sub(self, a: Self, b: Self) -> Self { + let re = $t::mul_add(a.im, b.im, $t::mul_add(-a.re, b.re, self.re)); + let im = $t::mul_add(-a.im, b.re, $t::mul_add(-a.re, b.im, self.im)); + Complex::new(re, im) + } + #[inline] + fn sqrt(self) -> Self { + Complex::sqrt(self) + } + #[inline] + fn to_complex(self) -> Complex<$t> { + self + } + #[inline] + fn to_c(self) -> Self { + self + } + #[inline] + fn from_c(c: Self) -> Self { + c + } + #[inline] + fn is_finite(self) -> bool { + Complex::is_finite(self) + } + #[inline] + fn gemm_real( + m: usize, + n: usize, + k: usize, + alpha: $t, + a: &[$t], + lda: usize, + b: &[$t], + ldb: usize, + beta: $t, + c: &mut [$t], + ldc: usize, + ) { + gemm(m, n, k, alpha, a, lda, Op::N, b, ldb, Op::N, beta, c, ldc); + } + gemm!(complex); + } + }; +} + +real!(f32); +real!(f64); diff --git a/lib/src/math/matrix/exact.rs b/lib/src/math/matrix/exact.rs new file mode 100644 index 0000000..b27aeef --- /dev/null +++ b/lib/src/math/matrix/exact.rs @@ -0,0 +1,273 @@ +//! Exact algorithms on matrices of numbers: determinants, ranks, characteristic polynomials, +//! adjugates, inverses and null spaces, without fractions. + +use super::Matrix; +use crate::math::numbers::Number; + +impl Matrix { + /// The determinant, exactly, by Bareiss's fraction-free elimination: each step's elements + /// are minors of the matrix, `(a_ij·a_kk - a_ik·a_kj) / p` with the previous pivot `p` + /// dividing exactly, so the numbers grow no longer than the determinant. About `n³` + /// products. The determinant of the `0 × 0` matrix is 1. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// use a::math::numbers::Number; + /// + /// let a = Matrix::from_rows(&[[2, -1, 0], [-1, 2, -1], [0, -1, 2]]).map(|&x| Number::from(x)); + /// assert_eq!(a.det(), 4); + /// ``` + #[track_caller] + pub fn det(&self) -> Number { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let mut m = self.clone(); + let (rank, swaps) = bareiss(&mut m); + if rank < n { + return Number::default(); + } + if n == 0 { + return Number::from(1u8); + } + let d = m[(n - 1, n - 1)].clone(); + if swaps % 2 == 1 { -d } else { d } + } + + /// The rank, exactly: the number of pivots of Bareiss's fraction-free elimination. + pub fn rank(&self) -> usize { + bareiss(&mut self.clone()).0 + } + + /// The coefficients of the characteristic polynomial `det(x·I - A)`, from `xⁿ` down, + /// exactly, by Berkowitz's method: the polynomial of each leading block from the one before + /// by a Toeplitz matrix of `1`, `-a_kk` and `-R·A_k^j·C`, without a division. About `n⁴/2` + /// products. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// use a::math::numbers::Number; + /// + /// let a = Matrix::from_rows(&[[2, 1], [1, 2]]).map(|&x| Number::from(x)); + /// assert_eq!(a.characteristic_polynomial(), [1, -4, 3].map(Number::from)); + /// ``` + #[track_caller] + pub fn characteristic_polynomial(&self) -> Vec { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let mut p = vec![Number::from(1u8)]; + for r in 0..n { + // `t = (1, -a_rr, -R·C, -R·A_r·C, …)` for the row `R` and the column `C` beside the + // leading `r × r` block `A_r`. + let mut t = Vec::with_capacity(r + 2); + t.push(Number::from(1u8)); + t.push(-&self[(r, r)]); + let mut v: Vec = (0..r).map(|i| self[(i, r)].clone()).collect(); + for j in 0..r { + let mut q = Number::default(); + for (k, x) in v.iter().enumerate() { + if !x.is_zero() { + q += &self[(r, k)] * x; + } + } + t.push(-q); + if j + 1 < r { + v = (0..r) + .map(|i| { + let mut s = Number::default(); + for (k, x) in v.iter().enumerate() { + if !x.is_zero() { + s += &self[(i, k)] * x; + } + } + s + }) + .collect(); + } + } + // `p ← T·p`, `T` lower triangular Toeplitz with the first column `t`. + let next = (0..r + 2) + .map(|i| { + let mut s = Number::default(); + for (j, c) in p.iter().enumerate().take(i + 1) { + if !c.is_zero() && !t[i - j].is_zero() { + s += &t[i - j] * c; + } + } + s + }) + .collect(); + p = next; + } + p + } + + /// The adjugate `adj(A)`, the transpose of the matrix of cofactors, with + /// `A·adj(A) = adj(A)·A = det(A)·I`, for singular matrices too: by Cayley and Hamilton, + /// `(-1)ⁿ⁻¹·(Aⁿ⁻¹ + c₁·Aⁿ⁻² + … + cₙ₋₁·I)` for the characteristic polynomial's `c`. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn adjugate(&self) -> Matrix { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + if n == 0 { + return Matrix::zeros(0, 0); + } + let c = self.characteristic_polynomial(); + let mut m = Matrix::::identity(n); + for c in &c[1..n] { + m = self * &m; + for i in 0..n { + m[(i, i)] += c; + } + } + if n.is_multiple_of(2) { -m } else { m } + } + + /// The inverse as the adjugate and the determinant, `A⁻¹ = adj(A) / det(A)`, exactly, or + /// `None` when the determinant is zero. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// use a::math::numbers::Number; + /// + /// let a = Matrix::from_rows(&[[2, 1], [5, 3]]).map(|&x| Number::from(x)); + /// let (adj, det) = a.inverse().unwrap(); + /// assert_eq!(det, 1); + /// assert_eq!(adj, Matrix::from_rows(&[[3, -1], [-5, 2]]).map(|&x| Number::from(x))); + /// ``` + #[track_caller] + pub fn inverse(&self) -> Option<(Matrix, Number)> { + let det = self.det(); + if det.is_zero() { + return None; + } + Some((self.adjugate(), det)) + } + + /// A basis of the null space `{x : A·x = 0}` over the rationals, as the columns of an + /// `n × (n - rank)` matrix of integers: one vector for each column without a pivot in the + /// fraction-free echelon form, its free coordinate positive and its elements without a + /// common factor. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// use a::math::numbers::Number; + /// + /// let a = Matrix::from_rows(&[[1, 2, 3], [2, 4, 6]]).map(|&x| Number::from(x)); + /// let null = a.null_space(); + /// assert_eq!(null.shape(), (3, 2)); + /// assert!((&a * &null).as_slice().iter().all(|x| x.is_zero())); + /// ``` + pub fn null_space(&self) -> Matrix { + let n = self.cols(); + let mut e = self.clone(); + let (rank, _) = bareiss(&mut e); + // The pivot columns of the echelon form. + let mut pivots = Vec::with_capacity(rank); + let mut row = 0; + for c in 0..n { + if row < rank && !e[(row, c)].is_zero() { + pivots.push(c); + row += 1; + } + } + let free: Vec = (0..n).filter(|c| !pivots.contains(c)).collect(); + let mut basis = Matrix::::zeros(n, free.len()); + for (b, &f) in free.iter().enumerate() { + // `y`, a multiple of the solution with `x_f = 1` and the other free coordinates 0. + let mut y = vec![Number::default(); n]; + y[f] = Number::from(1u8); + for i in (0..rank).rev() { + let mut s = Number::default(); + for c in i..n { + if c != pivots[i] && !y[c].is_zero() && !e[(i, c)].is_zero() { + s -= &e[(i, c)] * &y[c]; + } + } + let pivot = &e[(i, pivots[i])]; + let g = s.gcd(pivot); + if !g.is_zero() { + let scale = pivot.div_exact(&g).abs(); + if scale != 1 { + for x in &mut y { + *x *= &scale; + } + s *= &scale; + } + } + y[pivots[i]] = s.div_exact(pivot); + } + let g = y.iter().fold(Number::default(), |g, x| g.gcd(x)); + for (i, x) in y.into_iter().enumerate() { + basis[(i, b)] = if g > 1 { x.div_exact(&g) } else { x }; + } + if basis[(f, b)].is_negative() { + for i in 0..n { + basis[(i, b)] = -&basis[(i, b)]; + } + } + } + basis + } +} + +/// Bareiss's fraction-free elimination of `m` to an echelon form in place, the pivot of each +/// row the first non-zero element at or below it in the next column that has one: the rank, +/// and the number of row swaps. +fn bareiss(m: &mut Matrix) -> (usize, usize) { + let (rows, cols) = m.shape(); + let mut previous = Number::from(1u8); + let mut r = 0; + let mut swaps = 0; + for c in 0..cols { + if r == rows { + break; + } + let Some(p) = (r..rows).find(|&i| !m[(i, c)].is_zero()) else { + continue; + }; + if p != r { + m.swap_rows(p, r); + swaps += 1; + } + let pivot = m[(r, c)].clone(); + for i in r + 1..rows { + let below = m[(i, c)].clone(); + for j in c + 1..cols { + let x = &(&m[(i, j)] * &pivot) - &(&below * &m[(r, j)]); + m[(i, j)] = if previous == 1 { + x + } else { + x.div_exact(&previous) + }; + } + m[(i, c)] = Number::default(); + } + previous = pivot; + r += 1; + } + (r, swaps) +} diff --git a/lib/src/math/matrix/functions.rs b/lib/src/math/matrix/functions.rs new file mode 100644 index 0000000..5cd3171 --- /dev/null +++ b/lib/src/math/matrix/functions.rs @@ -0,0 +1,429 @@ +//! Functions of matrices: the exponential, the square root and the logarithm. + +use super::blas::{Op, Side, Uplo, gemm, trsm}; +use super::eigen::real_to_complex_schur; +use super::element::sealed::{Element as _, Scalar as _}; +use super::schur::schur; +use super::{Matrix, Norm, Scalar}; +use crate::math::complex::Float; + +/// The coefficients of the Padé approximants of `e^x` of degree 3, 5, 7, 9 and 13, from the +/// constant term up, and the largest 1-norm each is accurate for in double precision (Higham, +/// 2005). +const PADE: [(&[f64], f64); 5] = [ + (&[120.0, 60.0, 12.0, 1.0], 1.495_585_217_958_292e-2), + ( + &[30240.0, 15120.0, 3360.0, 420.0, 30.0, 1.0], + 2.539_398_330_063_23e-1, + ), + ( + &[ + 17_297_280.0, + 8_648_640.0, + 1_995_840.0, + 277_200.0, + 25200.0, + 1512.0, + 56.0, + 1.0, + ], + 9.504_178_996_162_932e-1, + ), + ( + &[ + 17_643_225_600.0, + 8_821_612_800.0, + 2_075_673_600.0, + 302_702_400.0, + 30_270_240.0, + 2_162_160.0, + 110_880.0, + 3960.0, + 90.0, + 1.0, + ], + 2.097_847_961_257_068, + ), + ( + &[ + 64_764_752_532_480_000.0, + 32_382_376_266_240_000.0, + 7_771_770_303_897_600.0, + 1_187_353_796_428_800.0, + 129_060_195_264_000.0, + 10_559_470_521_600.0, + 670_442_572_800.0, + 33_522_128_640.0, + 1_323_241_920.0, + 40_840_800.0, + 960_960.0, + 16380.0, + 182.0, + 1.0, + ], + 5.371_920_351_148_152, + ), +]; + +/// In double precision: the degrees 3, 5, 7 and 9, by their places in [`PADE`], and their bounds, +/// below 13's. +const PADE_DOUBLE: [(usize, f64); 4] = [ + (0, PADE[0].1), + (1, PADE[1].1), + (2, PADE[2].1), + (3, PADE[3].1), +]; + +/// In single precision: the degrees 3, 5 and 7 and their bounds. +const PADE_SINGLE: [(usize, f64); 3] = [ + (0, 4.258_730_016_922_831e-1), + (1, 1.880_152_677_804_762), + (2, 3.925_724_783_138_66), +]; + +/// The nodes and weights of Gauss and Legendre's rule of 7 points on `[0, 1]`. +const GAUSS: [(f64, f64); 7] = [ + (0.025_446_043_828_620_74, 0.064_742_483_084_434_85), + (0.129_234_407_200_302_78, 0.139_852_695_744_638_33), + (0.297_077_424_311_301_4, 0.190_915_025_252_559_47), + (0.5, 0.208_979_591_836_734_7), + (0.702_922_575_688_698_6, 0.190_915_025_252_559_47), + (0.870_765_592_799_697_2, 0.139_852_695_744_638_33), + (0.974_553_956_171_379_3, 0.064_742_483_084_434_85), +]; + +/// How close to `I` the repeated square roots take `T` before the logarithm's Padé +/// approximant of degree 7, in the 1-norm (Higham, 2001). +const LOG_THETA: f64 = 0.264; + +impl Matrix { + /// The exponential `e^A = I + A + A²/2! + …`, by scaling and squaring: `A / 2^s` small + /// enough for a Padé approximant of degree 3 to 13, chosen by its 1-norm (7 in single + /// precision), then squared `s` times, as Higham's `expm` (2005). + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// + /// // The generator of rotations: e^A rotates by θ. + /// let t = 0.5f64; + /// let a = Matrix::from_rows(&[[0.0, -t], [t, 0.0]]); + /// let r = Matrix::from_rows(&[[t.cos(), -t.sin()], [t.sin(), t.cos()]]); + /// assert!((&a.exp() - &r).as_slice().iter().all(|x| x.abs() < 1e-15)); + /// ``` + #[track_caller] + pub fn exp(&self) -> Self { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + if self.data.iter().any(|x| !x.is_finite()) { + let nan = T::from_parts(T::Real::NAN, T::Real::NAN); + return Matrix::from_fn(n, n, |_, _| nan); + } + let norm = self.norm(Norm::One).to_f64(); + let single = T::Real::MANTISSA_DIGITS <= 24; + let (table, top): (&[(usize, f64)], usize) = if single { + (&PADE_SINGLE, 2) + } else { + (&PADE_DOUBLE, 4) + }; + for &(d, theta) in table { + if norm <= theta { + return pade(self, PADE[d].0); + } + } + let theta = if single { PADE_SINGLE[2].1 } else { PADE[4].1 }; + // A norm that overflows the elements' type, of finite elements, is at most `n` times the + // largest part times √2: its logarithm from that keeps the squarings few, where `∞` would + // make them `i32::MAX`. + let log2 = if norm.is_finite() { + (norm / theta).log2() + } else { + let big = self.data.iter().fold(0.0f64, |s, x| { + s.max(x.re().abs().to_f64().max(x.im().abs().to_f64())) + }); + (n as f64).log2() + big.log2() + 0.5 - theta.log2() + }; + let s = log2.ceil().max(0.0) as i32; + let scaled = self.map(|&x| x.scale(T::Real::from_f64(0.5f64.powi(s)))); + let mut x = pade(&scaled, PADE[top].0); + for _ in 0..s { + x = &x * &x; + } + x + } + + /// The principal square root `√A`, whose eigenvalues have positive real parts, by the + /// Schur method: `R² = T` solved for an upper triangular `R` element by element, then + /// `Q·R·Q*`. `None` for a real matrix with a negative real eigenvalue, whose square roots + /// are not real, or for a singular matrix that has none. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// + /// let a: Matrix = Matrix::from_rows(&[[33.0, 24.0], [48.0, 57.0]]); + /// let r = a.sqrt().unwrap(); + /// assert!((&r - &Matrix::from_rows(&[[5.0, 2.0], [4.0, 7.0]])).as_slice().iter().all(|x| x.abs() < 1e-13)); + /// assert!(Matrix::from_rows(&[[-1.0, 0.0], [0.0, 1.0]]).sqrt().is_none()); + /// ``` + #[track_caller] + pub fn sqrt(&self) -> Option { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let (t, q) = self.complex_schur(|w| w.re < T::Real::ZERO)?; + let r = triangular_sqrt(&t, n)?; + Some(self.back_from_schur(&r, &q)) + } + + /// The principal logarithm `log A`, whose eigenvalues have imaginary parts in `(-π, π]`, by + /// inverse scaling and squaring: square roots of the Schur form `T` until it is near `I`, + /// then a Padé approximant of degree 7 in partial fractions, times `2^k` for `k` square + /// roots, as Higham's `logm` (2001) with Al-Mohy and Higham's accurate diagonal. `None` + /// for a singular matrix, and for a real one with an eigenvalue on the negative real axis, + /// whose logarithms are not real. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// + /// let a: Matrix = Matrix::from_rows(&[[1.0, 2.0], [0.5, 3.0]]); + /// let back = a.log().unwrap().exp(); + /// assert!((&back - &a).as_slice().iter().all(|x| x.abs() < 1e-13)); + /// assert!(Matrix::::zeros(2, 2).log().is_none()); + /// ``` + #[track_caller] + pub fn log(&self) -> Option { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let zero = T::Real::ZERO; + let (t, q) = self.complex_schur(|w| w.re <= zero)?; + if (0..n).any(|i| t[i + i * n] == T::C::zero()) { + return None; + } + // Square roots until `‖T - I‖₁ ≤ θ`. + let mut r = t; + let mut k = 0; + loop { + let mut x = r.clone(); + for i in 0..n { + x[i + i * n] -= T::C::one(); + } + let norm = Matrix::from_column_major(n, n, x).norm(Norm::One).to_f64(); + if norm <= LOG_THETA || k >= 64 { + break; + } + r = triangular_sqrt(&r, n)?; + k += 1; + } + // `X = T^(1/2^k) - I`, with an accurate diagonal. + let mut x = r; + for i in 0..n { + x[i + i * n] = root_minus_one(q.1[i], k); + } + // `r₇(X) = Σ w_j·(I + x_j·X)⁻¹·X`. + let mut sum = vec![T::C::zero(); n * n]; + for &(node, weight) in &GAUSS { + let node = T::Real::from_f64(node); + let mut m = x.clone(); + for v in m.iter_mut() { + *v = v.scale(node); + } + for i in 0..n { + m[i + i * n] += T::C::one(); + } + let mut y = x.clone(); + trsm( + Side::Left, + Uplo::Upper, + Op::N, + false, + n, + n, + &m, + n, + &mut y, + n, + ); + let w = T::Real::from_f64(weight); + for (s, y) in sum.iter_mut().zip(&y) { + *s += y.scale(w); + } + } + let two_k = T::Real::from_f64(2f64.powi(k)); + for s in &mut sum { + *s = s.scale(two_k); + } + Some(self.back_from_schur(&sum, &q)) + } + + /// The complex Schur form `T` of the matrix and `Q`, and for later the eigenvalues as the + /// original diagonal of `T`; `None` for a real matrix with a real eigenvalue `excluded` + /// accepts. + #[allow(clippy::type_complexity)] + fn complex_schur( + &self, + excluded: impl Fn(crate::math::complex::Complex) -> bool, + ) -> Option<(Vec, (Vec, Vec))> { + let n = self.rows(); + if self.data.iter().any(|x| !x.is_finite()) { + return None; + } + let (t, q, w) = schur(self, true, true); + if !T::IS_COMPLEX && w.iter().any(|&w| w.im == T::Real::ZERO && excluded(w)) { + return None; + } + let mut tc: Vec = t.data.iter().map(|x| x.to_c()).collect(); + let mut qc: Vec = q + .unwrap_or_default() + .data + .iter() + .map(|x| x.to_c()) + .collect(); + if !T::IS_COMPLEX { + real_to_complex_schur(&mut tc, &mut qc, n); + } + let diagonal = (0..n).map(|i| tc[i + i * n]).collect(); + Some((tc, (qc, diagonal))) + } + + /// `Q·R·Q*` in the matrix's own type: the real part for real elements. + fn back_from_schur(&self, r: &[T::C], q: &(Vec, Vec)) -> Self { + let n = self.rows(); + let q = &q.0; + let mut qr = vec![T::C::zero(); n * n]; + gemm( + n, + n, + n, + T::C::one(), + q, + n, + Op::N, + r, + n, + Op::N, + T::C::zero(), + &mut qr, + n, + ); + let mut x = vec![T::C::zero(); n * n]; + gemm( + n, + n, + n, + T::C::one(), + &qr, + n, + Op::N, + q, + n, + Op::C, + T::C::zero(), + &mut x, + n, + ); + Matrix::from_column_major(n, n, x.into_iter().map(T::from_c).collect()) + } +} + +/// The Padé approximant `(V - U)⁻¹·(V + U)` of `e^A` with the coefficients `b`, `U` its odd part +/// and `V` its even one, by the powers `A²`, `A⁴`, `A⁶`; for degree 13 by Higham's +/// evaluation in six products. +fn pade(a: &Matrix, b: &[f64]) -> Matrix { + let n = a.rows(); + let c = |k: usize| T::from_real(T::Real::from_f64(b[k])); + let identity = Matrix::::identity(n); + let a2 = a * a; + let (u, v) = if b.len() == 14 { + let a4 = &a2 * &a2; + let a6 = &a4 * &a2; + let inner_u = &(&(&a6 * c(13)) + &(&a4 * c(11))) + &(&a2 * c(9)); + let u = &(&(&(&a6 * &inner_u) + &(&a6 * c(7))) + &(&(&a4 * c(5)) + &(&a2 * c(3)))) + + &(&identity * c(1)); + let inner_v = &(&(&a6 * c(12)) + &(&a4 * c(10))) + &(&a2 * c(8)); + let v = &(&(&(&a6 * &inner_v) + &(&a6 * c(6))) + &(&(&a4 * c(4)) + &(&a2 * c(2)))) + + &(&identity * c(0)); + (a * &u, v) + } else { + // `Σ b_k·A^k`, the even powers by `A²` in turn. + let mut odd = &identity * c(1); + let mut even = &identity * c(0); + let mut power = identity.clone(); + for k in 1..b.len() / 2 { + power = &power * &a2; + odd += &power * c(2 * k + 1); + even += &power * c(2 * k); + } + (a * &odd, even) + }; + (&v - &u) + .solve(&(&v + &u)) + .unwrap_or_else(|| Matrix::from_fn(n, n, |_, _| T::from_parts(T::Real::NAN, T::Real::NAN))) +} + +/// The square root of the upper triangular `t` with the principal roots on its diagonal, by +/// `R² = T` column by column, as Björck and Hammarling's method; `None` where `r_ii + r_jj` is +/// zero and the equation for `r_ij` has no solution. +fn triangular_sqrt(t: &[C], n: usize) -> Option> { + let at = |i: usize, j: usize| i + j * n; + let mut r = vec![C::zero(); n * n]; + for j in 0..n { + r[at(j, j)] = t[at(j, j)].sqrt(); + for i in (0..j).rev() { + let mut s = t[at(i, j)]; + for k in i + 1..j { + s -= r[at(i, k)] * r[at(k, j)]; + } + let d = r[at(i, i)] + r[at(j, j)]; + if d == C::zero() { + if s != C::zero() { + return None; + } + continue; + } + r[at(i, j)] = s / d; + } + } + Some(r) +} + +/// `a^(1/2^k) - 1` without the cancellation of the subtraction, as Al-Mohy and Higham's +/// `sqrt_power_1`: `(a - 1) / ((1 + a^(1/2))·(1 + a^(1/4))·…·(1 + a^(1/2^k)))`. +fn root_minus_one(a: C, k: i32) -> C { + if k == 0 { + return a - C::one(); + } + let mut a = a; + let mut k = k; + // With an argument of at least π/2, `a - 1` itself cancels nothing, so one root first. + if a.re() < C::Real::ZERO { + a = a.sqrt(); + k -= 1; + if k == 0 { + return a - C::one(); + } + } + let z0 = a - C::one(); + a = a.sqrt(); + let mut r = C::one() + a; + for _ in 1..k { + a = a.sqrt(); + r *= C::one() + a; + } + z0 / r +} diff --git a/lib/src/math/matrix/gemm.rs b/lib/src/math/matrix/gemm.rs new file mode 100644 index 0000000..8759ca7 --- /dev/null +++ b/lib/src/math/matrix/gemm.rs @@ -0,0 +1,1077 @@ +//! The product of blocks, `c = β·c + α·op(a)·op(b)`, for [`gemm`](super::blas::gemm), of +//! `f32`, `f64` and complex numbers of either. +//! +//! # Small products +//! +//! A product of at most 64 × 64 × 64 whose blocks lie close together in memory ([`is_small`]) is +//! computed from the operands where they are ([`Real::direct`]): a tile of `c`, 8 rows of `f64` +//! or 16 of `f32` by 4 columns (fewer rows by 8), is summed in vector registers over all of `k`, a +//! column of `a` loaded as vectors at each step and multiplied by each element of `op(b)` in turn, +//! read where it is whatever the operation. Complex numbers split their parts as they are loaded +//! (`ld2`) and take four real products an element; an `op(a)` that is transposed is copied to the +//! stack first. Packing would cost more than it saves at these sizes: an `f64` product of +//! 8 × 8 × 8 takes about 85 cycles, faer's 160 and nalgebra's 300. +//! +//! # Products of few rows +//! +//! When `op(a)` has at most [`SKINNY`](Real::SKINNY) rows, 32, a larger real product goes by +//! [`skinny`]: `op(a)` is copied dense, a block of `k` at a time, and the direct tiles read `op(b)` +//! where it is, as packing it, the larger operand, would cost a copy of it for the few panels of +//! `a` that use it. `f64` products of 8 × 1024 by 1024 × 1024 so ran at 54 GFLOP/s against 35 by +//! blocks, of 16 rows at 54 against 44, and of 32 rows at 54 against 51; at 40 and 48 rows the two +//! tied, and `f32` products of 40 rows were faster by blocks. +//! +//! # Larger products +//! +//! Larger products go by blocks, as Goto and van de Geijn laid them out and BLIS does ([`blocked`]). +//! `op(b)` is packed a block of [`KC`](Real::KC) × [`NC`](Real::NC) at a time, in panels of a +//! few columns whose rows follow each other; for each, `op(a)` is packed a block of +//! [`MC`](Real::MC) rows at a time, in panels of [`MR`](Real::MR) = 8 rows whose columns follow each +//! other, each panel when it is first used, so that it is still in the L1 cache for the +//! micro-kernel. The micro-kernel ([`Real::kernel`]) computes an `8 × nr` tile of `c` from a panel +//! of each: at each step, the column of `a` in vectors, the row of `b` in vectors whose lanes +//! multiply the column by element (`fmla v.2d, v.2d, v.d[i]`), 24 vector sums for the widest +//! panels, 8 × 6 of `f64` and 8 × 12 of `f32`. An Apple M4 core runs four multiply-adds a cycle, +//! each four cycles long, so that 16 independent sums keep it busy, and 24 leave room; the shapes +//! with 24 sums all ran within 2 % of each other (8 × 6, 6 × 8, 4 × 12 and 12 × 4 of `f64`; +//! 16 × 6, 8 × 12, 12 × 8 of `f32`). The rows of `a` come in eights so that blocks of 32 and 64 +//! rows, the panels of the decompositions, need no partial tile. +//! +//! A panel of `op(b)` takes [`Real::width`] columns: 6, 4 or 2 of `f64`, 12, 8 or 4 of `f32`, +//! so that the last panels of a block fit its columns without padding where they can: 32 columns +//! of `f64` are 6 · 4 + 4 + 4, not 6 · 6 with 4 columns of zeros. The narrower kernels have 16 or +//! 8 sums, which take their step in as many cycles as the widest takes its own, so they cost +//! more a column. The last rows of `c`, fewer than 8, and padded panels are computed into a tile +//! on the stack and added to `c` from there. +//! +//! The blocks split each dimension evenly. [`KC`](Real::KC) = 512 steps (24 KiB of a panel of +//! `b` and 32 KiB of `a` in `f64`) was 1 to 3 % faster than 256 and as fast as 768 or 1024; +//! [`MC`](Real::MC) from 64 to 512 and [`NC`](Real::NC) from 1020 to 3060 made no difference +//! beyond the noise, so they are left at sizes that fit the L2 cache with room: 128 rows, and +//! blocks of `b` of 4 MiB. +//! +//! When `op(a)` is `a` itself and a block of `op(b)` has at most [`IN_PLACE`] panels, the kernel +//! reads `a`'s columns where they are rather than from packed panels, as each panel of `a` would +//! serve too few tiles to repay its copy: `f32` products of 1024 × 1024 by 1024 × 32 ran at 119 +//! GFLOP/s so against 96 with `a` packed, and of `f64` by 1024 × 8 at 52 against 38. +//! +//! Complex numbers go through the real kernels, in Van Zee's 1m method: element `x + y·i` of a +//! panel of `op(a)` takes two rows and two columns of a real panel, `[x, y]` and `[-y, x]` (the +//! column of the panel holding the real and imaginary parts of a column of `a` in turn), and an +//! element of `op(b)` two rows, its real and imaginary parts; the real product of the two panels is +//! then the complex product, its parts in turn as `c` holds them. A complex multiply-add so takes +//! four real ones, the least it can, at the speed of the real kernel, and the conjugations and +//! `alpha` are applied as the panels are packed. A `beta` off the real line scales `c` first. +//! +//! # Speed +//! +//! Measured on an Apple M4 Max, one core, in GFLOP/s (`2mnk`, `8mnk` for complex numbers), the +//! median of 5 to 11 runs, against faer 0.24 (`Par::Seq`) and nalgebra 0.35 (`DMatrix::gemm`): +//! +//! | product | `f64` | faer | nalgebra | `f32` | faer | nalgebra | +//! | --- | --- | --- | --- | --- | --- | --- | +//! | 8 × 8 · 8 × 8 | 44.9 | 23.9 | 13.3 | 63.5 | 36.3 | 20.6 | +//! | 64 × 64 · 64 × 64 | 55.6 | 48.4 | 46.3 | 112.9 | 96.5 | 100.4 | +//! | 1024 × 1024 · 1024 × 1024 | 59.4 | 50.6 | 57.3 | 118.7 | 106.5 | 108.3 | +//! | 1024 × 32 · 32 × 1024 | 59.0 | 55.2 | 49.8 | 116.9 | 109.1 | 112.4 | +//! | 32 × 1024 · 1024 × 1024 | 53.1 | 47.7 | 50.7 | 110.0 | 110.6 | 91.4 | +//! +//! | product | `Complex` | faer | nalgebra | `Complex` | faer | nalgebra | +//! | --- | --- | --- | --- | --- | --- | --- | +//! | 8 × 8 · 8 × 8 | 40.5 | 41.8 | 9.8 | 80.6 | 63.7 | 15.1 | +//! | 64 × 64 · 64 × 64 | 53.3 | 43.4 | 10.5 | 107.1 | 82.2 | 21.1 | +//! | 1024 × 1024 · 1024 × 1024 | 59.1 | 45.0 | 10.5 | 119.0 | 88.4 | 20.9 | +//! | 1024 × 32 · 32 × 1024 | 60.2 | 48.5 | 10.7 | 119.5 | 91.4 | 21.4 | +//! +//! The core's four multiply-adds a cycle at 4.5 GHz make 72 GFLOP/s of `f64`, but under a load +//! of them alone its clock falls: a loop of 24 independent multiply-adds held 56 to 60 GFLOP/s +//! over 30 to 300 ms, which the larger products match. Apple's Accelerate, on the matrix unit +//! (SME) rather than NEON, ran 3 to 15 times faster from 32 × 32 · 32 × 32 up. + +use super::blas::Op; +use crate::math::complex::{Complex, Float}; +use core::mem::MaybeUninit; +use core::ptr; + +// The micro-kernels, one set per target under the name `kernels`: NEON on little-endian aarch64 +// (the transposes reinterpret vectors of `f32` as `f64`), and portable Rust elsewhere and under +// Miri. +#[cfg(all(target_arch = "aarch64", target_endian = "little", not(miri)))] +#[path = "gemm/aarch64.rs"] +mod kernels; +#[cfg(not(all(target_arch = "aarch64", target_endian = "little", not(miri))))] +#[path = "gemm/portable.rs"] +mod kernels; + +/// `f32` or `f64`, with the micro-kernel of its products and the blocks it works on. +pub(super) trait Real: Float { + /// The rows of the tile of the micro-kernel, a multiple of its vectors' lanes and even. + const MR: usize; + /// The most columns of a tile of the micro-kernel. + const NR: usize; + /// The steps of `k` of a packed block, even. + const KC: usize; + /// The rows of a packed block of `op(a)`, a multiple of [`MR`](Self::MR). + const MC: usize; + /// The columns of a packed block of `op(b)`, a multiple of [`NR`](Self::NR). + const NC: usize; + /// The most rows of `op(a)` for which [`skinny`] computes the larger products. + const SKINNY: usize; + + /// The columns of the next panel of `op(b)` when `r` are left: a width of one of the kernels, + /// the widest but where narrower ones cover the rest with fewer steps, as a tile's step takes + /// as long for a narrower kernel whose sums wait on each other. + fn width(r: usize) -> usize; + + /// Sets the `MR × nr` tile at `c`, columns `ldc` apart, to `beta · c + a·b`: the sum over + /// `kc` steps of the products of a column of `MR` in `a`, `sa` apart from step to step, and a + /// row of `nr` packed in `b`. When `beta` is zero, `c` is not read. + /// + /// # Safety + /// + /// `kc >= 1` and `nr` a [`width`](Self::width); the columns of `a` must be valid for reads, + /// `b` for reads of `nr · kc` elements, and the tile at `c` for reads and writes. + #[allow(clippy::too_many_arguments)] + unsafe fn kernel( + nr: usize, + kc: usize, + a: *const Self, + sa: usize, + b: *const Self, + c: *mut Self, + ldc: usize, + beta: Self, + ); + + /// `c + a · b`, with one rounding where the target has fused multiply-adds. + fn fma(c: Self, a: Self, b: Self) -> Self; + + /// Packs a panel of `w` lines of `kc` steps times `scale`, from `h <= w` lines whose steps + /// lie in a row: element `(r, p)` is at `src[p + r · ld]` and goes to `dst[p · w + r]`, and + /// lines `h..w` are zeros. + /// + /// # Safety + /// + /// The elements must be valid for reads, and `dst` for writes of `w · kc` reals. + #[allow(clippy::too_many_arguments)] + unsafe fn pack_t( + dst: *mut Self, + src: *const Self, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: Self, + ); + + /// `c = beta · c + alpha · a · op(b)` without packing, for small products: `a` is `m × k` + /// with the leading dimension `lda`, the element `(p, j)` of `op(b)` is at + /// `b[p · rsb + j · csb]`. + /// + /// # Safety + /// + /// `m`, `n`, `k >= 1`; the blocks must be valid for their reads and `c` for its writes. + #[allow(clippy::too_many_arguments)] + unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: Self, + a: *const Self, + lda: usize, + b: *const Self, + rsb: usize, + csb: usize, + beta: Self, + c: *mut Self, + ldc: usize, + ); + + /// [`Real::direct`] of complex numbers: `c = beta · c + alpha · a · op(b)`, with the element + /// `(p, j)` of `op(b)` at `b[p · rsb + j · csb]`, conjugated when `CJ` is set. + /// + /// # Safety + /// + /// As for [`Real::direct`]. + #[allow(clippy::too_many_arguments)] + unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, + ); +} + +/// The elements of products: a real float, held as one real, or a complex number of one, held as +/// two. +trait Elem: Copy + PartialEq { + /// The real type the micro-kernel computes in. + type R: Real; + /// The reals an element takes: 1 or 2. + const W: usize; + /// 0. + const ZERO: Self; + /// 1. + const ONE: Self; + + /// Packs a panel of `R::MR · W · kc` reals from `h <= R::MR / W` rows and `kc` columns of + /// `op(a)`, zero-padded: element `(i, p)` of `op(a)` is at `a[i + p · lda]` when `op` is + /// [`Op::N`] and at `a[p + i · lda]` otherwise. + /// + /// # Safety + /// + /// The elements must be valid for reads, and `dst` for writing the panel. + unsafe fn pack_a(dst: *mut Self::R, a: *const Self, lda: usize, op: Op, h: usize, kc: usize); + + /// Packs a panel of `nr · W · kc` reals from `kc` rows and `h <= nr` columns of + /// `alpha · op(b)`, zero-padded: element `(p, j)` of `op(b)` is at `b[p + j · ldb]` when + /// `op` is [`Op::N`] and at `b[j + p · ldb]` otherwise. + /// + /// # Safety + /// + /// The elements must be valid for reads, and `dst` for writing the panel. + #[allow(clippy::too_many_arguments)] + unsafe fn pack_b( + dst: *mut Self::R, + b: *const Self, + ldb: usize, + op: Op, + kc: usize, + h: usize, + nr: usize, + alpha: Self, + ); +} + +impl Elem for R { + type R = R; + const W: usize = 1; + const ZERO: Self = R::ZERO; + const ONE: Self = R::ONE; + + #[inline(always)] + unsafe fn pack_a(dst: *mut R, a: *const R, lda: usize, op: Op, h: usize, kc: usize) { + // SAFETY: as the caller guarantees. + unsafe { + if op == Op::N { + pack_n(dst, a, lda, h, kc, R::MR, R::ONE); + } else { + R::pack_t(dst, a, lda, h, kc, R::MR, R::ONE); + } + } + } + + #[inline(always)] + #[allow(clippy::too_many_arguments)] + unsafe fn pack_b( + dst: *mut R, + b: *const R, + ldb: usize, + op: Op, + kc: usize, + h: usize, + nr: usize, + alpha: R, + ) { + // SAFETY: as the caller guarantees. + unsafe { + if op == Op::N { + R::pack_t(dst, b, ldb, h, kc, nr, alpha); + } else { + pack_n(dst, b, ldb, h, kc, nr, alpha); + } + } + } +} + +impl Elem for Complex { + type R = R; + const W: usize = 2; + const ZERO: Self = Complex::new(R::ZERO, R::ZERO); + const ONE: Self = Complex::new(R::ONE, R::ZERO); + + #[inline(always)] + unsafe fn pack_a(dst: *mut R, a: *const Self, lda: usize, op: Op, h: usize, kc: usize) { + let mr = R::MR / 2; + // SAFETY: every read is of an element of the block, and every write inside the panel. + unsafe { + for p in 0..kc { + let row = dst.add(2 * p * R::MR); + for i in 0..h { + let x = match op { + Op::N => *a.add(i + p * lda), + Op::T => *a.add(p + i * lda), + Op::C => (*a.add(p + i * lda)).conj(), + }; + *row.add(2 * i) = x.re; + *row.add(2 * i + 1) = x.im; + *row.add(R::MR + 2 * i) = -x.im; + *row.add(R::MR + 2 * i + 1) = x.re; + } + for i in h..mr { + for q in [2 * i, 2 * i + 1, R::MR + 2 * i, R::MR + 2 * i + 1] { + *row.add(q) = R::ZERO; + } + } + } + } + } + + #[inline(always)] + #[allow(clippy::too_many_arguments)] + unsafe fn pack_b( + dst: *mut R, + b: *const Self, + ldb: usize, + op: Op, + kc: usize, + h: usize, + nr: usize, + alpha: Self, + ) { + // SAFETY: every read is of an element of the block, and every write inside the panel. + unsafe { + for p in 0..kc { + let row = dst.add(2 * p * nr); + for j in 0..h { + let x = match op { + Op::N => *b.add(p + j * ldb), + Op::T => *b.add(j + p * ldb), + Op::C => (*b.add(j + p * ldb)).conj(), + }; + let x = x * alpha; + *row.add(j) = x.re; + *row.add(nr + j) = x.im; + } + for j in h..nr { + *row.add(j) = R::ZERO; + *row.add(nr + j) = R::ZERO; + } + } + } + } +} + +/// Packs a panel of `w` lines of `kc` steps times `scale`, from `h <= w` lines whose elements lie +/// in a column a step: element `(r, p)` is at `src[r + p · ld]` and goes to `dst[p · w + r]`, and +/// lines `h..w` are zeros. +/// +/// # Safety +/// +/// The elements must be valid for reads, and `dst` for writes of `w · kc` reals. +#[inline(always)] +unsafe fn pack_n( + dst: *mut R, + src: *const R, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: R, +) { + // SAFETY: as the caller guarantees. + unsafe { + if h == w && scale == R::ONE { + for p in 0..kc { + ptr::copy_nonoverlapping(src.add(p * ld), dst.add(p * w), w); + } + } else { + for p in 0..kc { + let (s, d) = (src.add(p * ld), dst.add(p * w)); + for r in 0..h { + *d.add(r) = *s.add(r) * scale; + } + for r in h..w { + *d.add(r) = R::ZERO; + } + } + } + } +} + +/// The most panels of `op(b)` in a block for which [`blocked`] reads `a` in place. +const IN_PLACE: usize = 16; + +/// Working memory of [`blocked`] on the stack, in reals: products whose packed blocks fit take no +/// allocation. +const STACK: usize = 4096; + +/// [`STACK`] reals of working memory, aligned to a cache line. +#[repr(C, align(128))] +struct Stack([MaybeUninit; STACK]); + +/// `c = beta · c + alpha · op(a) · op(b)` by blocks: `op(b)` is packed a block of `KC × NC` at a +/// time, `op(a)` a block of `MC × KC` for each, and the micro-kernel computes the tiles of `c` from +/// panels of the two, `c` seen as reals, columns `ldc` reals apart. +/// +/// # Safety +/// +/// `m`, `n`, `k >= 1`; the blocks must be valid for their reads and `c` for its writes, as +/// [`gemm`](super::blas::gemm) checks. +#[inline(never)] +#[allow(clippy::too_many_arguments)] +unsafe fn blocked( + m: usize, + n: usize, + k: usize, + alpha: E, + a: *const E, + lda: usize, + opa: Op, + b: *const E, + ldb: usize, + opb: Op, + beta: E::R, + c: *mut E::R, + ldc: usize, +) { + let w = E::W; + let (mr, nr) = (E::R::MR / w, E::R::NR); + // The blocks split each dimension evenly, so that none is much shorter than the others: a + // short block of `k` costs a whole pass over `c`. + let kcb = k.div_ceil(k.div_ceil(E::R::KC / w)); + let mcb = m.div_ceil(m.div_ceil(E::R::MC / w)).next_multiple_of(mr); + let ncb = n.div_ceil(n.div_ceil(E::R::NC)).next_multiple_of(nr); + let (kc_max, mc_max, nc_max) = (kcb, mcb, ncb + nr); + let a_len = mc_max * kc_max * w * w; + let b_len = nc_max * kc_max * w; + let run = |buf: *mut E::R| { + let (ap, bp) = (buf, buf.wrapping_add(a_len.next_multiple_of(16))); + let (zero, one) = (::ZERO, ::ONE); + let mut tile = [zero; 128]; + // SAFETY: the packed blocks fit the buffer, the blocks of `a` and `b` are read inside + // the matrices, and every tile of `c` written is inside `c`. + unsafe { + let mut jc = 0; + while jc < n { + let nc = ncb.min(n - jc); + let mut pc = 0; + while pc < k { + let kc = kcb.min(k - pc); + let beta = if pc == 0 { beta } else { one }; + let mut jr = 0; + while jr < nc { + let nr = E::R::width(nc - jr); + let (j, h) = (jc + jr, nr.min(nc - jr)); + let bs = if opb == Op::N { + pc + j * ldb + } else { + j + pc * ldb + }; + E::pack_b(bp.add(jr * kc * w), b.add(bs), ldb, opb, kc, h, nr, alpha); + jr += nr; + } + // `op(a)` in place when it is `a` itself, real, and each of its panels would + // serve few tiles: packing it would cost more than it saves. + let in_place = w == 1 && opa == Op::N && nc <= IN_PLACE * nr; + let mut ic = 0; + while ic < m { + let mc = mcb.min(m - ic); + let mut jr = 0; + while jr < nc { + let nr = E::R::width(nc - jr); + let nn = nr.min(nc - jr); + let bt = bp.add(jr * kc * w); + let mut ir = 0; + while ir < mc { + let mm = mr.min(mc - ir); + let i = ic + ir; + let as_ = if opa == Op::N { + i + pc * lda + } else { + pc + i * lda + }; + let (at, sa) = if in_place && mm == mr { + // SAFETY of the cast: `E` is `E::R` when `w` is 1. + (a.add(as_).cast::(), lda) + } else { + let at = ap.add(ir * kc * w * w); + if jr == 0 { + E::pack_a(at, a.add(as_), lda, opa, mm, kc); + } + (at.cast_const(), E::R::MR) + }; + let ct = c.add((ic + ir) * w + (jc + jr) * ldc); + if mm == mr && nn == nr { + E::R::kernel(nr, kc * w, at, sa, bt, ct, ldc, beta); + } else { + let t = tile.as_mut_ptr(); + E::R::kernel(nr, kc * w, at, sa, bt, t, E::R::MR, zero); + for j in 0..nn { + for i in 0..mm * w { + let x = ct.add(i + j * ldc); + let y = *t.add(i + j * E::R::MR); + *x = if beta == zero { + y + } else { + E::R::fma(y, beta, *x) + }; + } + } + } + ir += mr; + } + jr += nr; + } + ic += mc; + } + pc += kc; + } + jc += nc; + } + } + }; + let need = a_len.next_multiple_of(16) + b_len; + if need <= STACK { + let mut s = Stack::([const { MaybeUninit::uninit() }; STACK]); + run(s.0.as_mut_ptr().cast()); + } else { + let mut v: Vec = Vec::with_capacity(need + 32); + let p = v.as_mut_ptr(); + // Start on a cache line. + let skip = p.align_offset(128).min(32); + run(p.wrapping_add(skip)); + } +} + +/// An `MR × NR` tile of [`Real::direct`] from `a` and `op(b)` where they are, portable: arrays of +/// sums the compiler keeps in registers or vectors where it can, `MR` rows of `a` a step. +/// +/// # Safety +/// +/// As for [`Real::direct`], for the tile. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn tile( + k: usize, + a: *const R, + lda: usize, + b: *const R, + rsb: usize, + csb: usize, + c: *mut R, + ldc: usize, + alpha: R, + beta: R, +) { + // SAFETY: as the caller guarantees. + unsafe { + let mut acc = [[R::ZERO; MR]; NR]; + for p in 0..k { + let mut col = [R::ZERO; MR]; + for (i, x) in col.iter_mut().enumerate() { + *x = *a.add(i + p * lda); + } + for (j, sums) in acc.iter_mut().enumerate() { + let y = *b.add(p * rsb + j * csb); + for (s, x) in sums.iter_mut().zip(&col) { + *s = R::fma(*s, *x, y); + } + } + } + for (j, sums) in acc.iter().enumerate() { + for (i, s) in sums.iter().enumerate() { + let x = c.add(i + j * ldc); + *x = if beta == R::ZERO { + alpha * *s + } else { + R::fma(beta * *x, alpha, *s) + }; + } + } + } +} + +/// The tiles of [`tile`] of `MR` rows: four columns at a time, then the rest. +/// +/// # Safety +/// +/// As for [`Real::direct`], for the rows. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub(super) unsafe fn tiles( + n: usize, + k: usize, + a: *const R, + lda: usize, + b: *const R, + rsb: usize, + csb: usize, + c: *mut R, + ldc: usize, + alpha: R, + beta: R, +) { + // SAFETY: as the caller guarantees; each tile is inside the block. + unsafe { + let mut j = 0; + while j + 4 <= n { + tile::( + k, + a, + lda, + b.add(j * csb), + rsb, + csb, + c.add(j * ldc), + ldc, + alpha, + beta, + ); + j += 4; + } + // Past the end when no columns are left, so not dereferenced then. + let (b, c) = (b.wrapping_add(j * csb), c.wrapping_add(j * ldc)); + match n - j { + 3 => tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + 2 => tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + 1 => tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + _ => {} + } + } +} + +/// An `MR × NR` tile of [`Real::direct_complex`], portable: the real and the imaginary parts of the +/// sums apart, four real products a step and element. +/// +/// # Safety +/// +/// As for [`Real::direct`], for the tile. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub(super) unsafe fn complex_tile( + k: usize, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + c: *mut Complex, + ldc: usize, + alpha: Complex, + beta: Complex, +) { + // SAFETY: as the caller guarantees. + unsafe { + let mut re = [[R::ZERO; MR]; NR]; + let mut im = [[R::ZERO; MR]; NR]; + for p in 0..k { + let mut col = [Complex::::ZERO; MR]; + for (i, x) in col.iter_mut().enumerate() { + *x = *a.add(i + p * lda); + } + for j in 0..NR { + let y = *b.add(p * rsb + j * csb); + let yi = if CJ { -y.im } else { y.im }; + for (i, x) in col.iter().enumerate() { + re[j][i] = R::fma(R::fma(re[j][i], x.re, y.re), -x.im, yi); + im[j][i] = R::fma(R::fma(im[j][i], x.re, yi), x.im, y.re); + } + } + } + for j in 0..NR { + for i in 0..MR { + let x = c.add(i + j * ldc); + let s = alpha * Complex::new(re[j][i], im[j][i]); + *x = if beta == Complex::ZERO { + s + } else { + s + beta * *x + }; + } + } + } +} + +/// The tiles of [`complex_tile`] of `MR` rows from row `i0` on, portable: four columns at a time, +/// then the rest. +/// +/// # Safety +/// +/// As for [`Real::direct_complex`]. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +pub(super) unsafe fn complex_tiles( + n: usize, + k: usize, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + c: *mut Complex, + ldc: usize, + alpha: Complex, + beta: Complex, +) { + // SAFETY: as the caller guarantees; each tile is inside the block. + unsafe { + let mut j = 0; + while j + 4 <= n { + let (bj, cj) = (b.add(j * csb), c.add(j * ldc)); + complex_tile::(k, a, lda, bj, rsb, csb, cj, ldc, alpha, beta); + j += 4; + } + // Past the end when no columns are left, so not dereferenced then. + let (b, c) = (b.wrapping_add(j * csb), c.wrapping_add(j * ldc)); + match n - j { + 3 => complex_tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + 2 => complex_tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + 1 => complex_tile::(k, a, lda, b, rsb, csb, c, ldc, alpha, beta), + _ => {} + } + } +} + +/// Whether [`Real::direct`] computes the product rather than [`blocked`]: when it is short enough +/// that packing would cost more than it saves, `op(a)` small enough to be transposed on the stack +/// if it must be, and the columns the direct path reads again and again close enough together not +/// to evict each other from the cache. Columns a large power of two apart fall into the same few +/// sets of the L1 cache: with a leading dimension of 2048, a direct `f64` product of 32 × 32 × 32 +/// ran at 26 GFLOP/s, against 40 by blocks and 56 with dense columns. +#[allow(clippy::too_many_arguments)] +#[inline] +fn is_small( + m: usize, + n: usize, + k: usize, + lda: usize, + opa: Op, + ldb: usize, + opb: Op, + ldc: usize, +) -> bool { + if m.saturating_mul(n).saturating_mul(k) > SMALL_MNK || (opa != Op::N && m * k > SMALL_A) { + return false; + } + let a = if opa == Op::N { + lda * (k - 1) + m + } else { + m * k + }; + let b = if opb == Op::N { + ldb * (n - 1) + k + } else { + ldb * (k - 1) + n + }; + let c = ldc * (n - 1) + m; + k <= 8 || a.saturating_add(b).saturating_add(c) <= SMALL_SPAN +} + +/// The elements of `op(a)` the direct path transposes on the stack. +const SMALL_A: usize = 4096; + +/// The products `m · n · k` of the direct path. +const SMALL_MNK: usize = 64 * 64 * 64; + +/// The elements the three blocks of the direct path may span: three of 64 × 64. +const SMALL_SPAN: usize = 3 * 64 * 64; + +/// Runs `f` on `op(a)` held as `m × k` column-major, with its leading dimension: `a` itself when +/// `op` is [`Op::N`], and otherwise its transpose (conjugated for [`Op::C`]) in a buffer on the +/// stack. +/// +/// # Safety +/// +/// The block of `a` must be valid for reads; `m · k <= SMALL_A` when `op` is not [`Op::N`]. +#[inline(always)] +unsafe fn with_a( + m: usize, + k: usize, + a: *const E, + lda: usize, + op: Op, + f: impl FnOnce(*const E, usize) -> X, +) -> X { + if op == Op::N { + f(a, lda) + } else { + // SAFETY: as the caller guarantees. + unsafe { transposed(m, k, a, lda, op, f) } + } +} + +/// [`with_a`] of a transposed `op(a)`, out of line so that the frames of the others do not hold +/// its buffer. +/// +/// # Safety +/// +/// As for [`with_a`]. +#[inline(never)] +unsafe fn transposed( + m: usize, + k: usize, + a: *const E, + lda: usize, + op: Op, + f: impl FnOnce(*const E, usize) -> X, +) -> X { + let mut t = [const { MaybeUninit::::uninit() }; SMALL_A]; + let d = t.as_mut_ptr().cast::(); + // SAFETY: every read is of an element of the block, and every write inside the buffer, which + // then holds the `m · k` elements of `op(a)`. + unsafe { + for i in 0..m { + let s = a.add(i * lda); + for p in 0..k { + let x = *s.add(p); + *d.add(i + p * m) = if op == Op::C { x.conj() } else { x }; + } + } + } + f(d, m) +} + +/// The complex conjugate, of real and complex elements. +trait Conj { + /// The complex conjugate. + fn conj(self) -> Self; +} + +impl Conj for R { + #[inline(always)] + fn conj(self) -> Self { + self + } +} + +impl Conj for Complex { + #[inline(always)] + fn conj(self) -> Self { + Complex::conj(self) + } +} + +/// `c = beta · c + alpha · op(a) · op(b)` for an `op(a)` of few rows: `op(a)` is copied dense a +/// block of [`KC`](Real::KC) columns at a time, and [`Real::direct`] reads `op(b)`, the larger +/// operand, where it is, once for each group of rows. Packing `op(b)` would cost a copy of it for +/// the few panels of `a` that use it. +/// +/// # Safety +/// +/// `m`, `n`, `k >= 1`; the blocks must be valid for their reads and `c` for its writes. +#[inline(never)] +#[allow(clippy::too_many_arguments)] +unsafe fn skinny( + m: usize, + n: usize, + k: usize, + alpha: R, + a: *const R, + lda: usize, + opa: Op, + b: *const R, + ldb: usize, + opb: Op, + beta: R, + c: *mut R, + ldc: usize, +) { + let kcb = k.div_ceil(k.div_ceil(R::KC)); + let (rsb, csb) = if opb == Op::N { (1, ldb) } else { (ldb, 1) }; + let mut buf: Vec = Vec::with_capacity(m * kcb); + let d = buf.as_mut_ptr(); + // SAFETY: every read is of an element of the blocks, the buffer holds the `m · kc` elements + // copied before the direct product reads them, and `c` is written inside its block. + unsafe { + let mut pc = 0; + while pc < k { + let kc = kcb.min(k - pc); + if opa == Op::N { + for p in 0..kc { + ptr::copy_nonoverlapping(a.add((pc + p) * lda), d.add(p * m), m); + } + } else { + for i in 0..m { + let s = a.add(pc + i * lda); + for p in 0..kc { + *d.add(i + p * m) = *s.add(p); + } + } + } + let beta = if pc == 0 { beta } else { R::ONE }; + R::direct( + m, + n, + kc, + alpha, + d, + m, + b.add(pc * rsb), + rsb, + csb, + beta, + c, + ldc, + ); + pc += kc; + } + } +} + +/// `c = beta · c` on the `m × n` block, without reading `c` when `beta` is zero. +/// +/// # Safety +/// +/// The block must be valid for reads and writes. +unsafe fn scale>( + m: usize, + n: usize, + beta: E, + c: *mut E, + ldc: usize, +) { + if beta == E::ONE { + return; + } + // SAFETY: as the caller guarantees. + unsafe { + for j in 0..n { + let col = c.add(j * ldc); + for i in 0..m { + *col.add(i) = if beta == E::ZERO { + E::ZERO + } else { + *col.add(i) * beta + }; + } + } + } +} + +/// Panics unless the elements of a nonempty `rows × cols` block with the leading dimension `ld` +/// lie in a slice of `len`: an empty block may have any leading dimension. +#[inline] +#[track_caller] +fn check(len: usize, rows: usize, cols: usize, ld: usize) { + if rows > 0 && cols > 0 { + assert!(ld >= rows, "the leading dimension is shorter than a column"); + let end = (cols - 1).checked_mul(ld).and_then(|x| x.checked_add(rows)); + assert!( + end.is_some_and(|e| e <= len), + "the block is outside the slice" + ); + } +} + +/// [`gemm`](super::blas::gemm) of real floats. +#[allow(clippy::too_many_arguments)] +#[track_caller] +pub(super) fn real( + m: usize, + n: usize, + k: usize, + alpha: R, + a: &[R], + lda: usize, + opa: Op, + b: &[R], + ldb: usize, + opb: Op, + beta: R, + c: &mut [R], + ldc: usize, +) { + let (ar, ac) = if opa == Op::N { (m, k) } else { (k, m) }; + let (br, bc) = if opb == Op::N { (k, n) } else { (n, k) }; + check(a.len(), ar, ac, lda); + check(b.len(), br, bc, ldb); + check(c.len(), m, n, ldc); + if m == 0 || n == 0 { + return; + } + let (a, b, c) = (a.as_ptr(), b.as_ptr(), c.as_mut_ptr()); + // SAFETY: the blocks lie in the slices, as checked. + unsafe { + if k == 0 || alpha == R::ZERO { + scale(m, n, beta, c, ldc); + } else if is_small(m, n, k, lda, opa, ldb, opb, ldc) { + let (rsb, csb) = if opb == Op::N { (1, ldb) } else { (ldb, 1) }; + with_a(m, k, a, lda, opa, |a, lda| { + R::direct(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc) + }); + } else if m <= R::SKINNY { + skinny(m, n, k, alpha, a, lda, opa, b, ldb, opb, beta, c, ldc); + } else { + blocked(m, n, k, alpha, a, lda, opa, b, ldb, opb, beta, c, ldc); + } + } +} + +/// [`gemm`](super::blas::gemm) of complex numbers. +#[allow(clippy::too_many_arguments)] +#[track_caller] +pub(super) fn complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: &[Complex], + lda: usize, + opa: Op, + b: &[Complex], + ldb: usize, + opb: Op, + beta: Complex, + c: &mut [Complex], + ldc: usize, +) { + let (ar, ac) = if opa == Op::N { (m, k) } else { (k, m) }; + let (br, bc) = if opb == Op::N { (k, n) } else { (n, k) }; + check(a.len(), ar, ac, lda); + check(b.len(), br, bc, ldb); + check(c.len(), m, n, ldc); + if m == 0 || n == 0 { + return; + } + let (a, b, c) = (a.as_ptr(), b.as_ptr(), c.as_mut_ptr()); + // SAFETY: the blocks lie in the slices, as checked; `Complex` is `repr(C)`, its parts in + // turn, so `c` holds `2 · ldc` reals a column. + unsafe { + if k == 0 || alpha == Complex::ZERO { + scale(m, n, beta, c, ldc); + return; + } + if is_small(m, n, k, lda, opa, ldb, opb, ldc) { + let (rsb, csb) = if opb == Op::N { (1, ldb) } else { (ldb, 1) }; + with_a(m, k, a, lda, opa, |a, lda| { + if opb == Op::C { + R::direct_complex::(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc); + } else { + R::direct_complex::(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc); + } + }); + return; + } + // A `beta` off the real line scales `c` first, which the real kernel cannot. + let beta = if beta.im == R::ZERO { + beta.re + } else { + scale(m, n, beta, c, ldc); + R::ONE + }; + blocked( + m, + n, + k, + alpha, + a, + lda, + opa, + b, + ldb, + opb, + beta, + c.cast(), + 2 * ldc, + ); + } +} diff --git a/lib/src/math/matrix/gemm/aarch64.rs b/lib/src/math/matrix/gemm/aarch64.rs new file mode 100644 index 0000000..5e48bbc --- /dev/null +++ b/lib/src/math/matrix/gemm/aarch64.rs @@ -0,0 +1,1063 @@ +//! The aarch64 kernels of the products: NEON intrinsics, fused multiply-adds by element. +//! +//! The micro-kernels keep their sums in arrays of vectors that the compiler holds in registers: +//! 24 of the 32 for the widest tiles, 4 for the column of `a` and 3 for the row of `b`, with +//! nothing spilled, a step of 8 × 6 `f64` being 4 loads (`ldp`) and 24 `fmla` by element. Measured +//! on an Apple M4 Max, a tile's steps ran at 95 to 100 % of four multiply-adds a cycle with its +//! panels in the L1 cache, unrolled two or four times or not; the kernels are not unrolled. A +//! prefetch of the tile of `c` when the kernel starts made no difference. +//! +//! The direct tiles of small products are written with intrinsics too. The compiler's own +//! vectorization of the portable tiles reached 125 GFLOP/s of `f32` in a first test, but spilled +//! sums and shuffled lanes once inside the crate (76 GFLOP/s at 48 × 48 × 48, against 112 with +//! intrinsics), and left the complex tiles scalar: products of `Complex` of 8 × 8 × 8 at 23 +//! GFLOP/s against 48, and of 64 × 64 × 64 at 27 against 61. + +use super::{Real, complex_tiles, tiles}; +use crate::math::complex::Complex; +use core::arch::aarch64::*; + +/// Fills `$acc` with `a · b` over `$kc` steps: `$a` and `$b` are loaded a step at a time, and each +/// column's lane of `b` multiplies all of `a`'s vectors. +macro_rules! steps { + ($kc:expr, $a:ident, $sa:ident, $b:ident, $acc:ident, $ld:ident, $fma:ident, $lanes:literal, $mv:literal, $nv:literal, [$($l:literal),*]) => { + for _ in 0..$kc { + let av: [_; $mv] = core::array::from_fn(|i| $ld($a.add(i * $lanes))); + let bv: [_; $nv] = core::array::from_fn(|q| $ld($b.add(q * $lanes))); + for q in 0..$nv { + for i in 0..$mv { + $($acc[$lanes * q + $l][i] = $fma::<$l>($acc[$lanes * q + $l][i], av[i], bv[q]);)* + } + } + // Past the block after the last step, where `a` is read in place. + $a = $a.wrapping_add($sa); + $b = $b.wrapping_add($lanes * $nv); + } + }; +} + +/// Writes `$acc` to the tile at `$c`: `c = acc`, `c + acc` or `beta · c + acc`. +macro_rules! store { + ($c:ident, $ldc:ident, $beta:ident, $acc:ident, $ld:ident, $st:ident, $add:ident, $fman:ident, $lanes:literal) => { + if $beta == 0.0 { + for (j, col) in $acc.iter().enumerate() { + for (i, v) in col.iter().enumerate() { + $st($c.add(i * $lanes + j * $ldc), *v); + } + } + } else if $beta == 1.0 { + for (j, col) in $acc.iter().enumerate() { + for (i, v) in col.iter().enumerate() { + let p = $c.add(i * $lanes + j * $ldc); + $st(p, $add($ld(p), *v)); + } + } + } else { + for (j, col) in $acc.iter().enumerate() { + for (i, v) in col.iter().enumerate() { + let p = $c.add(i * $lanes + j * $ldc); + $st(p, $fman(*v, $ld(p), $beta)); + } + } + } + }; +} + +/// The `f64` micro-kernels: tiles of 8 rows, four vectors, by `$nv` pairs of columns, lanes of +/// the vectors of `b`. +macro_rules! kernel_f64 { + ($name:ident, $nv:literal) => { + /// An `f64` micro-kernel: see [`kernel_f64`]. + /// + /// # Safety + /// + /// As for [`Real::kernel`]. + #[inline(never)] + unsafe fn $name( + kc: usize, + mut a: *const f64, + sa: usize, + mut b: *const f64, + c: *mut f64, + ldc: usize, + beta: f64, + ) { + // SAFETY: as the caller guarantees. + unsafe { + let mut acc = [[vdupq_n_f64(0.0); 4]; 2 * $nv]; + steps!( + kc, + a, + sa, + b, + acc, + vld1q_f64, + vfmaq_laneq_f64, + 2, + 4, + $nv, + [0, 1] + ); + store!( + c, + ldc, + beta, + acc, + vld1q_f64, + vst1q_f64, + vaddq_f64, + vfmaq_n_f64, + 2 + ); + } + } + }; +} + +kernel_f64!(kernel_f64_6, 3); +kernel_f64!(kernel_f64_4, 2); +kernel_f64!(kernel_f64_2, 1); + +/// The `f32` micro-kernels: tiles of 8 rows, two vectors, by `$nv` quads of columns, lanes of the +/// vectors of `b`. +macro_rules! kernel_f32 { + ($name:ident, $nv:literal) => { + /// An `f32` micro-kernel: see [`kernel_f32`]. + /// + /// # Safety + /// + /// As for [`Real::kernel`]. + #[inline(never)] + unsafe fn $name( + kc: usize, + mut a: *const f32, + sa: usize, + mut b: *const f32, + c: *mut f32, + ldc: usize, + beta: f32, + ) { + // SAFETY: as the caller guarantees. + unsafe { + let mut acc = [[vdupq_n_f32(0.0); 2]; 4 * $nv]; + steps!( + kc, + a, + sa, + b, + acc, + vld1q_f32, + vfmaq_laneq_f32, + 4, + 2, + $nv, + [0, 1, 2, 3] + ); + store!( + c, + ldc, + beta, + acc, + vld1q_f32, + vst1q_f32, + vaddq_f32, + vfmaq_n_f32, + 4 + ); + } + } + }; +} + +kernel_f32!(kernel_f32_12, 3); +kernel_f32!(kernel_f32_8, 2); +kernel_f32!(kernel_f32_4, 1); + +/// [`Real::pack_t`] of `f64`: two lines by two steps at a time, a 2 × 2 transpose by `zip`. +/// +/// # Safety +/// +/// As for [`Real::pack_t`]. +#[inline(never)] +unsafe fn pack_t_f64( + dst: *mut f64, + src: *const f64, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f64, +) { + // SAFETY: every read is of an element of the lines, and every write inside the panel. + unsafe { + let s = vdupq_n_f64(scale); + let mut p = 0; + while p + 2 <= kc { + let (d0, d1) = (dst.add(p * w), dst.add(p * w + w)); + let mut r = 0; + while r + 2 <= h { + let x = vmulq_f64(vld1q_f64(src.add(r * ld + p)), s); + let y = vmulq_f64(vld1q_f64(src.add(r * ld + ld + p)), s); + vst1q_f64(d0.add(r), vzip1q_f64(x, y)); + vst1q_f64(d1.add(r), vzip2q_f64(x, y)); + r += 2; + } + if r < h { + *d0.add(r) = *src.add(r * ld + p) * scale; + *d1.add(r) = *src.add(r * ld + p + 1) * scale; + r += 1; + } + for r in r..w { + *d0.add(r) = 0.0; + *d1.add(r) = 0.0; + } + p += 2; + } + if p < kc { + let d = dst.add(p * w); + for r in 0..w { + *d.add(r) = if r < h { + *src.add(r * ld + p) * scale + } else { + 0.0 + }; + } + } + } +} + +/// [`Real::pack_t`] of `f32`: four lines by four steps at a time, a 4 × 4 transpose by `trn`. +/// +/// # Safety +/// +/// As for [`Real::pack_t`]. +#[inline(never)] +unsafe fn pack_t_f32( + dst: *mut f32, + src: *const f32, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f32, +) { + // SAFETY: every read is of an element of the lines, and every write inside the panel. + unsafe { + let s = vdupq_n_f32(scale); + let mut p = 0; + while p + 4 <= kc { + let d = dst.add(p * w); + let mut r = 0; + while r + 4 <= h { + let x0 = vmulq_f32(vld1q_f32(src.add(r * ld + p)), s); + let x1 = vmulq_f32(vld1q_f32(src.add((r + 1) * ld + p)), s); + let x2 = vmulq_f32(vld1q_f32(src.add((r + 2) * ld + p)), s); + let x3 = vmulq_f32(vld1q_f32(src.add((r + 3) * ld + p)), s); + let t0 = vreinterpretq_f64_f32(vtrn1q_f32(x0, x1)); + let t1 = vreinterpretq_f64_f32(vtrn2q_f32(x0, x1)); + let t2 = vreinterpretq_f64_f32(vtrn1q_f32(x2, x3)); + let t3 = vreinterpretq_f64_f32(vtrn2q_f32(x2, x3)); + vst1q_f32(d.add(r), vreinterpretq_f32_f64(vtrn1q_f64(t0, t2))); + vst1q_f32(d.add(w + r), vreinterpretq_f32_f64(vtrn1q_f64(t1, t3))); + vst1q_f32(d.add(2 * w + r), vreinterpretq_f32_f64(vtrn2q_f64(t0, t2))); + vst1q_f32(d.add(3 * w + r), vreinterpretq_f32_f64(vtrn2q_f64(t1, t3))); + r += 4; + } + for r in r..w { + for q in 0..4 { + *d.add(q * w + r) = if r < h { + *src.add(r * ld + p + q) * scale + } else { + 0.0 + }; + } + } + p += 4; + } + for p in p..kc { + let d = dst.add(p * w); + for r in 0..w { + *d.add(r) = if r < h { + *src.add(r * ld + p) * scale + } else { + 0.0 + }; + } + } + } +} + +/// A tile of [`Real::direct`] of `$t`, `$lanes` lanes a vector: `MV` vectors of rows by `NR` +/// columns, each element of `op(b)` multiplying the column of `a` by element. +macro_rules! real_tile { + ($name:ident, $t:ident, $v:ident, $lanes:literal, $ld:ident, $st:ident, $dup:ident, $fman:ident, $muln:ident) => { + /// A tile of [`Real::direct`]: see [`real_tile`]. + /// + /// # Safety + /// + /// As for [`Real::direct`], for the tile. + #[inline(always)] + #[allow(clippy::too_many_arguments)] + unsafe fn $name( + k: usize, + a: *const $t, + lda: usize, + b: *const $t, + rsb: usize, + csb: usize, + c: *mut $t, + ldc: usize, + alpha: $t, + beta: $t, + ) { + // SAFETY: as the caller guarantees. + unsafe { + let mut acc = [[$dup(0.0); MV]; NR]; + for p in 0..k { + let x: [$v; MV] = core::array::from_fn(|v| $ld(a.add(v * $lanes + p * lda))); + for (j, sums) in acc.iter_mut().enumerate() { + let y = *b.add(p * rsb + j * csb); + for (s, x) in sums.iter_mut().zip(&x) { + *s = $fman(*s, *x, y); + } + } + } + for (j, sums) in acc.iter().enumerate() { + for (v, s) in sums.iter().enumerate() { + let d = c.add(v * $lanes + j * ldc); + let y = $muln(*s, alpha); + $st( + d, + if beta == 0.0 { + y + } else { + $fman(y, $ld(d), beta) + }, + ); + } + } + } + } + }; +} + +real_tile!( + tile_f64, + f64, + float64x2_t, + 2, + vld1q_f64, + vst1q_f64, + vdupq_n_f64, + vfmaq_n_f64, + vmulq_n_f64 +); +real_tile!( + tile_f32, + f32, + float32x4_t, + 4, + vld1q_f32, + vst1q_f32, + vdupq_n_f32, + vfmaq_n_f32, + vmulq_n_f32 +); + +/// Runs `$tile::` over the columns of a group of rows: `$nr` at a time (8 for groups of +/// one or two vectors, so that a tile has 16 sums, as 8 wait on each other: `f32` products of +/// 24 × 1024 · 1024 × 1024 ran at 88 GFLOP/s with tiles of 8 × 4 for the last 8 rows), then 4, +/// then the rest. +macro_rules! real_columns { + ($tile:ident, $mv:literal, $nr:literal, $n:ident, $k:ident, $a:expr, $lda:ident, $b:ident, $rsb:ident, $csb:ident, $c:expr, $ldc:ident, $alpha:ident, $beta:ident) => {{ + let (a, c) = ($a, $c); + let mut j = 0; + while j + $nr <= $n { + let (b, c) = ($b.add(j * $csb), c.add(j * $ldc)); + $tile::<$mv, $nr>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta); + j += $nr; + } + if $nr > 4 && j + 4 <= $n { + let (b, c) = ($b.add(j * $csb), c.add(j * $ldc)); + $tile::<$mv, 4>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta); + j += 4; + } + // Past the end when no columns are left, so not dereferenced then. + let (b, c) = ($b.wrapping_add(j * $csb), c.wrapping_add(j * $ldc)); + match $n - j { + 3 => $tile::<$mv, 3>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + 2 => $tile::<$mv, 2>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + 1 => $tile::<$mv, 1>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + _ => {} + } + }}; +} + +/// A tile of [`Real::direct_complex`] of `f64`: `MV` pairs of rows by `NR` columns, the real and +/// the imaginary parts apart (`ld2`), each part of `b` a lane for the fused multiply-adds. +/// +/// # Safety +/// +/// As for [`Real::direct_complex`], for the tile. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn complex_tile_f64( + k: usize, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + c: *mut Complex, + ldc: usize, + alpha: Complex, + beta: Complex, +) { + // SAFETY: as the caller guarantees; `Complex` is `repr(C)`, its parts in turn. + unsafe { + let z = vdupq_n_f64(0.0); + let mut re = [[z; MV]; NR]; + let mut im = [[z; MV]; NR]; + let (a, b) = (a.cast::(), b.cast::()); + for p in 0..k { + let x: [float64x2x2_t; MV] = + core::array::from_fn(|v| vld2q_f64(a.add(2 * (2 * v + p * lda)))); + for j in 0..NR { + let y = vld1q_f64(b.add(2 * (p * rsb + j * csb))); + for v in 0..MV { + re[j][v] = vfmaq_laneq_f64::<0>(re[j][v], x[v].0, y); + im[j][v] = vfmaq_laneq_f64::<0>(im[j][v], x[v].1, y); + if CJ { + re[j][v] = vfmaq_laneq_f64::<1>(re[j][v], x[v].1, y); + im[j][v] = vfmsq_laneq_f64::<1>(im[j][v], x[v].0, y); + } else { + re[j][v] = vfmsq_laneq_f64::<1>(re[j][v], x[v].1, y); + im[j][v] = vfmaq_laneq_f64::<1>(im[j][v], x[v].0, y); + } + } + } + } + let c = c.cast::(); + let zero = beta.re == 0.0 && beta.im == 0.0; + for j in 0..NR { + for v in 0..MV { + let d = c.add(2 * (2 * v + j * ldc)); + let (sr, si) = (re[j][v], im[j][v]); + let mut yr = vfmsq_n_f64(vmulq_n_f64(sr, alpha.re), si, alpha.im); + let mut yi = vfmaq_n_f64(vmulq_n_f64(si, alpha.re), sr, alpha.im); + if !zero { + let o = vld2q_f64(d); + yr = vfmsq_n_f64(vfmaq_n_f64(yr, o.0, beta.re), o.1, beta.im); + yi = vfmaq_n_f64(vfmaq_n_f64(yi, o.1, beta.re), o.0, beta.im); + } + vst2q_f64(d, float64x2x2_t(yr, yi)); + } + } + } +} + +/// A tile of [`Real::direct_complex`] of `f32`: `MV` quads of rows by `NR` columns, as +/// [`complex_tile_f64`] does it. +/// +/// # Safety +/// +/// As for [`Real::direct_complex`], for the tile. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn complex_tile_f32( + k: usize, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + c: *mut Complex, + ldc: usize, + alpha: Complex, + beta: Complex, +) { + // SAFETY: as the caller guarantees; `Complex` is `repr(C)`, its parts in turn. + unsafe { + let z = vdupq_n_f32(0.0); + let mut re = [[z; MV]; NR]; + let mut im = [[z; MV]; NR]; + let (a, b) = (a.cast::(), b.cast::()); + for p in 0..k { + let x: [float32x4x2_t; MV] = + core::array::from_fn(|v| vld2q_f32(a.add(2 * (4 * v + p * lda)))); + for j in 0..NR { + let y = vld1_f32(b.add(2 * (p * rsb + j * csb))); + for v in 0..MV { + re[j][v] = vfmaq_lane_f32::<0>(re[j][v], x[v].0, y); + im[j][v] = vfmaq_lane_f32::<0>(im[j][v], x[v].1, y); + if CJ { + re[j][v] = vfmaq_lane_f32::<1>(re[j][v], x[v].1, y); + im[j][v] = vfmsq_lane_f32::<1>(im[j][v], x[v].0, y); + } else { + re[j][v] = vfmsq_lane_f32::<1>(re[j][v], x[v].1, y); + im[j][v] = vfmaq_lane_f32::<1>(im[j][v], x[v].0, y); + } + } + } + } + let c = c.cast::(); + let zero = beta.re == 0.0 && beta.im == 0.0; + for j in 0..NR { + for v in 0..MV { + let d = c.add(2 * (4 * v + j * ldc)); + let (sr, si) = (re[j][v], im[j][v]); + let mut yr = vfmsq_n_f32(vmulq_n_f32(sr, alpha.re), si, alpha.im); + let mut yi = vfmaq_n_f32(vmulq_n_f32(si, alpha.re), sr, alpha.im); + if !zero { + let o = vld2q_f32(d); + yr = vfmsq_n_f32(vfmaq_n_f32(yr, o.0, beta.re), o.1, beta.im); + yi = vfmaq_n_f32(vfmaq_n_f32(yi, o.1, beta.re), o.0, beta.im); + } + vst2q_f32(d, float32x4x2_t(yr, yi)); + } + } + } +} + +/// A tile of [`Real::direct_complex`] of `f32` of two rows by `NR` columns, as +/// [`complex_tile_f32`] does it with vectors of two lanes, for the rows left after the groups of +/// four: products of 6 × 6 × 6 ran at 41 GFLOP/s with the portable tile for the last two rows. +/// +/// # Safety +/// +/// As for [`Real::direct_complex`], for the tile. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn complex_tile_f32_2( + k: usize, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + c: *mut Complex, + ldc: usize, + alpha: Complex, + beta: Complex, +) { + // SAFETY: as the caller guarantees; `Complex` is `repr(C)`, its parts in turn. + unsafe { + let z = vdup_n_f32(0.0); + let mut re = [z; NR]; + let mut im = [z; NR]; + let (a, b) = (a.cast::(), b.cast::()); + for p in 0..k { + let x = vld2_f32(a.add(2 * p * lda)); + for j in 0..NR { + let y = vld1_f32(b.add(2 * (p * rsb + j * csb))); + re[j] = vfma_lane_f32::<0>(re[j], x.0, y); + im[j] = vfma_lane_f32::<0>(im[j], x.1, y); + if CJ { + re[j] = vfma_lane_f32::<1>(re[j], x.1, y); + im[j] = vfms_lane_f32::<1>(im[j], x.0, y); + } else { + re[j] = vfms_lane_f32::<1>(re[j], x.1, y); + im[j] = vfma_lane_f32::<1>(im[j], x.0, y); + } + } + } + let c = c.cast::(); + let zero = beta.re == 0.0 && beta.im == 0.0; + for j in 0..NR { + let d = c.add(2 * j * ldc); + let (sr, si) = (re[j], im[j]); + let mut yr = vfms_n_f32(vmul_n_f32(sr, alpha.re), si, alpha.im); + let mut yi = vfma_n_f32(vmul_n_f32(si, alpha.re), sr, alpha.im); + if !zero { + let o = vld2_f32(d); + yr = vfms_n_f32(vfma_n_f32(yr, o.0, beta.re), o.1, beta.im); + yi = vfma_n_f32(vfma_n_f32(yi, o.1, beta.re), o.0, beta.im); + } + vst2_f32(d, float32x2x2_t(yr, yi)); + } + } +} + +/// Runs `$tile::` over the columns of a group of rows: four at a time, then the rest. +macro_rules! complex_columns { + ($tile:ident, $mv:literal, $cj:ident, $n:ident, $k:ident, $a:expr, $lda:ident, $b:ident, $rsb:ident, $csb:ident, $c:expr, $ldc:ident, $alpha:ident, $beta:ident) => {{ + let (a, c) = ($a, $c); + let mut j = 0; + while j + 4 <= $n { + $tile::<$mv, 4, $cj>( + $k, + a, + $lda, + $b.add(j * $csb), + $rsb, + $csb, + c.add(j * $ldc), + $ldc, + $alpha, + $beta, + ); + j += 4; + } + // Past the end when no columns are left, so not dereferenced then. + let (b, c) = ($b.wrapping_add(j * $csb), c.wrapping_add(j * $ldc)); + match $n - j { + 3 => $tile::<$mv, 3, $cj>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + 2 => $tile::<$mv, 2, $cj>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + 1 => $tile::<$mv, 1, $cj>($k, a, $lda, b, $rsb, $csb, c, $ldc, $alpha, $beta), + _ => {} + } + }}; +} + +impl Real for f64 { + const MR: usize = 8; + const NR: usize = 6; + const KC: usize = 512; + const MC: usize = 128; + const NC: usize = 1020; + const SKINNY: usize = 32; + + #[inline(always)] + fn width(r: usize) -> usize { + if r >= 9 || r == 6 || r == 5 { + 6 + } else if r >= 3 { + 4 + } else { + 2 + } + } + + #[inline(always)] + unsafe fn kernel( + nr: usize, + kc: usize, + a: *const f64, + sa: usize, + b: *const f64, + c: *mut f64, + ldc: usize, + beta: f64, + ) { + // SAFETY: as the caller guarantees. + unsafe { + match nr { + 6 => kernel_f64_6(kc, a, sa, b, c, ldc, beta), + 4 => kernel_f64_4(kc, a, sa, b, c, ldc, beta), + _ => kernel_f64_2(kc, a, sa, b, c, ldc, beta), + } + } + } + + #[inline(always)] + fn fma(c: f64, a: f64, b: f64) -> f64 { + a.mul_add(b, c) + } + + #[inline(always)] + unsafe fn pack_t( + dst: *mut f64, + src: *const f64, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f64, + ) { + // SAFETY: as the caller guarantees. + unsafe { pack_t_f64(dst, src, ld, h, kc, w, scale) } + } + + #[inline(never)] + unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: f64, + a: *const f64, + lda: usize, + b: *const f64, + rsb: usize, + csb: usize, + beta: f64, + c: *mut f64, + ldc: usize, + ) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 8 <= m { + real_columns!( + tile_f64, + 4, + 4, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 8; + } + if i + 4 <= m { + real_columns!( + tile_f64, + 2, + 8, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 4; + } + if i + 2 <= m { + real_columns!( + tile_f64, + 1, + 8, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 2; + } + if i < m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + } + } + } + + #[inline(never)] + unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, + ) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 4 <= m { + complex_columns!( + complex_tile_f64, + 2, + CJ, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 4; + } + if i + 2 <= m { + complex_columns!( + complex_tile_f64, + 1, + CJ, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 2; + } + if i < m { + complex_tiles::( + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta, + ); + } + } + } +} + +impl Real for f32 { + const MR: usize = 8; + const NR: usize = 12; + const KC: usize = 512; + const MC: usize = 128; + const NC: usize = 2040; + const SKINNY: usize = 32; + + #[inline(always)] + fn width(r: usize) -> usize { + if r >= 17 || (9..=12).contains(&r) { + 12 + } else if r >= 5 { + 8 + } else { + 4 + } + } + + #[inline(always)] + unsafe fn kernel( + nr: usize, + kc: usize, + a: *const f32, + sa: usize, + b: *const f32, + c: *mut f32, + ldc: usize, + beta: f32, + ) { + // SAFETY: as the caller guarantees. + unsafe { + match nr { + 12 => kernel_f32_12(kc, a, sa, b, c, ldc, beta), + 8 => kernel_f32_8(kc, a, sa, b, c, ldc, beta), + _ => kernel_f32_4(kc, a, sa, b, c, ldc, beta), + } + } + } + + #[inline(always)] + fn fma(c: f32, a: f32, b: f32) -> f32 { + a.mul_add(b, c) + } + + #[inline(always)] + unsafe fn pack_t( + dst: *mut f32, + src: *const f32, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f32, + ) { + // SAFETY: as the caller guarantees. + unsafe { pack_t_f32(dst, src, ld, h, kc, w, scale) } + } + + #[inline(never)] + unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: f32, + a: *const f32, + lda: usize, + b: *const f32, + rsb: usize, + csb: usize, + beta: f32, + c: *mut f32, + ldc: usize, + ) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 16 <= m { + real_columns!( + tile_f32, + 4, + 4, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 16; + } + if i + 8 <= m { + real_columns!( + tile_f32, + 2, + 8, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 8; + } + if i + 4 <= m { + real_columns!( + tile_f32, + 1, + 8, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 4; + } + if i + 2 <= m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + i += 2; + } + if i < m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + } + } + } + + #[inline(never)] + unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, + ) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 8 <= m { + complex_columns!( + complex_tile_f32, + 2, + CJ, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 8; + } + if i + 4 <= m { + complex_columns!( + complex_tile_f32, + 1, + CJ, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 4; + } + if i + 2 <= m { + complex_columns!( + complex_tile_f32_2, + 1, + CJ, + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta + ); + i += 2; + } + if i < m { + complex_tiles::( + n, + k, + a.add(i), + lda, + b, + rsb, + csb, + c.add(i), + ldc, + alpha, + beta, + ); + } + } + } +} diff --git a/lib/src/math/matrix/gemm/portable.rs b/lib/src/math/matrix/gemm/portable.rs new file mode 100644 index 0000000..499f818 --- /dev/null +++ b/lib/src/math/matrix/gemm/portable.rs @@ -0,0 +1,346 @@ +//! The portable kernels of the products, of the shapes of the aarch64 ones, so that Miri runs the +//! blocks and the panels the aarch64 kernels see: arrays of sums, which the compiler keeps in +//! registers or vectors where it can. + +use super::{Real, complex_tiles, tiles}; +use crate::math::complex::Complex; + +/// `c = beta · c + a · b` on an `MR × NR` tile, an array of sums the compiler keeps in +/// registers or vectors where it can. +/// +/// # Safety +/// +/// As for [`Real::kernel`]. +#[inline(always)] +unsafe fn kernel( + kc: usize, + a: *const R, + sa: usize, + b: *const R, + c: *mut R, + ldc: usize, + beta: R, +) { + // SAFETY: as the caller guarantees. + unsafe { + let mut acc = [[R::ZERO; MR]; NR]; + for p in 0..kc { + let (a, b) = (a.add(p * sa), b.add(p * NR)); + for (j, col) in acc.iter_mut().enumerate() { + let x = *b.add(j); + for (i, s) in col.iter_mut().enumerate() { + *s = R::fma(*s, *a.add(i), x); + } + } + } + for (j, col) in acc.iter().enumerate() { + for (i, s) in col.iter().enumerate() { + let p = c.add(i + j * ldc); + *p = if beta == R::ZERO { + *s + } else { + R::fma(*s, beta, *p) + }; + } + } + } +} + +/// [`Real::pack_t`], a line at a time. +/// +/// # Safety +/// +/// As for [`Real::pack_t`]. +#[inline(always)] +unsafe fn pack_t( + dst: *mut R, + src: *const R, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: R, +) { + // SAFETY: as the caller guarantees. + unsafe { + for r in 0..h { + let s = src.add(r * ld); + for p in 0..kc { + *dst.add(p * w + r) = *s.add(p) * scale; + } + } + for r in h..w { + for p in 0..kc { + *dst.add(p * w + r) = R::ZERO; + } + } + } +} + +/// [`Real::direct`]: four rows at a time, then in halves. +/// +/// # Safety +/// +/// As for [`Real::direct`]. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: R, + a: *const R, + lda: usize, + b: *const R, + rsb: usize, + csb: usize, + beta: R, + c: *mut R, + ldc: usize, +) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 4 <= m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + i += 4; + } + if i + 2 <= m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + i += 2; + } + if i < m { + tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + } + } +} + +/// [`Real::direct_complex`]: two rows at a time, then the last. +/// +/// # Safety +/// +/// As for [`Real::direct_complex`]. +#[inline(always)] +#[allow(clippy::too_many_arguments)] +unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, +) { + // SAFETY: as the caller guarantees; each group of rows is inside the block. + unsafe { + let mut i = 0; + while i + 2 <= m { + complex_tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + i += 2; + } + if i < m { + complex_tiles::(n, k, a.add(i), lda, b, rsb, csb, c.add(i), ldc, alpha, beta); + } + } +} + +impl Real for f64 { + const MR: usize = 8; + const NR: usize = 6; + const KC: usize = 512; + const MC: usize = 128; + const NC: usize = 1020; + const SKINNY: usize = 32; + + #[inline(always)] + fn width(r: usize) -> usize { + if r >= 9 || r == 6 || r == 5 { + 6 + } else if r >= 3 { + 4 + } else { + 2 + } + } + + #[inline(never)] + unsafe fn kernel( + nr: usize, + kc: usize, + a: *const f64, + sa: usize, + b: *const f64, + c: *mut f64, + ldc: usize, + beta: f64, + ) { + // SAFETY: as the caller guarantees. + unsafe { + match nr { + 6 => kernel::(kc, a, sa, b, c, ldc, beta), + 4 => kernel::(kc, a, sa, b, c, ldc, beta), + _ => kernel::(kc, a, sa, b, c, ldc, beta), + } + } + } + + #[inline(always)] + fn fma(c: f64, a: f64, b: f64) -> f64 { + c + a * b + } + + #[inline(never)] + unsafe fn pack_t( + dst: *mut f64, + src: *const f64, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f64, + ) { + // SAFETY: as the caller guarantees. + unsafe { pack_t(dst, src, ld, h, kc, w, scale) } + } + + #[inline(never)] + unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: f64, + a: *const f64, + lda: usize, + b: *const f64, + rsb: usize, + csb: usize, + beta: f64, + c: *mut f64, + ldc: usize, + ) { + // SAFETY: as the caller guarantees. + unsafe { direct(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc) } + } + + #[inline(never)] + unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, + ) { + // SAFETY: as the caller guarantees. + unsafe { direct_complex::(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc) } + } +} + +impl Real for f32 { + const MR: usize = 8; + const NR: usize = 12; + const KC: usize = 512; + const MC: usize = 128; + const NC: usize = 2040; + const SKINNY: usize = 32; + + #[inline(always)] + fn width(r: usize) -> usize { + if r >= 17 || (9..=12).contains(&r) { + 12 + } else if r >= 5 { + 8 + } else { + 4 + } + } + + #[inline(never)] + unsafe fn kernel( + nr: usize, + kc: usize, + a: *const f32, + sa: usize, + b: *const f32, + c: *mut f32, + ldc: usize, + beta: f32, + ) { + // SAFETY: as the caller guarantees. + unsafe { + match nr { + 12 => kernel::(kc, a, sa, b, c, ldc, beta), + 8 => kernel::(kc, a, sa, b, c, ldc, beta), + _ => kernel::(kc, a, sa, b, c, ldc, beta), + } + } + } + + #[inline(always)] + fn fma(c: f32, a: f32, b: f32) -> f32 { + c + a * b + } + + #[inline(never)] + unsafe fn pack_t( + dst: *mut f32, + src: *const f32, + ld: usize, + h: usize, + kc: usize, + w: usize, + scale: f32, + ) { + // SAFETY: as the caller guarantees. + unsafe { pack_t(dst, src, ld, h, kc, w, scale) } + } + + #[inline(never)] + unsafe fn direct( + m: usize, + n: usize, + k: usize, + alpha: f32, + a: *const f32, + lda: usize, + b: *const f32, + rsb: usize, + csb: usize, + beta: f32, + c: *mut f32, + ldc: usize, + ) { + // SAFETY: as the caller guarantees. + unsafe { direct(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc) } + } + + #[inline(never)] + unsafe fn direct_complex( + m: usize, + n: usize, + k: usize, + alpha: Complex, + a: *const Complex, + lda: usize, + b: *const Complex, + rsb: usize, + csb: usize, + beta: Complex, + c: *mut Complex, + ldc: usize, + ) { + // SAFETY: as the caller guarantees. + unsafe { direct_complex::(m, n, k, alpha, a, lda, b, rsb, csb, beta, c, ldc) } + } +} diff --git a/lib/src/math/matrix/hermitian.rs b/lib/src/math/matrix/hermitian.rs new file mode 100644 index 0000000..7dbd2d9 --- /dev/null +++ b/lib/src/math/matrix/hermitian.rs @@ -0,0 +1,594 @@ +//! The Hermitian eigenproblem: a blocked reduction to a real symmetric tridiagonal matrix, its +//! eigenvalues and eigenvectors by divide and conquer, and the reflections applied back to them +//! in blocks; small matrices without the blocks, in [`small`]. + +use super::blas::{Op, gemm}; +use super::element::sealed::Scalar as _; +use super::householder::{Block, reflection, reflector}; +use super::tridiagonal; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; + +pub(super) mod small; + +/// The width of the panels of the tridiagonal reduction. +const NB: usize = 32; + +/// Trailing blocks up to this order are reduced without panels: the panels' products of blocks +/// overtake the rank-two updates only past it, as measured, sooner for single precision complex +/// elements. +fn unblocked_largest() -> usize { + if T::IS_COMPLEX && size_of::() == 4 { + 128 + } else { + 192 + } +} + +/// The width of the blocks of reflections applied back to the eigenvectors. +const BACK: usize = 32; + +/// The eigenvalues of the Hermitian `a`, ascending, and its eigenvectors when `want_v` is set. +pub(crate) fn hermitian( + a: &Matrix, + want_v: bool, +) -> (Vec, Option>) { + let n = a.rows(); + if n <= small::largest::(want_v) { + return small::hermitian(a, want_v); + } + let nan = || { + let nan = T::from_parts(T::Real::NAN, T::Real::NAN); + let v = want_v.then(|| Matrix::from_fn(n, n, |_, _| nan)); + (vec![T::Real::NAN; n], v) + }; + if a.data.iter().any(|x| !x.is_finite()) { + return nan(); + } + // The lower triangle, scaled into a range where the reduction neither overflows nor + // underflows, as LAPACK's `heevd`. + let mut h = vec![T::zero(); n * n]; + let (mut norm, mut big) = (T::Real::ZERO, T::Real::ZERO); + for j in 0..n { + for i in j..n { + let x = a.data[i + j * n]; + h[i + j * n] = x; + norm = norm.max(x.abs()); + big = big.max(x.re().abs().max(x.im().abs())); + } + } + // A complex element with finite parts whose absolute value overflows: its largest part + // stands for it, within the factor √2 that the range below has room for. + if !norm.is_finite() { + norm = big; + } + let eps = T::Real::EPSILON * T::Real::HALF; + let rmin = (T::Real::MIN_POSITIVE / eps).sqrt(); + let rmax = (T::Real::ONE / (T::Real::MIN_POSITIVE / eps)).sqrt(); + let scale = if norm > T::Real::ZERO && norm < rmin { + rmin / norm + } else if norm > rmax { + rmax / norm + } else { + T::Real::ONE + }; + if scale != T::Real::ONE { + for j in 0..n { + for x in &mut h[j * n + j..(j + 1) * n] { + *x = x.scale(scale); + } + } + } + let (mut d, mut e, taus) = tridiagonalize(&mut h, n); + let unscale = |d: &mut Vec| { + if scale != T::Real::ONE { + let r = T::Real::ONE / scale; + for x in d.iter_mut() { + *x *= r; + } + } + }; + if !want_v { + if !tridiagonal::values(&mut d, &mut e) { + return nan(); + } + unscale(&mut d); + return (d, None); + } + let Some(z) = tridiagonal::eigen::(&mut d, &mut e) else { + return nan(); + }; + let mut v: Vec = z.iter().map(|x| T::from_real(x.re())).collect(); + back_transform(&h, n, &taus, &mut v); + unscale(&mut d); + (d, Some(Matrix::from_column_major(n, n, v))) +} + +/// Reduces the Hermitian `n × n` matrix `h`, from its lower triangle, to the real symmetric +/// tridiagonal `T = Q*·A·Q`, `Q = H₁·H₂·…·H_{n-1}`, as LAPACK's `hetrd`: returns the diagonal, +/// the subdiagonal and the factors `τ` of the reflections, whose vectors are left below the +/// subdiagonal of `h`. Panels of [`NB`] columns are reduced with the trailing matrix untouched +/// but for the matrix-vector products, which accumulate the update `A -= V·W* + W·V*` as LAPACK's +/// `latrd`; it is then made by products of blocks. +fn tridiagonalize(h: &mut [T], n: usize) -> (Vec, Vec, Vec) { + let mut d = vec![T::Real::ZERO; n]; + let mut e = vec![T::Real::ZERO; n.saturating_sub(1)]; + let mut taus = vec![T::zero(); n.saturating_sub(1)]; + let mut p0 = 0; + let (mut w, mut x, mut y) = (Vec::new(), Vec::new(), Vec::new()); + let unblocked_from = unblocked_largest::(); + while n - p0 > unblocked_from { + let m = n - p0; + w.clear(); + w.resize(m * NB, T::zero()); + panel(h, n, p0, &mut w, &mut d, &mut e, &mut taus); + // `A₂₂ -= V·W* + W·V*` on the lower triangle, as `[V W]·[W V]*`. + let m2 = m - NB; + x.resize(m2 * 2 * NB, T::zero()); + y.resize(m2 * 2 * NB, T::zero()); + for j in 0..NB { + let v = &h[(p0 + j) * n + p0 + NB..(p0 + j + 1) * n]; + let wj = &w[j * m + NB..(j + 1) * m]; + x[j * m2..(j + 1) * m2].copy_from_slice(v); + x[(NB + j) * m2..(NB + j + 1) * m2].copy_from_slice(wj); + y[j * m2..(j + 1) * m2].copy_from_slice(wj); + y[(NB + j) * m2..(NB + j + 1) * m2].copy_from_slice(v); + } + let off = (p0 + NB) * (n + 1); + her2k_lower(m2, 2 * NB, &x, &y, &mut h[off..], n); + p0 += NB; + } + unblocked(h, n, p0, &mut d, &mut e, &mut taus, &mut vec![T::zero(); n]); + (d, e, taus) +} + +/// `c -= x·y*` on the lower triangle of the `m × m` block `c`, `x` and `y` of `m × k` with the +/// leading dimension `m`: block columns, each from the diagonal down, by [`gemm`]. The diagonal +/// blocks are updated whole, above their diagonals too, where nothing reads. +fn her2k_lower(m: usize, k: usize, x: &[T], y: &[T], c: &mut [T], ldc: usize) { + const COLUMNS: usize = 64; + let mut j0 = 0; + while j0 < m { + let w = COLUMNS.min(m - j0); + let (one, minus) = (T::one(), -T::one()); + gemm( + m - j0, + w, + k, + minus, + &x[j0..], + m, + Op::N, + &y[j0..], + m, + Op::C, + one, + &mut c[j0 * (ldc + 1)..], + ldc, + ); + j0 += w; + } +} + +/// Reduces the [`NB`] columns from `p0` of `h`, as LAPACK's `latrd`: the columns are updated by +/// the reflections before them in the panel as they come, and `w` (`m × NB`, `m = n - p0`) gets +/// the `W` of the update `A -= V·W* + W·V*` of the trailing matrix. The reflection vectors are +/// left in `h` with their leading ones. +fn panel( + h: &mut [T], + n: usize, + p0: usize, + w: &mut [T], + d: &mut [T::Real], + e: &mut [T::Real], + taus: &mut [T], +) { + let m = n - p0; + let mut t = [T::zero(); 2 * NB]; + for i in 0..NB { + let ci = p0 + i; + { + // Column `ci` from row `ci`, less what the panel's reflections before it change: + // `a -= V·conj(W[ci, :]) + W·conj(V[ci, :])`. + let (left, right) = h.split_at_mut(ci * n); + let col = &mut right[ci..n]; + let mut cols: Vec<&[T]> = Vec::with_capacity(2 * i); + let mut coef = Vec::with_capacity(2 * i); + for j in 0..i { + let v = &left[(p0 + j) * n + ci..(p0 + j + 1) * n]; + let wj = &w[j * m + i..(j + 1) * m]; + cols.extend([v, wj]); + coef.extend([-wj[0].conj(), -v[0].conj()]); + } + add_columns(col, &cols, &coef); + col[0] = T::from_real(col[0].re()); + d[ci] = col[0].re(); + } + if ci + 1 == n { + break; + } + let (tau, beta) = { + let col = &mut h[ci * n + ci + 1..(ci + 1) * n]; + let (alpha, x) = col.split_first_mut().expect("a subdiagonal"); + let r = reflect(*alpha, x); + *alpha = T::one(); + r + }; + e[ci] = beta.re(); + taus[ci] = tau; + // `W[:, i] = τ·(A·v - V·(W*·v) - W·(V*·v))`, then `-= (τ/2)·(W[:, i]*·v)·v`. + let (wprev, wcur) = w.split_at_mut(i * m); + let wi = &mut wcur[i + 1..m]; + let v = &h[ci * n + ci + 1..(ci + 1) * n]; + hemv(m - i - 1, &h[(ci + 1) * (n + 1)..], n, v, wi); + if i > 0 { + let mut cols: Vec<&[T]> = Vec::with_capacity(2 * i); + for j in 0..i { + cols.push(&wprev[j * m + i + 1..(j + 1) * m]); + cols.push(&h[(p0 + j) * n + ci + 1..(p0 + j + 1) * n]); + } + // `t = [W V]*·v`, then `W[:, i] -= [V W]·t`. + let t = &mut t[..2 * i]; + dot_columns(&cols, v, t); + for pair in cols.as_chunks_mut::<2>().0 { + pair.swap(0, 1); + } + for x in t.iter_mut() { + *x = -*x; + } + add_columns(wi, &cols, t); + } + for x in wi.iter_mut() { + *x *= tau; + } + let alpha = -(tau * dotc(wi, v)).scale(T::Real::HALF); + for (x, v) in wi.iter_mut().zip(v) { + *x += alpha * *v; + } + } +} + +/// Reduces the columns from `p0` on one at a time, as LAPACK's `hetd2`: each reflection is +/// applied to the trailing matrix by the rank-two update `A -= v·w* + w·v*`, `w` of length `n` +/// the room for its `w`. +fn unblocked( + h: &mut [T], + n: usize, + p0: usize, + d: &mut [T::Real], + e: &mut [T::Real], + taus: &mut [T], + w: &mut [T], +) { + for j in p0..n { + let diag = h[j * (n + 1)].re(); + d[j] = diag; + if j + 1 == n { + break; + } + let (tau, beta) = { + let col = &mut h[j * n + j + 1..(j + 1) * n]; + let (alpha, x) = col.split_first_mut().expect("a subdiagonal"); + let r = reflect(*alpha, x); + *alpha = T::one(); + r + }; + e[j] = beta.re(); + taus[j] = tau; + if tau == T::zero() { + continue; + } + let m = n - j - 1; + let (left, right) = h.split_at_mut((j + 1) * n); + let v = &left[j * n + j + 1..]; + let a22 = &mut right[j + 1..]; + let w = &mut w[..m]; + // `w = τ·A·v - (τ/2)·(τ·v*·A·v)·v`. + hemv(m, a22, n, v, w); + for x in w.iter_mut() { + *x *= tau; + } + let alpha = -(tau * dotc(w, v)).scale(T::Real::HALF); + for (x, v) in w.iter_mut().zip(v) { + *x += alpha * *v; + } + for c in 0..m { + let (vc, wc) = (-v[c].conj(), -w[c].conj()); + let col = &mut a22[c * n + c..c * n + m]; + for ((x, &v), &w) in col.iter_mut().zip(&v[c..]).zip(&w[c..]) { + *x = fma(w, vc, fma(v, wc, *x)); + } + } + } +} + +/// [`reflector`] with the norm of `x` from one pass of sums of squares in lanes when that sum is +/// in the range where no square can have overflowed or underflowed to matter, as is usual; from +/// the scaled norm otherwise. +fn reflect(alpha: T, x: &mut [T]) -> (T, T) { + let mut s = [T::Real::ZERO; 4]; + let (quads, rest) = x.as_chunks::<4>(); + for c in quads { + for l in 0..4 { + s[l] += c[l].abs2(); + } + } + for v in rest { + s[0] += v.abs2(); + } + let ssq = (s[0] + s[2]) + (s[1] + s[3]); + let safe = ssq >= T::Real::MIN_POSITIVE / T::Real::EPSILON && ssq <= T::Real::MAX; + if !safe { + return reflector(alpha, x); + } + reflection(alpha, alpha.abs().hypot(ssq.sqrt()), x) +} + +/// Whether the target fuses multiply-adds in hardware; elsewhere `mul_add` is a library call. +const FMA: bool = cfg!(any(target_arch = "aarch64", target_feature = "fma")); + +/// `c + a·b`, rounded once where the target fuses multiply-adds. +#[inline(always)] +pub(super) fn fma(a: T, b: T, c: T) -> T { + if !FMA { + c + a * b + } else if T::IS_COMPLEX { + let (ar, ai, br, bi) = (a.re(), a.im(), b.re(), b.im()); + let re = (-ai).mul_add(bi, ar.mul_add(br, c.re())); + let im = ai.mul_add(br, ar.mul_add(bi, c.im())); + T::from_parts(re, im) + } else { + T::from_real(a.re().mul_add(b.re(), c.re())) + } +} + +/// `c + conj(a)·b`, rounded once where the target fuses multiply-adds. +#[inline(always)] +pub(super) fn fma_conj(a: T, b: T, c: T) -> T { + if !FMA { + c + a.conj() * b + } else if T::IS_COMPLEX { + let (ar, ai, br, bi) = (a.re(), a.im(), b.re(), b.im()); + let re = ai.mul_add(bi, ar.mul_add(br, c.re())); + let im = (-ai).mul_add(br, ar.mul_add(bi, c.im())); + T::from_parts(re, im) + } else { + T::from_real(a.re().mul_add(b.re(), c.re())) + } +} + +/// The sum of the lanes of an accumulator. +#[inline(always)] +pub(super) fn lanes(s: [T; L]) -> T { + let mut t = T::zero(); + for x in s { + t += x; + } + t +} + +/// `x*·y`. +pub(super) fn dotc(x: &[T], y: &[T]) -> T { + let mut t = [T::zero()]; + dot_columns(&[x], y, &mut t); + t[0] +} + +/// `t_j = c_j*·v` for the columns `c_j`, each at least as long as `v`: four at a time, in lanes. +fn dot_columns(cols: &[&[T]], v: &[T], t: &mut [T]) { + if T::IS_COMPLEX { + dot_columns_by::(cols, v, t); + } else { + dot_columns_by::(cols, v, t); + } +} + +/// [`dot_columns`] with `L` lanes. +fn dot_columns_by(cols: &[&[T]], v: &[T], t: &mut [T]) { + let len = v.len(); + let (vb, vt) = v.as_chunks::(); + let body = len - vt.len(); + let (quads, rest) = cols.as_chunks::<4>(); + let (tq, tr) = t.as_chunks_mut::<4>(); + for (cols, t) in quads.iter().zip(tq) { + let c = cols.map(|c| &c[..len]); + let (a0, a1, a2, a3) = ( + lanes_of::(c[0]), + lanes_of::(c[1]), + lanes_of::(c[2]), + lanes_of::(c[3]), + ); + let mut s = [[T::zero(); L]; 4]; + let it = vb.iter().zip(a0).zip(a1).zip(a2.iter().zip(a3)); + for (((x, a0), a1), (a2, a3)) in it { + for l in 0..L { + s[0][l] = fma_conj(a0[l], x[l], s[0][l]); + s[1][l] = fma_conj(a1[l], x[l], s[1][l]); + s[2][l] = fma_conj(a2[l], x[l], s[2][l]); + s[3][l] = fma_conj(a3[l], x[l], s[3][l]); + } + } + for (k, (s, c)) in s.iter_mut().zip(c).enumerate() { + for (&x, &a) in vt.iter().zip(&c[body..]) { + s[0] = fma_conj(a, x, s[0]); + } + t[k] = lanes(*s); + } + } + for (c, t) in rest.iter().zip(tr) { + let c = &c[..len]; + let mut s = [T::zero(); L]; + for (x, a) in vb.iter().zip(lanes_of::(c)) { + for l in 0..L { + s[l] = fma_conj(a[l], x[l], s[l]); + } + } + for (&x, &a) in vt.iter().zip(&c[body..]) { + s[0] = fma_conj(a, x, s[0]); + } + *t = lanes(s); + } +} + +/// The whole chunks of `L` of `x`. +#[inline(always)] +pub(super) fn lanes_of(x: &[T]) -> &[[T; L]] { + x.as_chunks::().0 +} + +/// `y += Σ_j c_j·a_j` for the columns `a_j`, each at least as long as `y`: four at a time. +fn add_columns(y: &mut [T], cols: &[&[T]], c: &[T]) { + let len = y.len(); + let (quads, rest) = cols.as_chunks::<4>(); + let (cq, cr) = c.as_chunks::<4>(); + for (cols, c) in quads.iter().zip(cq) { + let [a0, a1, a2, a3] = cols.map(|c| &c[..len]); + let it = y.iter_mut().zip(a0).zip(a1).zip(a2.iter().zip(a3)); + for (((y, &a0), &a1), (&a2, &a3)) in it { + *y = fma(a3, c[3], fma(a2, c[2], fma(a1, c[1], fma(a0, c[0], *y)))); + } + } + for (a, &c) in rest.iter().zip(cr) { + for (y, &a) in y.iter_mut().zip(&a[..len]) { + *y = fma(a, c, *y); + } + } +} + +/// `y = A·x` for the Hermitian `m × m` block `a` held in its lower triangle, the imaginary parts +/// of its diagonal taken as zeros: each element below the diagonal read once, for both the +/// column and the row it is in, four columns at a time. +fn hemv(m: usize, a: &[T], lda: usize, x: &[T], y: &mut [T]) { + if T::IS_COMPLEX { + hemv_by::(m, a, lda, x, y); + } else { + hemv_by::(m, a, lda, x, y); + } +} + +/// [`hemv`] with `L` rows a step, the rows' sums for the columns in `L` lanes. +fn hemv_by(m: usize, a: &[T], lda: usize, x: &[T], y: &mut [T]) { + let y = &mut y[..m]; + y.fill(T::zero()); + let mut j = 0; + while j + 4 <= m { + // The 4 × 4 block on the diagonal. + let mut t = [T::zero(); 4]; + for k in 0..4 { + let col = &a[(j + k) * lda..]; + t[k] += x[j + k].scale(col[j + k].re()); + for l in k + 1..4 { + let v = col[j + l]; + y[j + l] = fma(v, x[j + k], y[j + l]); + t[k] = fma_conj(v, x[j + l], t[k]); + } + } + // Below it. + let b = j + 4; + let col = |k: usize| &a[(j + k) * lda + b..(j + k) * lda + m]; + let (c0, c1, c2, c3) = (col(0), col(1), col(2), col(3)); + let (x0, x1, x2, x3) = (x[j], x[j + 1], x[j + 2], x[j + 3]); + let (ys, xr) = (&mut y[b..m], &x[b..m]); + let (head, tail) = ys.as_chunks_mut::(); + let body = head.len() * L; + let mut s = [[T::zero(); L]; 4]; + let it = head + .iter_mut() + .zip(lanes_of::(xr)) + .zip(lanes_of::(c0)); + let it = it + .zip(lanes_of::(c1)) + .zip(lanes_of::(c2).iter().zip(lanes_of::(c3))); + for ((((yv, xv), a0), a1), (a2, a3)) in it { + for l in 0..L { + let y = fma(a1[l], x1, fma(a0[l], x0, yv[l])); + yv[l] = fma(a3[l], x3, fma(a2[l], x2, y)); + s[0][l] = fma_conj(a0[l], xv[l], s[0][l]); + s[1][l] = fma_conj(a1[l], xv[l], s[1][l]); + s[2][l] = fma_conj(a2[l], xv[l], s[2][l]); + s[3][l] = fma_conj(a3[l], xv[l], s[3][l]); + } + } + for (i, y) in tail.iter_mut().enumerate() { + let r = body + i; + *y = fma( + c3[r], + x3, + fma(c2[r], x2, fma(c1[r], x1, fma(c0[r], x0, *y))), + ); + s[0][0] = fma_conj(c0[r], xr[r], s[0][0]); + s[1][0] = fma_conj(c1[r], xr[r], s[1][0]); + s[2][0] = fma_conj(c2[r], xr[r], s[2][0]); + s[3][0] = fma_conj(c3[r], xr[r], s[3][0]); + } + for k in 0..4 { + y[j + k] += t[k] + lanes(s[k]); + } + j += 4; + } + while j < m { + let col = &a[j * lda..j * lda + m]; + let xj = x[j]; + let mut t = xj.scale(col[j].re()); + for r in j + 1..m { + y[r] = fma(col[r], xj, y[r]); + t = fma_conj(col[r], x[r], t); + } + y[j] += t; + j += 1; + } +} + +/// `V = Q·V` for the `n × n` eigenvectors `v` of the tridiagonal matrix, `Q` from the reflections +/// [`tridiagonalize`] left in `h`, as LAPACK's `ormtr`: blocks of [`BACK`] of them, from the +/// last, each `I - Y·T·Y*` applied by three products of blocks, `Y*` copied out so that every +/// product takes its blocks as they are. +fn back_transform(h: &[T], n: usize, taus: &[T], v: &mut [T]) { + if n < 2 { + return; + } + let (one, zero) = (T::one(), T::zero()); + let mut yh = Vec::new(); + let mut w = Vec::new(); + let mut tw = Vec::new(); + // The reflections act on the rows from 1 on, held as a QR factorization's are. + let mut end = n - 1; + while end > 0 { + let start = (end - 1) / BACK * BACK; + let block = Block::new( + &h[1 + start * (n + 1)..], + n, + n - 1 - start, + &taus[start..end], + ); + let (m, k) = (block.m, block.k); + yh.clear(); + yh.resize(k * m, zero); + for (j, col) in block.v.chunks_exact(m).enumerate() { + for (i, &y) in col.iter().enumerate() { + yh[j + i * k] = y.conj(); + } + } + w.resize(k * n, zero); + tw.resize(k * n, zero); + let c = &mut v[1 + start..]; + gemm(k, n, m, one, &yh, k, Op::N, c, n, Op::N, zero, &mut w, k); + gemm( + k, + n, + k, + one, + &block.t, + k, + Op::N, + &w, + k, + Op::N, + zero, + &mut tw, + k, + ); + gemm(m, n, k, -one, &block.v, m, Op::N, &tw, k, Op::N, one, c, n); + end = start; + } +} diff --git a/lib/src/math/matrix/hermitian/small.rs b/lib/src/math/matrix/hermitian/small.rs new file mode 100644 index 0000000..4118fef --- /dev/null +++ b/lib/src/math/matrix/hermitian/small.rs @@ -0,0 +1,444 @@ +//! The Hermitian eigenproblem of small matrices, where the blocked path's fixed costs outweigh +//! its arithmetic: the reduction one column at a time, `Q` formed explicitly in the place of its +//! reflections, and the implicit QL algorithm's rotations applied straight to `Q`, as EISPACK's +//! `tred2` and `tql2`; the eigenvalues alone by the QL and QR algorithm without square roots. No +//! allocation but the results': the scratch is on the stack. + +use super::super::tridiagonal; +use super::super::{Matrix, Scalar}; +use super::{fma, fma_conj, lanes, lanes_of, unblocked}; +use crate::math::complex::Float; + +/// The most any type has solved here. +const LARGEST: usize = 160; + +/// The orders whose scratch is the smallest of the sizes of it, then the middle one, so that +/// the smaller problems don't clear room for the largest. +const TIERS: [usize; 2] = [16, 48]; + +/// The largest order of the elements `T` solved here, as measured on an M4: past it, divide and +/// conquer and the reflections applied in blocks take less time. The rotations of the QL +/// algorithm fill the most lanes on real vectors, twice as many on single precision ones, which +/// moves their crossovers furthest. Without the vectors the paths differ in little but their +/// allocations: the large path's reduction is unblocked up to about there too. +pub(super) fn largest(want_v: bool) -> usize { + match (want_v, T::IS_COMPLEX, size_of::()) { + (false, _, _) => 128, + (true, false, 4) => LARGEST, + (true, false, _) => 112, + (true, true, _) => 80, + } +} + +/// [`hermitian`](super::hermitian) of an `a` of order up to [`largest`]. +pub(super) fn hermitian( + a: &Matrix, + want_v: bool, +) -> (Vec, Option>) { + let n = a.rows(); + if n <= TIERS[0] { + solve::(a, want_v) + } else if n <= TIERS[1] { + solve::(a, want_v) + } else { + solve::(a, want_v) + } +} + +/// [`hermitian`] with room for orders up to `N`. +fn solve( + a: &Matrix, + want_v: bool, +) -> (Vec, Option>) { + let n = a.rows(); + debug_assert!(n <= N); + if n == 0 { + return (Vec::new(), want_v.then(Matrix::default)); + } + let nan = T::from_parts(T::Real::NAN, T::Real::NAN); + let Some((mut h, scale)) = lower(a) else { + let v = want_v.then(|| Matrix::from_column_major(n, n, vec![nan; n * n])); + return (vec![T::Real::NAN; n], v); + }; + let zero = T::Real::ZERO; + let mut d = vec![zero; n]; + let (mut e, mut taus, mut w) = ([zero; N], [T::zero(); N], [T::zero(); N]); + let m = n - 1; + unblocked( + &mut h, + n, + 0, + &mut d, + &mut e[..m], + &mut taus[..m], + &mut w[..n], + ); + let ok = if want_v { + form_q(&mut h, n, &taus[..m]); + tridiagonal::ql(&mut d, &e[..m], Some((&mut h, n, n))) + } else { + tridiagonal::values(&mut d, &mut e[..m]) + }; + if !ok { + h.fill(nan); + d.fill(T::Real::NAN); + } else if scale != T::Real::ONE { + let r = T::Real::ONE / scale; + for x in &mut d { + *x *= r; + } + } + (d, want_v.then(|| Matrix::from_column_major(n, n, h))) +} + +/// The lower triangle of `a`, scaled into the range where the reduction neither overflows nor +/// underflows as LAPACK's `heevd`, and the factor; `None` if any element of `a` is not finite. +fn lower(a: &Matrix) -> Option<(Vec, T::Real)> { + let n = a.rows(); + let zero = T::Real::ZERO; + let mut h = vec![T::zero(); n * n]; + // The largest part, within a factor √2 of the largest absolute value, which costs a square + // root of a complex element; and whether any part is not finite, everywhere. + let mut big = zero; + let mut bad = false; + for (j, (dst, col)) in h + .chunks_exact_mut(n) + .zip(a.data.chunks_exact(n)) + .enumerate() + { + for &x in &col[..j] { + bad |= !x.is_finite(); + } + for (y, &x) in dst[j..].iter_mut().zip(&col[j..]) { + *y = x; + big = big.max(x.re().abs().max(x.im().abs())); + bad |= !x.is_finite(); + } + } + if bad { + return None; + } + let eps = T::Real::EPSILON * T::Real::HALF; + let rmin = (T::Real::MIN_POSITIVE / eps).sqrt(); + let rmax = (T::Real::ONE / (T::Real::MIN_POSITIVE / eps)).sqrt(); + if big >= rmin && big * T::Real::TWO.sqrt() <= rmax { + return Some((h, T::Real::ONE)); + } + // Near an end of the range: the largest absolute value itself decides, as `heevd`'s. + let mut norm = zero; + for j in 0..n { + for x in &h[j * n + j..(j + 1) * n] { + norm = norm.max(x.abs()); + } + } + // A complex element with finite parts whose absolute value overflows: its largest part + // stands for it, within the factor √2 that the range has room for. + if !norm.is_finite() { + norm = big; + } + let scale = if norm > zero && norm < rmin { + rmin / norm + } else if norm > rmax { + rmax / norm + } else { + return Some((h, T::Real::ONE)); + }; + for j in 0..n { + for x in &mut h[j * n + j..(j + 1) * n] { + *x = x.scale(scale); + } + } + Some((h, scale)) +} + +/// `Q = H₀·H₁·…·H_{n-2}`, explicitly, over the reflections whose vectors the reduction left +/// below the subdiagonal of the `n × n` `h`, as LAPACK's `ungtr` and `ung2r`: from the last +/// reflections, two at a time, each pair applied to the columns of `Q` it changes, then turned +/// into its own two columns, which are the ones to the right of their vectors', so that nothing +/// need be moved first. `Q` has the first column and row of the identity. +fn form_q(h: &mut [T], n: usize, taus: &[T]) { + if n == 0 { + return; + } + // `H_r` acts on the rows from `r + 1`, its vector after the leading one in column `r` from + // row `r + 2`, and makes column `r + 1` of `Q`. + let mut r = n - 1; + while r >= 2 { + // `H_{r-2}·H_{r-1}`, on the rows from `r - 1`. + let (b, a) = (r - 1, r - 2); + let (left, right) = h.split_at_mut(r * n); + let va = &left[a * n + b + 1..b * n]; + let vb = &left[b * n + r + 1..]; + if r + 1 < n { + let pair = ((taus[a], va), (taus[b], vb)); + reflect_columns2(pair.0, pair.1, &mut right[n + b..], n, n - r - 1); + } + let col = &mut right[..n]; + make_column(col, r, taus[b], vb); + if taus[a] != T::zero() { + reflect_columns(taus[a], va, &mut col[b..], n, 1); + } + let (left, right) = h.split_at_mut(b * n); + make_column(&mut right[..n], b, taus[a], &left[a * n + b + 1..]); + r -= 2; + } + if r == 1 { + // `H₀`, on the rows from 1. + let (left, right) = h.split_at_mut(n); + let v = &left[2..]; + if taus[0] != T::zero() && n > 2 { + reflect_columns(taus[0], v, &mut right[n + 1..], n, n - 2); + } + make_column(&mut right[..n], 1, taus[0], v); + } + h[0] = T::one(); + h[1..n].fill(T::zero()); +} + +/// Sets `col` to the column `c` of the reflection `I - τ·(1, v)·(1, v)*` on the rows from `c`: +/// zeros above, `1 - τ` on the diagonal, `-τ·v` below. +pub(crate) fn make_column(col: &mut [T], c: usize, tau: T, v: &[T]) { + col[..c].fill(T::zero()); + col[c] = T::one() - tau; + let minus = -tau; + for (x, &v) in col[c + 1..].iter_mut().zip(v) { + *x = minus * v; + } +} + +/// `x -= τ·v·(v*·x)`, the reflection `I - τ·(1, v)·(1, v)*` applied from the left, for the +/// `cols` columns `x` of the block `c` (leading dimension `ld`), of `v.len() + 1` elements +/// each: four columns a pass, so that each element of `v` is loaded once for the four products +/// and once for the four updates, the products in lanes of two vector registers each, so that +/// their additions don't wait on one another. +pub(crate) fn reflect_columns(tau: T, v: &[T], c: &mut [T], ld: usize, cols: usize) { + match size_of::() { + 4 => reflect_columns_by::(tau, v, c, ld, cols), + 8 => reflect_columns_by::(tau, v, c, ld, cols), + _ => reflect_columns_by::(tau, v, c, ld, cols), + } +} + +/// [`reflect_columns`] with `L` lanes. +fn reflect_columns_by( + tau: T, + v: &[T], + c: &mut [T], + ld: usize, + cols: usize, +) { + let len = v.len(); + let (vb, vt) = v.as_chunks::(); + let body = len - vt.len(); + let mut j = 0; + while j + 4 <= cols { + let block = &mut c[j * ld..]; + let (c0, block) = block.split_at_mut(ld); + let (c1, block) = block.split_at_mut(ld); + let (c2, c3) = block.split_at_mut(ld); + let (h0, r0) = c0[..=len].split_first_mut().expect("a row"); + let (h1, r1) = c1[..=len].split_first_mut().expect("a row"); + let (h2, r2) = c2[..=len].split_first_mut().expect("a row"); + let (h3, r3) = c3[..=len].split_first_mut().expect("a row"); + // `s_k = v*·x_k`, past the leading elements. + let mut s = [[T::zero(); L]; 4]; + let it = vb + .iter() + .zip(lanes_of::(r0)) + .zip(lanes_of::(r1)) + .zip(lanes_of::(r2).iter().zip(lanes_of::(r3))); + for (((v, x0), x1), (x2, x3)) in it { + for l in 0..L { + s[0][l] = fma_conj(v[l], x0[l], s[0][l]); + s[1][l] = fma_conj(v[l], x1[l], s[1][l]); + s[2][l] = fma_conj(v[l], x2[l], s[2][l]); + s[3][l] = fma_conj(v[l], x3[l], s[3][l]); + } + } + let mut w = [T::zero(); 4]; + for (k, (r, h)) in [&*r0, &*r1, &*r2, &*r3] + .into_iter() + .zip([&*h0, &*h1, &*h2, &*h3]) + .enumerate() + { + let mut t = lanes(s[k]); + for (&v, &x) in vt.iter().zip(&r[body..]) { + t = fma_conj(v, x, t); + } + w[k] = tau * (*h + t); + } + *h0 -= w[0]; + *h1 -= w[1]; + *h2 -= w[2]; + *h3 -= w[3]; + let m = [-w[0], -w[1], -w[2], -w[3]]; + let it = r0 + .iter_mut() + .zip(r1.iter_mut()) + .zip(r2.iter_mut().zip(r3.iter_mut())) + .zip(v); + for (((x0, x1), (x2, x3)), &v) in it { + *x0 = fma(v, m[0], *x0); + *x1 = fma(v, m[1], *x1); + *x2 = fma(v, m[2], *x2); + *x3 = fma(v, m[3], *x3); + } + j += 4; + } + if j == cols { + return; + } + for col in c[j * ld..].chunks_mut(ld).take(cols - j) { + let (h, r) = col[..=len].split_first_mut().expect("a row"); + let mut s = [T::zero(); L]; + for (v, x) in vb.iter().zip(lanes_of::(r)) { + for l in 0..L { + s[l] = fma_conj(v[l], x[l], s[l]); + } + } + let mut t = lanes(s); + for (&v, &x) in vt.iter().zip(&r[body..]) { + t = fma_conj(v, x, t); + } + let w = tau * (*h + t); + *h -= w; + let m = -w; + for (x, &v) in r.iter_mut().zip(v) { + *x = fma(v, m, *x); + } + } +} + +/// `x ← H_a·H_b·x` for the `cols` columns `x` of the block `c` (leading dimension `ld`), of +/// `va.len() + 1` elements each: `H_a = I - τ_a·(1, va)·(1, va)*` acting on all of them and +/// `H_b = I - τ_b·(1, vb)·(1, vb)*` on all but the first. The two reflections are applied in +/// one pass of products and one of updates, with each element stored once for both, through +/// `va*·H_b = va* - τ_b·(va*·vb)·vb*`; four columns a pass, the products in lanes. +pub(crate) fn reflect_columns2( + (ta, va): (T, &[T]), + (tb, vb): (T, &[T]), + c: &mut [T], + ld: usize, + cols: usize, +) { + match size_of::() { + 4 => reflect_columns2_by::(ta, va, tb, vb, c, ld, cols), + 8 => reflect_columns2_by::(ta, va, tb, vb, c, ld, cols), + _ => reflect_columns2_by::(ta, va, tb, vb, c, ld, cols), + } +} + +/// [`reflect_columns2`] with `L` lanes. +fn reflect_columns2_by( + ta: T, + va: &[T], + tb: T, + vb: &[T], + c: &mut [T], + ld: usize, + cols: usize, +) { + debug_assert_eq!(va.len(), vb.len() + 1); + let len = va.len() + 1; + // Rows from 2, where both vectors are held: `a` the rest of `va`, `b` that of `vb`. + let (a1, a) = (va[0], &va[1..]); + let b = vb; + let rest = len - 2; + let (ab, at) = a.as_chunks::(); + let (bb, bt) = b.as_chunks::(); + let body = rest - at.len(); + // `γ = va*·(0, 1, vb)`. + let mut g = a1.conj(); + for (&a, &b) in a.iter().zip(b) { + g = fma_conj(a, b, g); + } + // The factors of one column from its products: `w_b = τ_b·s_b`, `w_a = τ_a·(s_a - w_b·γ)`. + let factors = |x0: T, x1: T, sa: T, sb: T| { + let wb = tb * (x1 + sb); + let wa = ta * (x0 + fma_conj(a1, x1, sa) - wb * g); + (wa, wb) + }; + let mut j = 0; + while j + 4 <= cols { + let block = &mut c[j * ld..]; + let (c0, block) = block.split_at_mut(ld); + let (c1, block) = block.split_at_mut(ld); + let (c2, c3) = block.split_at_mut(ld); + let mut x = [c0, c1, c2, c3].map(|c| &mut c[..len]); + let mut sa = [[T::zero(); L]; 4]; + let mut sb = [[T::zero(); L]; 4]; + { + let [x0, x1, x2, x3] = x.each_ref().map(|x| lanes_of::(&x[2..])); + let it = ab + .iter() + .zip(bb) + .zip(x0.iter().zip(x1)) + .zip(x2.iter().zip(x3)); + for (((a, b), (x0, x1)), (x2, x3)) in it { + for l in 0..L { + sa[0][l] = fma_conj(a[l], x0[l], sa[0][l]); + sb[0][l] = fma_conj(b[l], x0[l], sb[0][l]); + sa[1][l] = fma_conj(a[l], x1[l], sa[1][l]); + sb[1][l] = fma_conj(b[l], x1[l], sb[1][l]); + sa[2][l] = fma_conj(a[l], x2[l], sa[2][l]); + sb[2][l] = fma_conj(b[l], x2[l], sb[2][l]); + sa[3][l] = fma_conj(a[l], x3[l], sa[3][l]); + sb[3][l] = fma_conj(b[l], x3[l], sb[3][l]); + } + } + } + let mut wa = [T::zero(); 4]; + let mut wb = [T::zero(); 4]; + for (k, x) in x.iter().enumerate() { + let (mut ta_, mut tb_) = (lanes(sa[k]), lanes(sb[k])); + for ((&a, &b), &x) in at.iter().zip(bt).zip(&x[2 + body..]) { + ta_ = fma_conj(a, x, ta_); + tb_ = fma_conj(b, x, tb_); + } + (wa[k], wb[k]) = factors(x[0], x[1], ta_, tb_); + } + for (k, x) in x.iter_mut().enumerate() { + x[0] -= wa[k]; + x[1] -= fma(wa[k], a1, wb[k]); + } + let (ma, mb) = (wa.map(|w| -w), wb.map(|w| -w)); + let [x0, x1, x2, x3] = &mut x; + let it = x0[2..] + .iter_mut() + .zip(x1[2..].iter_mut()) + .zip(x2[2..].iter_mut().zip(x3[2..].iter_mut())) + .zip(a.iter().zip(b)); + for (((x0, x1), (x2, x3)), (&a, &b)) in it { + *x0 = fma(b, mb[0], fma(a, ma[0], *x0)); + *x1 = fma(b, mb[1], fma(a, ma[1], *x1)); + *x2 = fma(b, mb[2], fma(a, ma[2], *x2)); + *x3 = fma(b, mb[3], fma(a, ma[3], *x3)); + } + j += 4; + } + if j == cols { + return; + } + for col in c[j * ld..].chunks_mut(ld).take(cols - j) { + let x = &mut col[..len]; + let (mut sa, mut sb) = ([T::zero(); L], [T::zero(); L]); + for ((a, b), y) in ab.iter().zip(bb).zip(lanes_of::(&x[2..])) { + for l in 0..L { + sa[l] = fma_conj(a[l], y[l], sa[l]); + sb[l] = fma_conj(b[l], y[l], sb[l]); + } + } + let (mut ta_, mut tb_) = (lanes(sa), lanes(sb)); + for ((&a, &b), &y) in at.iter().zip(bt).zip(&x[2 + body..]) { + ta_ = fma_conj(a, y, ta_); + tb_ = fma_conj(b, y, tb_); + } + let (wa, wb) = factors(x[0], x[1], ta_, tb_); + x[0] -= wa; + x[1] -= fma(wa, a1, wb); + let (ma, mb) = (-wa, -wb); + for (y, (&a, &b)) in x[2..].iter_mut().zip(a.iter().zip(b)) { + *y = fma(b, mb, fma(a, ma, *y)); + } + } +} diff --git a/lib/src/math/matrix/householder.rs b/lib/src/math/matrix/householder.rs new file mode 100644 index 0000000..303e41a --- /dev/null +++ b/lib/src/math/matrix/householder.rs @@ -0,0 +1,362 @@ +//! Householder reflections, alone and in blocks, which QR, Hessenberg and bidiagonal reductions +//! are built of. + +use super::blas::{Op, gemm}; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; + +/// The 2-norm of `x`, scaled so that the squares neither overflow nor underflow. +pub(crate) fn norm2(x: &[T]) -> T::Real { + let mut scale = T::Real::ZERO; + for v in x { + scale = scale.max(v.re().abs()).max(v.im().abs()); + } + if scale == T::Real::ZERO || !scale.is_finite() { + // `max` ignores NaNs, which the sum below would carry: zeros and NaNs alone are NaN. + if scale == T::Real::ZERO && x.iter().any(|v| v.re().is_nan() || v.im().is_nan()) { + return T::Real::NAN; + } + return scale; + } + let r = T::Real::ONE / scale; + let mut s = T::Real::ZERO; + if r.is_finite() { + for v in x { + s += v.scale(r).abs2(); + } + } else { + // A subnormal scale, whose reciprocal overflows: a division for each part. + for v in x { + let (a, b) = (v.re() / scale, v.im() / scale); + s += a * a + b * b; + } + } + s.sqrt() * scale +} + +/// The reflection `H = I - τ·v·v*`, `v = (1, x)`, with `H*·(alpha, x) = (β, 0)` and `β` real, +/// as LAPACK's `larfg`: overwrites `x` with the rest of `v` and returns `(τ, β)`. `τ` is zero, +/// and `H` the identity, when `x` is zero and `alpha` real. +pub(crate) fn reflector(alpha: T, x: &mut [T]) -> (T, T) { + let xnorm = norm2(x); + if xnorm == T::Real::ZERO && alpha.im() == T::Real::ZERO { + return (T::zero(), alpha); + } + reflection(alpha, alpha.abs().hypot(xnorm), x) +} + +/// The rest of [`reflector`] once the norm of `(alpha, x)` is known, which every reflector here +/// shares: `β = ∓norm`, `τ` and `x / (alpha - β)`, as LAPACK's `larfg`, but without overflowing +/// where the norm is near the largest float. `τ = 1 - alpha/β` rather than `(β - alpha)/β`, +/// whose difference adds two magnitudes (`alpha/β` is not positive, so nothing cancels); and +/// `alpha - β` halved when it overflows, or each element divided when `1/(alpha - β)` would. +pub(crate) fn reflection(alpha: T, norm: T::Real, x: &mut [T]) -> (T, T) { + // A norm below `MIN/ε` has lost digits to underflow, so that `(u, u)` for the smallest + // subnormal `u` would take `‖(u, u)‖ = u` and a reflection that is not unitary: `alpha` and + // `x` are scaled up by a power of two first and the norm computed again, as LAPACK's `larfg` + // does, and only `β` scaled back. + let small = T::Real::MIN_POSITIVE / T::Real::EPSILON; + if norm > T::Real::ZERO && norm < small { + let up = T::Real::ONE / small; + for v in x.iter_mut() { + *v = v.scale(up); + } + let alpha = alpha.scale(up); + let (tau, beta) = reflection(alpha, alpha.abs().hypot(norm2(x)), x); + return (tau, beta.scale(small)); + } + let beta = if alpha.re() >= T::Real::ZERO { + -norm + } else { + norm + }; + let tau = T::from_parts(T::Real::ONE - alpha.re() / beta, -alpha.im() / beta); + let d = alpha - T::from_real(beta); + if !d.is_finite() { + let half = alpha.scale(T::Real::HALF) - T::from_real(beta * T::Real::HALF); + let r = T::one() / half; + for v in x.iter_mut() { + *v = (*v * r).scale(T::Real::HALF); + } + } else { + let r = T::one() / d; + if r.is_finite() { + for v in x.iter_mut() { + *v *= r; + } + } else { + for v in x.iter_mut() { + *v /= d; + } + } + } + (tau, T::from_real(beta)) +} + +/// Applies `H = I - τ·v·v*` (`conj` false) or `H*` (`conj` true) from the left to the `m × n` +/// block `c`, `v` of length `m` with its first element taken as 1. +pub(crate) fn reflect_left( + tau: T, + conj: bool, + v: &[T], + m: usize, + n: usize, + c: &mut [T], + ldc: usize, +) { + if tau == T::zero() { + return; + } + let tau = if conj { tau.conj() } else { tau }; + for j in 0..n { + let col = &mut c[j * ldc..j * ldc + m]; + // `w = v*·c_j`. + let mut w = col[0]; + for (x, v) in col[1..].iter().zip(&v[1..m]) { + w += v.conj() * *x; + } + let w = tau * w; + col[0] -= w; + for (x, v) in col[1..].iter_mut().zip(&v[1..m]) { + *x -= *v * w; + } + } +} + +/// Applies `H = I - τ·v·v*` from the right to the `m × n` block `c`, `v` of length `n` with its +/// first element taken as 1. +pub(crate) fn reflect_right( + tau: T, + v: &[T], + m: usize, + n: usize, + c: &mut [T], + ldc: usize, +) { + if tau == T::zero() { + return; + } + // `w = c·v`, then `c -= τ·w·v*`. + let mut w: Vec = c[..m].to_vec(); + for p in 1..n { + let x = v[p]; + for (w, c) in w.iter_mut().zip(&c[p * ldc..p * ldc + m]) { + *w += *c * x; + } + } + for w in &mut w { + *w *= tau; + } + for (c, w) in c[..m].iter_mut().zip(&w) { + *c -= *w; + } + for p in 1..n { + let x = v[p].conj(); + for (c, w) in c[p * ldc..p * ldc + m].iter_mut().zip(&w) { + *c -= *w * x; + } + } +} + +/// A block of `k` reflections `H₁·H₂·…·H_k = I - V·T·V*`, LAPACK's compact WY form: `V` of +/// `m × k`, unit lower trapezoidal, held with its ones and zeros, and `T` of `k × k` upper +/// triangular. +pub(crate) struct Block { + /// `V`, `m × k`. + pub v: Vec, + /// `T`, `k × k`. + pub t: Vec, + /// Rows. + pub m: usize, + /// Reflections. + pub k: usize, +} + +impl Block { + /// The block of the reflections whose vectors are below the diagonal of the `m × k` block + /// `a` (as a QR panel leaves them) and whose factors are `taus`, as LAPACK's `larft`. + pub fn new(a: &[T], lda: usize, m: usize, taus: &[T]) -> Block { + let k = taus.len(); + let mut v = vec![T::zero(); m * k]; + for j in 0..k { + v[j + j * m] = T::one(); + v[j * m + j + 1..(j + 1) * m].copy_from_slice(&a[j * lda + j + 1..j * lda + m]); + } + let mut t = vec![T::zero(); k * k]; + for j in 0..k { + let tau = taus[j]; + t[j + j * k] = tau; + if j == 0 || tau == T::zero() { + continue; + } + // `T[0..j, j] = -τ·T[0..j, 0..j]·V[:, 0..j]*·v_j`. + let mut w = vec![T::zero(); j]; + gemm( + j, + 1, + m - j, + -tau, + &v[j..], + m, + Op::C, + &v[j * m + j..], + m, + Op::N, + T::zero(), + &mut w, + j, + ); + for i in 0..j { + let mut s = T::zero(); + for (p, w) in w.iter().enumerate().skip(i) { + s += t[i + p * k] * *w; + } + t[i + j * k] = s; + } + } + Block { v, t, m, k } + } + + /// Applies the block, or its conjugate transpose when `conj` is set, from the left to the + /// `m × n` block `c`: `c -= V·op(T)·(V*·c)`. + pub fn apply_left(&self, conj: bool, n: usize, c: &mut [T], ldc: usize) { + let (m, k) = (self.m, self.k); + if n == 0 || k == 0 { + return; + } + let mut w = vec![T::zero(); k * n]; + gemm( + k, + n, + m, + T::one(), + &self.v, + m, + Op::C, + c, + ldc, + Op::N, + T::zero(), + &mut w, + k, + ); + let mut tw = vec![T::zero(); k * n]; + let op = if conj { Op::C } else { Op::N }; + gemm( + k, + n, + k, + T::one(), + &self.t, + k, + op, + &w, + k, + Op::N, + T::zero(), + &mut tw, + k, + ); + gemm( + m, + n, + k, + -T::one(), + &self.v, + m, + Op::N, + &tw, + k, + Op::N, + T::one(), + c, + ldc, + ); + } + + /// Applies the block from the right to the `n × m` block `c`: `c -= (c·V)·T·V*`. + pub fn apply_right(&self, n: usize, c: &mut [T], ldc: usize) { + let (m, k) = (self.m, self.k); + if n == 0 || k == 0 { + return; + } + let mut w = vec![T::zero(); n * k]; + gemm( + n, + k, + m, + T::one(), + c, + ldc, + Op::N, + &self.v, + m, + Op::N, + T::zero(), + &mut w, + n, + ); + let mut wt = vec![T::zero(); n * k]; + gemm( + n, + k, + k, + T::one(), + &w, + n, + Op::N, + &self.t, + k, + Op::N, + T::zero(), + &mut wt, + n, + ); + gemm( + n, + m, + k, + -T::one(), + &wt, + n, + Op::N, + &self.v, + m, + Op::C, + T::one(), + c, + ldc, + ); + } +} + +/// The `m × m` (or `m × cols`) matrix `H₁·H₂·…·H_k` applied to the first `cols` columns of the +/// identity, for the reflections below the diagonal of the `m × k` block `a`, as LAPACK's +/// `orgqr`: the explicit `Q` of a QR decomposition. +pub(crate) fn form_q( + a: &[T], + lda: usize, + m: usize, + taus: &[T], + cols: usize, +) -> Matrix { + let mut q = Matrix::zeros(m, cols); + for i in 0..m.min(cols) { + q.data[i + i * m] = T::one(); + } + let k = taus.len(); + // The blocks of `PANEL` reflections from the last: each changes only the rows and columns + // from its first reflection on, the columns before being those of the identity still. + let mut end = k; + while end > 0 { + let start = (end - 1) / PANEL * PANEL; + let block = Block::new(&a[start + start * lda..], lda, m - start, &taus[start..end]); + let n = cols - start; + block.apply_left(false, n, &mut q.data[start + start * m..], m); + end = start; + } + q +} + +/// The width of the panels of blocked reductions. +pub(crate) const PANEL: usize = 32; diff --git a/lib/src/math/matrix/lu.rs b/lib/src/math/matrix/lu.rs new file mode 100644 index 0000000..039bc85 --- /dev/null +++ b/lib/src/math/matrix/lu.rs @@ -0,0 +1,772 @@ +//! The LU decomposition with partial pivoting, and what it gives: determinants, solutions of +//! linear systems and inverses. +//! +//! # The decomposition +//! +//! Square matrices up to 16 × 16 are factored by [`tiny`]: a copy on the stack, its order known +//! so that its loops unroll, the pivots chosen and the rows swapped by selects rather than +//! branches, which keeps the whole matrix in registers. Up to 24 columns, [`unblocked`] +//! eliminates a column at a time by rank-one updates of whole columns. Larger matrices go +//! Toledo's way ([`factor`]): the left half, then the right half updated by it, down to panels +//! of [`NARROW`] = 16 columns. The row swaps of each half are applied to the other a column at a +//! time, each column read once, rather than a row at a time across all of them, which took a +//! third of the time at 512 × 512; and `U12 = L11⁻¹·A12` comes from the triangular solve with a +//! dense copy of it that the product updating `A22` reads, in working memory taken once. +//! +//! Every path chooses the same pivots, the first element of the largest `|re| + |im|` in its +//! column, as LAPACK does, and divides by a pivot so small that its reciprocal overflows rather +//! than multiplying by the reciprocal, as LAPACK's `getf2` does. +//! +//! # The inverse +//! +//! Up to 12 × 12, [`tiny_inverse`] inverts on the stack, in unrolled loops, and allocates only +//! the inverse. Larger matrices are factored in working memory, then inverted as LAPACK's +//! `getri` does: `U⁻¹` by [`upper_inverse`], then `X·L = U⁻¹` by one triangular solve on the +//! right, then the columns swapped back. That is `2n³` operations in all, against `8n³/3` for +//! the solve against the identity that it replaced, nearly all of them in products of blocks. +//! +//! # Speed +//! +//! Measured on an Apple M4 Max, one core, `f64`, in µs, the median of 15 to 21 interleaved +//! batches, against faer 0.24 (`partial_piv_lu`, then its `inverse`, `Par::Seq`) and nalgebra +//! 0.35 (`lu`, `try_inverse`): +//! +//! | n | LU | faer | nalgebra | inverse | faer | nalgebra | +//! | --- | --- | --- | --- | --- | --- | --- | +//! | 2 | 0.034 | 0.122 | 0.036 | 0.014 | 0.233 | 0.015 | +//! | 4 | 0.061 | 0.223 | 0.071 | 0.054 | 0.490 | 0.042 | +//! | 8 | 0.140 | 0.381 | 0.173 | 0.209 | 0.970 | 0.582 | +//! | 16 | 0.525 | 0.933 | 0.541 | 1.21 | 2.57 | 2.43 | +//! | 32 | 2.03 | 3.19 | 2.59 | 5.10 | 9.24 | 13.8 | +//! | 64 | 9.45 | 14.2 | 14.7 | 24.0 | 38.1 | 70.8 | +//! | 128 | 52.5 | 71.2 | 99.8 | 146 | 183 | 439 | +//! | 256 | 302 | 400 | 1 109 | 903 | 1 061 | 4 071 | +//! | 512 | 2 024 | 2 697 | 8 417 | 5 769 | 7 675 | 31 030 | +//! | 1024 | 14 424 | 18 348 | 63 443 | 43 363 | 53 809 | 231 502 | +//! +//! nalgebra inverts matrices up to 4 × 4 by their cofactors, a formula with half the operations +//! and none of the pivoting, at 3 × 3 and 4 × 4 a little faster than the decomposition. Apple's +//! Accelerate, on the matrix unit (SME) rather than NEON, ran 2 to 5 times faster from 128 × 128 +//! up (`dgetrf` 836 µs and `dgetrf` with `dgetri` 1 785 µs at 512). + +use super::blas::{Op, Side, Uplo, gemm, trsm, trsm_into, with_scratch}; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; +use core::hint::select_unpredictable; + +/// The LU decomposition `P·A = L·U` of an `m × n` matrix, by Gaussian elimination with partial +/// pivoting: `P` a permutation, `L` of `m × min(m, n)` lower triangular with ones on its +/// diagonal, `U` of `min(m, n) × n` upper triangular. From [`Matrix::lu`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a = Matrix::from_rows(&[[2.0, 1.0], [4.0, 3.0]]); +/// let lu = a.lu(); +/// // The larger first column's element is the pivot: the rows are swapped. +/// assert_eq!(lu.permutation(), [1, 0]); +/// assert_eq!(lu.l(), Matrix::from_rows(&[[1.0, 0.0], [0.5, 1.0]])); +/// assert_eq!(lu.u(), Matrix::from_rows(&[[4.0, 3.0], [0.0, -0.5]])); +/// assert_eq!(&lu.p() * &a, &lu.l() * &lu.u()); +/// assert_eq!(lu.det(), 2.0); +/// let x = lu.solve(&Matrix::from_rows(&[[3.0], [7.0]])).unwrap(); +/// assert_eq!(x, Matrix::from_rows(&[[1.0], [1.0]])); +/// ``` +#[derive(Clone, Debug)] +pub struct Lu { + /// `L` below the diagonal and `U` on and above it. + lu: Matrix, + /// Row `i` of `P·A` is row `perm[i]` of `A`. + perm: Vec, + /// Whether `P` is an odd permutation. + odd: bool, +} + +impl Matrix { + /// The LU decomposition with partial pivoting, `P·A = L·U`: see [`Lu`]. It takes about + /// `2n³/3` operations for an `n × n` matrix, nearly all of them in products of blocks. + pub fn lu(&self) -> Lu { + let (m, n) = self.shape(); + let mut lu = self.clone(); + let mut perm: Vec = (0..m).collect(); + let mut odd = false; + if m == n && n <= TINY { + // Tiny squares unrolled, their loops of known lengths. + match n { + 0 => {} + 1 => odd = tiny::(&mut lu.data, &mut perm), + 2 => odd = tiny::(&mut lu.data, &mut perm), + 3 => odd = tiny::(&mut lu.data, &mut perm), + 4 => odd = tiny::(&mut lu.data, &mut perm), + 5 => odd = tiny::(&mut lu.data, &mut perm), + 6 => odd = tiny::(&mut lu.data, &mut perm), + 7 => odd = tiny::(&mut lu.data, &mut perm), + 8 => odd = tiny::(&mut lu.data, &mut perm), + 9 => odd = tiny::(&mut lu.data, &mut perm), + 10 => odd = tiny::(&mut lu.data, &mut perm), + 11 => odd = tiny::(&mut lu.data, &mut perm), + 12 => odd = tiny::(&mut lu.data, &mut perm), + 13 => odd = tiny::(&mut lu.data, &mut perm), + 14 => odd = tiny::(&mut lu.data, &mut perm), + 15 => odd = tiny::(&mut lu.data, &mut perm), + _ => odd = tiny::(&mut lu.data, &mut perm), + } + return Lu { lu, perm, odd }; + } + with_pivots(m.min(n), |pivots| { + factor_all(&mut lu.data, m, n, pivots); + for (i, &p) in pivots.iter().enumerate() { + if p != i { + perm.swap(i, p); + odd = !odd; + } + } + }); + Lu { lu, perm, odd } + } + + /// The determinant `det(A)`, from the LU decomposition: the product of the pivots, with the + /// sign of the permutation. The determinant of the `0 × 0` matrix is 1. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn det(&self) -> T { + assert!(self.is_square(), "the matrix is not square"); + self.lu().det() + } + + /// The inverse `A⁻¹`, from the LU decomposition, or `None` when a pivot is zero: the matrix + /// is singular. A matrix that is singular only up to rounding gives an inverse with huge + /// elements; its [condition number](Matrix::cond) tells how far to trust it. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn inverse(&self) -> Option { + assert!(self.is_square(), "the matrix is not square"); + match self.rows() { + 0 => Some(self.clone()), + 1 => tiny_inverse_inline::(self), + 2 => tiny_inverse_inline::(self), + 3 => tiny_inverse_inline::(self), + 4 => tiny_inverse_inline::(self), + 5 => tiny_inverse::(self), + 6 => tiny_inverse::(self), + 7 => tiny_inverse::(self), + 8 => tiny_inverse::(self), + 9 => tiny_inverse::(self), + 10 => tiny_inverse::(self), + 11 => tiny_inverse::(self), + 12 => tiny_inverse::(self), + n => { + // The factors in working memory, their row swaps kept as they are, so that the + // inverse is the only allocation. + with_scratch(n * n, |lu| { + lu.copy_from_slice(&self.data); + with_pivots(n, |pivots| { + factor_all(lu, n, n, pivots); + if (0..n).any(|i| lu[i + i * n] == T::zero()) { + return None; + } + let mut x = Matrix::zeros(n, n); + inverse_from(lu, n, &mut x.data); + // Column `j` of `X·P` is column `pivots[j]` of `X` and the other way + // round, from the last swap to the first. + for (j, &p) in pivots.iter().enumerate().rev() { + if p != j { + let (a, b) = x.data.split_at_mut(p * n); + a[j * n..j * n + n].swap_with_slice(&mut b[..n]); + } + } + Some(x) + }) + }) + } + } + } + + /// The solution `X` of `A·X = B`, from the LU decomposition, or `None` when the matrix is + /// singular. + /// + /// # Panics + /// + /// Panics if the matrix is not square, or `B` has not as many rows. + #[track_caller] + pub fn solve(&self, b: &Matrix) -> Option { + assert!(self.is_square(), "the matrix is not square"); + self.lu().solve(b) + } + + /// `Aⁿ` for any integer `n`: [`pow`](Matrix::pow) of the inverse for a negative `n`, `None` + /// when that doesn't exist. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn powi(&self, n: i32) -> Option { + if n >= 0 { + Some(self.pow(n.unsigned_abs())) + } else { + Some(self.inverse()?.pow(n.unsigned_abs())) + } + } +} + +impl Lu { + /// `L`, `m × min(m, n)`, lower triangular with ones on the diagonal. + pub fn l(&self) -> Matrix { + let (m, n) = self.lu.shape(); + Matrix::from_fn(m, m.min(n), |i, j| match i.cmp(&j) { + core::cmp::Ordering::Greater => self.lu[(i, j)], + core::cmp::Ordering::Equal => T::one(), + core::cmp::Ordering::Less => T::zero(), + }) + } + + /// `U`, `min(m, n) × n`, upper triangular. + pub fn u(&self) -> Matrix { + let (m, n) = self.lu.shape(); + Matrix::from_fn(m.min(n), n, |i, j| { + if i <= j { self.lu[(i, j)] } else { T::zero() } + }) + } + + /// `P`, the `m × m` permutation matrix with `P·A = L·U`. + pub fn p(&self) -> Matrix { + let m = self.perm.len(); + Matrix::from_fn(m, m, |i, j| { + if self.perm[i] == j { + T::one() + } else { + T::zero() + } + }) + } + + /// The permutation: row `i` of `P·A` is row `permutation()[i]` of `A`. + pub fn permutation(&self) -> &[usize] { + &self.perm + } + + /// Whether a pivot is zero, which for a square matrix makes it singular. + pub fn is_singular(&self) -> bool { + let (m, n) = self.lu.shape(); + (0..m.min(n)).any(|i| self.lu[(i, i)] == T::zero()) + } + + /// The determinant of the square matrix: the product of the pivots, negated for an odd + /// permutation. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn det(&self) -> T { + assert!(self.lu.is_square(), "the matrix is not square"); + let mut d = T::one(); + for i in 0..self.lu.rows() { + d *= self.lu[(i, i)]; + } + if self.odd { -d } else { d } + } + + /// The solution `X` of `A·X = B` for the square matrix, or `None` when it is singular. + /// + /// # Panics + /// + /// Panics if the matrix is not square, or `B` has not as many rows. + #[track_caller] + pub fn solve(&self, b: &Matrix) -> Option> { + assert!(self.lu.is_square(), "the matrix is not square"); + let n = self.lu.rows(); + assert_eq!(b.rows(), n, "the right-hand side has not as many rows"); + if self.is_singular() { + return None; + } + let mut x = Matrix::from_fn(n, b.cols(), |i, j| b[(self.perm[i], j)]); + let cols = x.cols(); + trsm( + Side::Left, + Uplo::Lower, + Op::N, + true, + n, + cols, + &self.lu.data, + n, + &mut x.data, + n, + ); + trsm( + Side::Left, + Uplo::Upper, + Op::N, + false, + n, + cols, + &self.lu.data, + n, + &mut x.data, + n, + ); + Some(x) + } + + /// The inverse of the square matrix, or `None` when it is singular. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn inverse(&self) -> Option> { + assert!(self.lu.is_square(), "the matrix is not square"); + if self.is_singular() { + return None; + } + // `A⁻¹ = U⁻¹·L⁻¹·P`, as LAPACK's `getri`: `U⁻¹`, then `X·L = U⁻¹` for `X`, then the + // columns of `X` permuted; about `4n³/3` operations, nearly all of them in products of + // blocks. + let n = self.lu.rows(); + let mut x = Matrix::zeros(n, n); + inverse_from(&self.lu.data, n, &mut x.data); + // Column `perm[j]` of `X·P` is column `j` of `X`: the cycles of the permutation, a swap + // of whole columns at a time. + let mut to = self.perm.clone(); + for j in 0..n { + while to[j] != j { + let k = to[j]; + let (a, b) = x.data.split_at_mut(j.max(k) * n); + a[j.min(k) * n..j.min(k) * n + n].swap_with_slice(&mut b[..n]); + to.swap(j, k); + } + } + Some(x) + } +} + +/// Writes `X = U⁻¹·L⁻¹` for the factors `lu` of order `n` to `x`, which holds zeros: `U⁻¹`, +/// then `X·L = U⁻¹` for `X`. +fn inverse_from(lu: &[T], n: usize, x: &mut [T]) { + upper_inverse(lu, n, x, n, n); + trsm(Side::Right, Uplo::Lower, Op::N, true, n, n, lu, n, x, n); +} + +/// Triangles up to this order are inverted by a solve against the identity. +const INVERSE_LEAF: usize = 16; + +/// Writes the inverse of the upper triangle of order `n` at the start of `u` to the upper +/// triangle of `w`, recursively: `W11 = U11⁻¹`, `W22 = U22⁻¹` and `W12 = -U11⁻¹·U12·U22⁻¹`, the +/// last by two triangular solves from the triangles of `U` itself, so that no block is read +/// where another is written. The lower triangle of `w` is left as it is. +fn upper_inverse(u: &[T], ldu: usize, w: &mut [T], ldw: usize, n: usize) { + if n <= INVERSE_LEAF { + for j in 0..n { + for i in 0..=j { + w[i + j * ldw] = if i == j { T::one() } else { T::zero() }; + } + } + trsm(Side::Left, Uplo::Upper, Op::N, false, n, n, u, ldu, w, ldw); + return; + } + let n1 = split(n); + let n2 = n - n1; + let (w1, w2) = w.split_at_mut(n1 * ldw); + for j in 0..n2 { + let (src, dst) = (&u[(n1 + j) * ldu..], &mut w2[j * ldw..]); + for (d, s) in dst[..n1].iter_mut().zip(&src[..n1]) { + *d = -*s; + } + } + let u22 = &u[n1 + n1 * ldu..]; + trsm( + Side::Left, + Uplo::Upper, + Op::N, + false, + n1, + n2, + u, + ldu, + w2, + ldw, + ); + trsm( + Side::Right, + Uplo::Upper, + Op::N, + false, + n1, + n2, + u22, + ldu, + w2, + ldw, + ); + upper_inverse(u, ldu, w1, ldw, n1); + upper_inverse(u22, ldu, &mut w2[n1..], ldw, n2); +} + +/// [`Matrix::inverse`] of an `N × N` matrix, `N` known, so that its loops unroll: the LU +/// decomposition of a copy on the stack, as [`tiny`] finds it, then `U⁻¹`, `X·L = U⁻¹` and the +/// permutation of the columns, all on the stack, then the only allocation, the inverse. +/// +/// The decomposition stops at the first zero pivot, and does not skip the updates by zeros: these +/// change nothing that is finite, and the branches cost more than they save here. The pivots are +/// chosen and the rows swapped by selects, not branches, which keeps the whole matrix in +/// registers. Orders from 5 are functions of their own, as one function holding them all compiled +/// worse; up to 4, the call itself cost a fifth of the time, so they are inlined. +#[inline(never)] +fn tiny_inverse(a: &Matrix) -> Option> { + tiny_inverse_inline::(a) +} + +/// [`tiny_inverse`], inlined. Its loops index rows and columns as the elimination is written: every +/// index is known once they unroll. +#[inline(always)] +#[allow(clippy::needless_range_loop)] +fn tiny_inverse_inline(a: &Matrix) -> Option> { + let mut lu = [[T::zero(); N]; N]; + lu.as_flattened_mut().copy_from_slice(&a.data); + let mut perm: [usize; N] = core::array::from_fn(|i| i); + let mut r = [T::zero(); N]; + for j in 0..N { + // The first element of the largest `|re| + |im|`, the first itself when it is NaN. + let mut p = j; + let mut best = lu[j][j].abs1(); + for i in j + 1..N { + let v = lu[j][i].abs1(); + let greater = v > best; + best = select_unpredictable(greater, v, best); + p = select_unpredictable(greater, i, p); + } + // The swap by selects, every index known: the whole matrix stays in registers. + for i in j + 1..N { + let swap = p == i; + for col in &mut lu { + let (x, y) = (col[j], col[i]); + col[j] = select_unpredictable(swap, y, x); + col[i] = select_unpredictable(swap, x, y); + } + let (x, y) = (perm[j], perm[i]); + perm[j] = select_unpredictable(swap, y, x); + perm[i] = select_unpredictable(swap, x, y); + } + let d = lu[j][j]; + if d == T::zero() { + return None; + } + r[j] = T::one() / d; + if r[j].is_finite() { + for i in j + 1..N { + lu[j][i] *= r[j]; + } + } else { + // A reciprocal that overflows, of a pivot near the bottom of the range: divide. + for i in j + 1..N { + lu[j][i] /= d; + } + } + for c in j + 1..N { + let u = lu[c][j]; + for i in j + 1..N { + lu[c][i] = lu[c][i].mul_sub(lu[j][i], u); + } + } + } + // `W = U⁻¹` by columns, each by back substitution with the reciprocals of the diagonal. + let mut w = [[T::zero(); N]; N]; + for j in 0..N { + w[j][j] = r[j]; + for i in (0..j).rev() { + let mut s = T::zero(); + for k in i + 1..=j { + s = s.mul_sub(lu[k][i], w[j][k]); + } + w[j][i] = s * r[i]; + } + } + // `X·L = W` from the last column: `X[:, j] = W[:, j] - Σ_{k > j} X[:, k]·L[k, j]`. + let mut x = [[T::zero(); N]; N]; + for j in (0..N).rev() { + let mut col = w[j]; + for k in j + 1..N { + let l = lu[j][k]; + for (c, x) in col.iter_mut().zip(&x[k]) { + *c = c.mul_sub(*x, l); + } + } + x[j] = col; + } + // Column `perm[j]` of `X·P` is column `j` of `X`. + let mut inverse = vec![T::zero(); N * N]; + for (j, &p) in perm.iter().enumerate() { + inverse[p * N..p * N + N].copy_from_slice(&x[j]); + } + Some(Matrix::from_column_major(N, N, inverse)) +} + +/// Pivots up to this many are kept on the stack. +const STACK_PIVOTS: usize = 32; + +/// Runs `f` on `k` zeros, for the row swaps of a factorization. +fn with_pivots(k: usize, f: impl FnOnce(&mut [usize]) -> X) -> X { + if k <= STACK_PIVOTS { + f(&mut [0; STACK_PIVOTS][..k]) + } else { + f(&mut vec![0; k]) + } +} + +/// Factors the `m × n` matrix `a` in place, `P·A = L·U`: `pivots[i]` is the row swapped with +/// row `i`, in turn, for `i < min(m, n)`. +fn factor_all(a: &mut [T], m: usize, n: usize, pivots: &mut [usize]) { + let k = m.min(n); + if k == 0 { + return; + } + if k <= UNBLOCKED { + unblocked(a, m, k, m, pivots); + } else { + with_scratch(work_size(k), |work| factor(a, m, k, m, pivots, work)); + } + if n > k { + // The columns right of the square part: the same row swaps, then `L⁻¹`. + let (left, right) = a.split_at_mut(k * m); + swap_rows(right, m, n - k, pivots, 0); + trsm( + Side::Left, + Uplo::Lower, + Op::N, + true, + k, + n - k, + left, + m, + right, + m, + ); + } +} + +/// Columns narrower than this are factored without recursion. +const NARROW: usize = 16; + +/// Matrices of up to this many columns are factored without recursion at all. +const UNBLOCKED: usize = 24; + +/// Where the recursion splits `n` columns: the first part a multiple of [`NARROW`], so that the +/// unblocked panels are full. +fn split(n: usize) -> usize { + (n / 2).next_multiple_of(NARROW).min(n - 1) +} + +/// The working memory of [`factor`] of `n` columns: the dense copy of the largest `U12` of the +/// recursion, which an uneven split may leave below its top. +fn work_size(n: usize) -> usize { + if n <= NARROW { + return 0; + } + let n1 = split(n); + (n1 * (n - n1)).max(work_size(n1)).max(work_size(n - n1)) +} + +/// Factors the `m × n` block at the start of `a`, `n ≤ m`, in place, Toledo's way: the left +/// half, then the right half updated by it, each half the same way down to narrow panels. +/// `pivots[i]` is the row swapped with row `i`, within the block. `work` holds the dense copy of +/// `U12` for the product that updates `A22`: at least [`work_size`] elements. +fn factor( + a: &mut [T], + m: usize, + n: usize, + ld: usize, + pivots: &mut [usize], + work: &mut [T], +) { + if n <= NARROW { + unblocked(a, m, n, ld, pivots); + return; + } + let n1 = split(n); + let n2 = n - n1; + factor(a, m, n1, ld, &mut pivots[..n1], work); + let (left, right) = a.split_at_mut(n1 * ld); + swap_rows(right, ld, n2, &pivots[..n1], 0); + // `U12 = L11⁻¹·A12`, then `A22 -= L21·U12`, `U12` read from its copy. + let u12 = &mut work[..n1 * n2]; + trsm_into( + Uplo::Lower, + Op::N, + true, + n1, + n2, + left, + ld, + right, + ld, + u12, + n1, + ); + gemm( + m - n1, + n2, + n1, + -T::one(), + &left[n1..], + ld, + Op::N, + u12, + n1, + Op::N, + T::one(), + &mut right[n1..], + ld, + ); + factor(&mut right[n1..], m - n1, n2, ld, &mut pivots[n1..], work); + for p in &mut pivots[n1..] { + *p += n1; + } + swap_rows(left, ld, n1, &pivots[n1..], n1); +} + +/// Factors a narrow block by rank-one updates, choosing as pivot the element of the largest +/// `|re| + |im|` in each column, as LAPACK does. A zero column is left as it is. +fn unblocked(a: &mut [T], m: usize, n: usize, ld: usize, pivots: &mut [usize]) { + for j in 0..n { + let p = j + pivot(&a[j * ld + j..j * ld + m]); + pivots[j] = p; + if p != j { + for c in 0..n { + a.swap(j + c * ld, p + c * ld); + } + } + let (left, right) = a.split_at_mut(((j + 1) * ld).min(a.len())); + let col = &mut left[j * ld + j..j * ld + m]; + let (d, l) = col.split_first_mut().unwrap(); + if *d == T::zero() { + continue; + } + let r = T::one() / *d; + if r.is_finite() { + for x in l.iter_mut() { + *x *= r; + } + } else { + // A reciprocal that overflows, of a pivot near the bottom of the range: divide, as + // LAPACK's `getf2` does. + for x in l.iter_mut() { + *x /= *d; + } + } + for c in 0..n - j - 1 { + let col = &mut right[c * ld + j..c * ld + m]; + let (u, x) = col.split_first_mut().unwrap(); + let u = *u; + if u == T::zero() { + continue; + } + for (x, l) in x.iter_mut().zip(l.iter()) { + *x = x.mul_sub(*l, u); + } + } + } +} + +/// The order up to which square matrices are factored by [`tiny`]: past it, [`unblocked`] ran +/// as fast. +const TINY: usize = 16; + +/// [`unblocked`] of an `N × N` matrix, `N` known, so that its loops unroll; the row swaps are +/// applied to `perm`, and the result is whether they make an odd permutation. The matrix is +/// copied to the stack, its pivots chosen and its rows swapped by selects, not branches, every +/// index known, so that it stays in registers: a 5 × 5 decomposition so took 67 ns against 88. +/// Each order is a function of its own. Its loops index rows and columns as the elimination is +/// written: every index is known once they unroll. +#[inline(never)] +#[allow(clippy::needless_range_loop)] +fn tiny(data: &mut [T], perm: &mut [usize]) -> bool { + let mut a = [[T::zero(); N]; N]; + a.as_flattened_mut().copy_from_slice(&data[..N * N]); + let perm: &mut [usize; N] = (&mut perm[..N]).try_into().unwrap(); + let mut odd = false; + for j in 0..N { + // The first element of the largest `|re| + |im|`, the first itself when it is NaN. + let mut p = j; + let mut best = a[j][j].abs1(); + for i in j + 1..N { + let v = a[j][i].abs1(); + let greater = v > best; + best = select_unpredictable(greater, v, best); + p = select_unpredictable(greater, i, p); + } + odd ^= p != j; + for i in j + 1..N { + let swap = p == i; + for col in &mut a { + let (x, y) = (col[j], col[i]); + col[j] = select_unpredictable(swap, y, x); + col[i] = select_unpredictable(swap, x, y); + } + let (x, y) = (perm[j], perm[i]); + perm[j] = select_unpredictable(swap, y, x); + perm[i] = select_unpredictable(swap, x, y); + } + let d = a[j][j]; + if d == T::zero() { + continue; + } + let r = T::one() / d; + if r.is_finite() { + for i in j + 1..N { + a[j][i] *= r; + } + } else { + // A reciprocal that overflows, of a pivot near the bottom of the range: divide. + for i in j + 1..N { + a[j][i] /= d; + } + } + for c in j + 1..N { + let u = a[c][j]; + if u == T::zero() { + continue; + } + for i in j + 1..N { + a[c][i] = a[c][i].mul_sub(a[j][i], u); + } + } + } + data[..N * N].copy_from_slice(a.as_flattened()); + odd +} + +/// The index of the pivot of a column: the first element of the largest `|re| + |im|`, the +/// first element itself when it is NaN, as the scan from it with `>` finds. +fn pivot(col: &[T]) -> usize { + let first = col[0].abs1(); + if first.is_nan() { + return 0; + } + // The largest by `max`, which passes over NaNs, then where it first is. + let best = col.iter().fold(first, |b, x| b.max(x.abs1())); + col.iter().position(|x| x.abs1() == best).unwrap_or(0) +} + +/// Swaps rows `first + i` and `pivots[i]` (an absolute row) of the `n` columns of `a`, in order, +/// a column at a time. +fn swap_rows(a: &mut [T], ld: usize, n: usize, pivots: &[usize], first: usize) { + for c in 0..n { + let col = &mut a[c * ld..]; + for (i, &p) in pivots.iter().enumerate() { + let i = first + i; + if p != i { + col.swap(i, p); + } + } + } +} diff --git a/lib/src/math/matrix/mod.rs b/lib/src/math/matrix/mod.rs new file mode 100644 index 0000000..a50ea80 --- /dev/null +++ b/lib/src/math/matrix/mod.rs @@ -0,0 +1,403 @@ +//! Matrices: a [`Matrix`] is a rectangular array of elements of the type `T`, which is `f32`, +//! `f64`, [`Complex`](crate::math::complex::Complex), +//! [`Complex`](crate::math::complex::Complex) or [`Number`](crate::math::numbers::Number). + +mod basic; +mod blas; +mod cholesky; +mod eigen; +mod element; +mod exact; +mod functions; +mod gemm; +mod hermitian; +mod householder; +mod lu; +mod ops; +mod properties; +mod qr; +mod schur; +mod svd; +mod tridiagonal; + +pub use cholesky::Cholesky; +pub use eigen::{Eigen, HermitianEigen}; +pub use element::{Element, Scalar}; +pub use lu::Lu; +pub use properties::Norm; +pub use qr::Qr; +pub use schur::Schur; +pub use svd::Svd; + +use core::fmt; +use core::ops::{Index, IndexMut}; + +/// A matrix of `rows × cols` elements of the type `T`, held column after column in one vector. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a = Matrix::from_rows(&[[1.0, 2.0], [3.0, 4.0]]); +/// let b = Matrix::identity(2); +/// assert_eq!(&a * &b, a); +/// assert_eq!(a[(1, 0)], 3.0); +/// assert_eq!(a.transpose(), Matrix::from_rows(&[[1.0, 3.0], [2.0, 4.0]])); +/// ``` +#[derive(Clone, PartialEq, Eq, Hash, Default)] +pub struct Matrix { + /// The number of rows. + rows: usize, + /// The number of columns. + cols: usize, + /// The elements, column after column. + data: Vec, +} + +/// The number of elements of a `rows × cols` matrix, which a release build would otherwise let +/// wrap around and leave a matrix of nonempty dimensions with no elements. +/// +/// # Panics +/// +/// Panics if it overflows `usize`. +#[track_caller] +fn size(rows: usize, cols: usize) -> usize { + rows.checked_mul(cols) + .expect("the matrix has more elements than a vector can hold") +} + +impl Matrix { + /// The matrix of `rows × cols` elements given column after column. + /// + /// # Panics + /// + /// Panics if `data` doesn't hold `rows · cols` elements. + #[track_caller] + pub fn from_column_major(rows: usize, cols: usize, data: Vec) -> Self { + assert_eq!( + Some(data.len()), + rows.checked_mul(cols), + "the data doesn't hold rows × cols elements" + ); + Matrix { rows, cols, data } + } + + /// The matrix of `rows × cols` elements given row after row. + /// + /// # Panics + /// + /// Panics if `data` doesn't hold `rows · cols` elements. + #[track_caller] + pub fn from_row_major(rows: usize, cols: usize, data: Vec) -> Self { + Matrix::from_column_major(cols, rows, data).into_transpose() + } + + /// The matrix whose element in row `i` and column `j` is `f(i, j)`, computed column after + /// column. + /// + /// # Panics + /// + /// Panics if `rows · cols` overflows `usize`. + #[track_caller] + pub fn from_fn(rows: usize, cols: usize, mut f: impl FnMut(usize, usize) -> T) -> Self { + let mut data = Vec::with_capacity(size(rows, cols)); + for j in 0..cols { + // A column at a time, from a range whose length is known, so that the vector is not + // checked for room at each element. + data.extend((0..rows).map(|i| f(i, j))); + } + Matrix { rows, cols, data } + } + + /// The number of rows. + #[inline] + pub fn rows(&self) -> usize { + self.rows + } + + /// The number of columns. + #[inline] + pub fn cols(&self) -> usize { + self.cols + } + + /// `(rows, cols)`. + #[inline] + pub fn shape(&self) -> (usize, usize) { + (self.rows, self.cols) + } + + /// Whether it has as many rows as columns. + #[inline] + pub fn is_square(&self) -> bool { + self.rows == self.cols + } + + /// Whether it has no elements. + #[inline] + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + /// The element in row `i` and column `j`, or `None` outside the matrix. + #[inline] + pub fn get(&self, i: usize, j: usize) -> Option<&T> { + if i < self.rows && j < self.cols { + Some(&self.data[i + j * self.rows]) + } else { + None + } + } + + /// The element in row `i` and column `j`, to change, or `None` outside the matrix. + #[inline] + pub fn get_mut(&mut self, i: usize, j: usize) -> Option<&mut T> { + if i < self.rows && j < self.cols { + Some(&mut self.data[i + j * self.rows]) + } else { + None + } + } + + /// Column `j`. + /// + /// # Panics + /// + /// Panics if `j` is not below the number of columns. + #[inline] + #[track_caller] + pub fn column(&self, j: usize) -> &[T] { + assert!(j < self.cols, "the column is outside the matrix"); + &self.data[j * self.rows..(j + 1) * self.rows] + } + + /// Column `j`, to change. + /// + /// # Panics + /// + /// Panics if `j` is not below the number of columns. + #[inline] + #[track_caller] + pub fn column_mut(&mut self, j: usize) -> &mut [T] { + assert!(j < self.cols, "the column is outside the matrix"); + &mut self.data[j * self.rows..(j + 1) * self.rows] + } + + /// The elements of row `i`, from the left. + /// + /// # Panics + /// + /// Panics if `i` is not below the number of rows. + #[track_caller] + pub fn row(&self, i: usize) -> impl ExactSizeIterator + '_ { + assert!(i < self.rows, "the row is outside the matrix"); + (0..self.cols).map(move |j| &self.data[i + j * self.rows]) + } + + /// The elements, column after column. + #[inline] + pub fn as_slice(&self) -> &[T] { + &self.data + } + + /// The elements, column after column, to change. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + &mut self.data + } + + /// The vector of the elements, column after column. + #[inline] + pub fn into_vec(self) -> Vec { + self.data + } + + /// The matrix of `f` of each element. + pub fn map(&self, f: impl FnMut(&T) -> U) -> Matrix { + Matrix { + rows: self.rows, + cols: self.cols, + data: self.data.iter().map(f).collect(), + } + } + + /// The transpose, moving the elements. + fn into_transpose(self) -> Self { + let (rows, cols) = (self.rows, self.cols); + if rows <= 1 || cols <= 1 { + return Matrix { + rows: cols, + cols: rows, + data: self.data, + }; + } + let mut slots: Vec> = self.data.into_iter().map(Some).collect(); + Matrix::from_fn(cols, rows, |i, j| slots[j + i * rows].take().unwrap()) + } +} + +impl Matrix { + /// The matrix with the given rows. + /// + /// # Panics + /// + /// Panics if the rows are not all of the same length. + #[track_caller] + pub fn from_rows>(rows: &[R]) -> Self { + let cols = rows.first().map_or(0, |r| r.as_ref().len()); + assert!( + rows.iter().all(|r| r.as_ref().len() == cols), + "the rows are not all of the same length" + ); + Matrix::from_fn(rows.len(), cols, |i, j| rows[i].as_ref()[j].clone()) + } + + /// The matrix with the given columns. + /// + /// # Panics + /// + /// Panics if the columns are not all of the same length. + #[track_caller] + pub fn from_columns>(columns: &[C]) -> Self { + let rows = columns.first().map_or(0, |c| c.as_ref().len()); + assert!( + columns.iter().all(|c| c.as_ref().len() == rows), + "the columns are not all of the same length" + ); + let data = columns + .iter() + .flat_map(|c| c.as_ref().iter().cloned()) + .collect(); + Matrix { + rows, + cols: columns.len(), + data, + } + } + + /// The elements of the main diagonal, from the top left. + pub fn diagonal(&self) -> Vec { + (0..self.rows.min(self.cols)) + .map(|i| self.data[i + i * self.rows].clone()) + .collect() + } +} + +impl Matrix { + /// The `rows × cols` matrix of zeros. + /// + /// # Panics + /// + /// Panics if `rows · cols` overflows `usize`. + #[track_caller] + pub fn zeros(rows: usize, cols: usize) -> Self { + // Filled at once, rather than element by element: for floats the allocator's zeroed + // pages, which took 1.4 µs for `256 × 256` against 69 by `from_fn`. + Matrix { + rows, + cols, + data: vec![T::zero(); size(rows, cols)], + } + } + + /// The `n × n` identity matrix. + pub fn identity(n: usize) -> Self { + Matrix::from_diagonal(&vec![T::one(); n]) + } + + /// The square matrix with `diagonal` on its main diagonal and zeros elsewhere. + pub fn from_diagonal(diagonal: &[T]) -> Self { + let n = diagonal.len(); + let mut a = Matrix::zeros(n, n); + for (i, d) in diagonal.iter().enumerate() { + a.data[i + i * n] = d.clone(); + } + a + } +} + +impl Index<(usize, usize)> for Matrix { + type Output = T; + + /// The element in row `i` and column `j`. + /// + /// # Panics + /// + /// Panics outside the matrix. + #[inline] + #[track_caller] + fn index(&self, (i, j): (usize, usize)) -> &T { + assert!( + i < self.rows && j < self.cols, + "the index is outside the matrix" + ); + &self.data[i + j * self.rows] + } +} + +impl IndexMut<(usize, usize)> for Matrix { + /// The element in row `i` and column `j`, to change. + /// + /// # Panics + /// + /// Panics outside the matrix. + #[inline] + #[track_caller] + fn index_mut(&mut self, (i, j): (usize, usize)) -> &mut T { + assert!( + i < self.rows && j < self.cols, + "the index is outside the matrix" + ); + &mut self.data[i + j * self.rows] + } +} + +impl fmt::Display for Matrix { + /// Writes the rows on lines of their own, each element as `Display` writes it with the + /// formatter's precision, right-aligned in its column, the columns two spaces apart. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let text: Vec = self + .data + .iter() + .map(|x| match f.precision() { + Some(p) => format!("{x:.p$}"), + None => format!("{x}"), + }) + .collect(); + let widths: Vec = (0..self.cols) + .map(|j| { + let column = &text[j * self.rows..(j + 1) * self.rows]; + column.iter().map(|s| s.chars().count()).max().unwrap_or(0) + }) + .collect(); + for i in 0..self.rows { + if i > 0 { + writeln!(f)?; + } + for (j, width) in widths.iter().enumerate() { + let gap = if j == 0 { "" } else { " " }; + write!(f, "{gap}{:>width$}", text[i + j * self.rows])?; + } + } + Ok(()) + } +} + +impl fmt::Debug for Matrix { + /// Writes the rows as lists: `[[1, 2], [3, 4]]`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list() + .entries((0..self.rows).map(|i| DebugRow(self, i))) + .finish() + } +} + +/// A row of a matrix for [`fmt::Debug`]. +struct DebugRow<'a, T>(&'a Matrix, usize); + +impl fmt::Debug for DebugRow<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.row(self.1)).finish() + } +} diff --git a/lib/src/math/matrix/ops.rs b/lib/src/math/matrix/ops.rs new file mode 100644 index 0000000..928226d --- /dev/null +++ b/lib/src/math/matrix/ops.rs @@ -0,0 +1,280 @@ +//! The operators of matrices. + +use super::element::sealed::Element as Ring; +use super::{Element, Matrix, Scalar}; +use crate::math::complex::Complex; +use crate::math::numbers::Number; +use core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign}; + +/// Panics unless the two matrices have the same shape. +#[inline] +#[track_caller] +fn same_shape(a: &Matrix, b: &Matrix) { + if a.shape() != b.shape() { + different_shapes(); + } +} + +#[cold] +#[inline(never)] +#[track_caller] +fn different_shapes() -> ! { + panic!("the matrices are not of the same shape") +} + +/// `a · b`. +#[track_caller] +fn product(a: &Matrix, b: &Matrix) -> Matrix { + assert!( + a.cols == b.rows, + "the columns of the left matrix are not as many as the rows of the right one" + ); + let mut c = Matrix::zeros(a.rows, b.cols); + T::product(&mut c.data, &a.data, &b.data, a.rows, a.cols, b.cols); + c +} + +/// An elementwise operator between two matrices, owned or borrowed, and its assignment. +macro_rules! elementwise { + ($op:ident, $method:ident, $assign:ident, $assign_method:ident, $f:ident, $f_assign:ident) => { + impl $op<&Matrix> for &Matrix { + type Output = Matrix; + #[track_caller] + fn $method(self, rhs: &Matrix) -> Matrix { + same_shape(self, rhs); + Matrix { + rows: self.rows, + cols: self.cols, + data: self + .data + .iter() + .zip(&rhs.data) + .map(|(a, b)| T::$f(a, b)) + .collect(), + } + } + } + impl $op<&Matrix> for Matrix { + type Output = Matrix; + #[track_caller] + fn $method(mut self, rhs: &Matrix) -> Matrix { + self.$assign_method(rhs); + self + } + } + impl $op> for Matrix { + type Output = Matrix; + #[track_caller] + fn $method(mut self, rhs: Matrix) -> Matrix { + self.$assign_method(&rhs); + self + } + } + impl $op> for &Matrix { + type Output = Matrix; + #[track_caller] + fn $method(self, mut rhs: Matrix) -> Matrix { + same_shape(self, &rhs); + for (b, a) in rhs.data.iter_mut().zip(&self.data) { + *b = T::$f(a, b); + } + rhs + } + } + impl $assign<&Matrix> for Matrix { + #[track_caller] + fn $assign_method(&mut self, rhs: &Matrix) { + same_shape(self, rhs); + for (a, b) in self.data.iter_mut().zip(&rhs.data) { + T::$f_assign(a, b); + } + } + } + impl $assign> for Matrix { + #[track_caller] + fn $assign_method(&mut self, rhs: Matrix) { + self.$assign_method(&rhs); + } + } + }; +} + +elementwise!(Add, add, AddAssign, add_assign, add, add_assign); +elementwise!(Sub, sub, SubAssign, sub_assign, sub, sub_assign); + +impl Neg for &Matrix { + type Output = Matrix; + fn neg(self) -> Matrix { + self.map(T::neg) + } +} + +impl Neg for Matrix { + type Output = Matrix; + fn neg(mut self) -> Matrix { + for x in &mut self.data { + *x = T::neg(x); + } + self + } +} + +impl Mul<&Matrix> for &Matrix { + type Output = Matrix; + /// The matrix product. + /// + /// # Panics + /// + /// Panics if the columns of `self` are not as many as the rows of `rhs`. + #[track_caller] + fn mul(self, rhs: &Matrix) -> Matrix { + product(self, rhs) + } +} + +impl Mul> for &Matrix { + type Output = Matrix; + #[track_caller] + fn mul(self, rhs: Matrix) -> Matrix { + product(self, &rhs) + } +} + +impl Mul<&Matrix> for Matrix { + type Output = Matrix; + #[track_caller] + fn mul(self, rhs: &Matrix) -> Matrix { + product(&self, rhs) + } +} + +impl Mul> for Matrix { + type Output = Matrix; + #[track_caller] + fn mul(self, rhs: Matrix) -> Matrix { + product(&self, &rhs) + } +} + +impl MulAssign<&Matrix> for Matrix { + #[track_caller] + fn mul_assign(&mut self, rhs: &Matrix) { + *self = product(self, rhs); + } +} + +impl MulAssign> for Matrix { + #[track_caller] + fn mul_assign(&mut self, rhs: Matrix) { + *self = product(self, &rhs); + } +} + +impl Mul<&T> for &Matrix { + type Output = Matrix; + /// The product of every element by `rhs`. + fn mul(self, rhs: &T) -> Matrix { + self.map(|x| T::mul(x, rhs)) + } +} + +impl Mul for &Matrix { + type Output = Matrix; + fn mul(self, rhs: T) -> Matrix { + self * &rhs + } +} + +impl Mul<&T> for Matrix { + type Output = Matrix; + fn mul(mut self, rhs: &T) -> Matrix { + self *= rhs; + self + } +} + +impl Mul for Matrix { + type Output = Matrix; + fn mul(mut self, rhs: T) -> Matrix { + self *= &rhs; + self + } +} + +impl MulAssign<&T> for Matrix { + fn mul_assign(&mut self, rhs: &T) { + for x in &mut self.data { + T::mul_assign(x, rhs); + } + } +} + +impl MulAssign for Matrix { + fn mul_assign(&mut self, rhs: T) { + *self *= &rhs; + } +} + +impl Div for &Matrix { + type Output = Matrix; + /// The quotient of every element by `rhs`. + fn div(self, rhs: T) -> Matrix { + self.map(|&x| x / rhs) + } +} + +impl Div for Matrix { + type Output = Matrix; + fn div(mut self, rhs: T) -> Matrix { + self /= rhs; + self + } +} + +impl DivAssign for Matrix { + fn div_assign(&mut self, rhs: T) { + for x in &mut self.data { + *x /= rhs; + } + } +} + +/// A scalar on the left of a matrix, which the orphan rule leaves to one implementation for each +/// type of element. +macro_rules! left { + ($t:ty) => { + impl Mul<&Matrix<$t>> for &$t { + type Output = Matrix<$t>; + fn mul(self, rhs: &Matrix<$t>) -> Matrix<$t> { + rhs.map(|x| <$t as Ring>::mul(self, x)) + } + } + impl Mul> for &$t { + type Output = Matrix<$t>; + fn mul(self, mut rhs: Matrix<$t>) -> Matrix<$t> { + for x in &mut rhs.data { + *x = <$t as Ring>::mul(self, x); + } + rhs + } + } + impl Mul<&Matrix<$t>> for $t { + type Output = Matrix<$t>; + fn mul(self, rhs: &Matrix<$t>) -> Matrix<$t> { + &self * rhs + } + } + impl Mul> for $t { + type Output = Matrix<$t>; + fn mul(self, rhs: Matrix<$t>) -> Matrix<$t> { + &self * rhs + } + } + }; +} + +left!(f32); +left!(f64); +left!(Complex); +left!(Complex); +left!(Number); diff --git a/lib/src/math/matrix/properties.rs b/lib/src/math/matrix/properties.rs new file mode 100644 index 0000000..d3110eb --- /dev/null +++ b/lib/src/math/matrix/properties.rs @@ -0,0 +1,235 @@ +//! Norms, condition numbers, ranks, null spaces and characteristic polynomials. + +use super::householder::norm2; +use super::schur::hessenberg; +use super::svd::{svd, svd_scaled}; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; + +/// The power of two `k` by which the finite elements `data` are scaled, `2^k·A`, as LAPACK's +/// drivers scale, when the largest of them is so small or so large that products of elements +/// could underflow or overflow, outside `[√MIN/ε, ε/√MIN]`; 0 inside it, which the largest part +/// decides alone unless it is near an end. A complex element can have finite parts and an +/// absolute value that overflows: its largest part, within a factor √2, stands for it then. +pub(crate) fn range_exponent(data: &[T]) -> i32 { + let zero = T::Real::ZERO; + let small = T::Real::MIN_POSITIVE.sqrt() / T::Real::EPSILON; + let big = data + .iter() + .fold(zero, |s, x| s.max(x.re().abs().max(x.im().abs()))); + if big >= small && big * T::Real::TWO.sqrt() <= T::Real::ONE / small { + return 0; + } + let max = data.iter().fold(zero, |s, x| s.max(x.abs())); + let log2 = |x: T::Real| x.ln().to_f64() / core::f64::consts::LN_2; + let top = if max.is_finite() { + log2(max) + } else { + log2(big) + 0.5 + }; + if max > zero && max < small { + (log2(small) - top).ceil() as i32 + } else if max > T::Real::ONE / small { + (-log2(small) - top).floor() as i32 + } else { + 0 + } +} + +/// The larger of `m` and `x`, or a NaN when either is one: `Float::max` ignores NaNs, which +/// would let a norm of a matrix holding one be finite. +fn max_nan(m: R, x: R) -> R { + if m.is_nan() || x.is_nan() { + R::NAN + } else { + m.max(x) + } +} + +/// A matrix norm, for [`Matrix::norm`] and [`Matrix::cond`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Norm { + /// The largest sum of the absolute values in a column. + One, + /// The spectral norm, the largest singular value. + Two, + /// The largest sum of the absolute values in a row. + Infinity, + /// Frobenius's, the square root of the sum of the squares of the absolute values. + Frobenius, + /// The largest absolute value of an element; not submultiplicative. + Max, + /// The nuclear norm, the sum of the singular values. + Nuclear, +} + +impl Matrix { + /// The norm `‖A‖` of the kind given: see [`Norm`]. The 2-norm and the nuclear norm take the + /// singular values; the others one pass over the elements, Frobenius's scaled so that the + /// squares neither overflow nor underflow. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::{Matrix, Norm}; + /// + /// let a: Matrix = Matrix::from_rows(&[[1.0, -2.0], [-3.0, 4.0]]); + /// assert_eq!(a.norm(Norm::One), 6.0); + /// assert_eq!(a.norm(Norm::Infinity), 7.0); + /// assert_eq!(a.norm(Norm::Max), 4.0); + /// assert!((a.norm(Norm::Frobenius) - 30f64.sqrt()).abs() < 1e-15); + /// ``` + pub fn norm(&self, norm: Norm) -> T::Real { + let zero = T::Real::ZERO; + let (m, n) = self.shape(); + match norm { + Norm::One => (0..n) + .map(|j| self.column(j).iter().map(|x| x.abs()).sum::()) + .fold(zero, max_nan), + Norm::Infinity => { + let mut rows = vec![zero; m]; + for j in 0..n { + for (r, x) in rows.iter_mut().zip(self.column(j)) { + *r += x.abs(); + } + } + rows.into_iter().fold(zero, max_nan) + } + Norm::Frobenius => norm2(&self.data), + Norm::Max => self.data.iter().map(|x| x.abs()).fold(zero, max_nan), + Norm::Two => svd(self, false).0.first().copied().unwrap_or(zero), + Norm::Nuclear => svd(self, false).0.into_iter().sum(), + } + } + + /// The condition number `κ(A) = ‖A‖·‖A⁻¹‖` in the norm given, which bounds how much a + /// relative change of `A` or of `b` can change the solution of `A·x = b`: infinite for a + /// singular matrix. In the 2-norm it is the ratio of the largest singular value to the + /// smallest, for a matrix of any shape; in the other norms it takes the inverse. + /// + /// # Panics + /// + /// Panics if the norm is not [`Norm::Two`] and the matrix is not square. + #[track_caller] + pub fn cond(&self, norm: Norm) -> T::Real { + let zero = T::Real::ZERO; + match norm { + // From the singular values of the matrix scaled into range, whose ratios are the + // same, so that neither overflows where the matrix's own would. + Norm::Two => { + let s = svd_scaled(self, false).0; + match (s.first(), s.last()) { + (Some(&max), Some(&min)) if max.is_nan() || min.is_nan() => T::Real::NAN, + (Some(&max), Some(&min)) if min > zero => max / min, + (Some(_), Some(_)) => T::Real::INFINITY, + _ => zero, + } + } + Norm::Nuclear => { + assert!(self.is_square(), "the matrix is not square"); + let s = svd_scaled(self, false).0; + if s.last().is_some_and(|&x| x == zero) { + return T::Real::INFINITY; + } + // `(Σσ/σ₁)·(Σσ₁/σ)`, each sum at most `n` times its largest term. + let top = s.first().copied().unwrap_or(T::Real::ONE); + let sum: T::Real = s.iter().map(|&x| x / top).sum(); + let inv: T::Real = s.iter().map(|&x| top / x).sum(); + sum * inv + } + _ => { + assert!(self.is_square(), "the matrix is not square"); + match self.inverse() { + Some(inv) => self.norm(norm) * inv.norm(norm), + None => T::Real::INFINITY, + } + } + } + } + + /// The rank: the number of singular values above `max(m, n)·ε·σ₁`, `ε` the unit roundoff + /// of the type and `σ₁` the largest singular value, as NumPy's `matrix_rank`. + pub fn rank(&self) -> usize { + let s = svd_scaled(self, false).0; + let tol = self.rank_tolerance(&s); + s.iter().filter(|&&x| x > tol).count() + } + + /// An orthonormal basis of the null space `{x : A·x = 0}`, as the columns of an + /// `n × (n - rank)` matrix: the right singular vectors of the singular values at or below + /// the tolerance of [`rank`](Matrix::rank). + pub fn null_space(&self) -> Matrix { + let (m, n) = self.shape(); + // With fewer rows than columns, rows of zeros make all `n` right singular vectors. + let a = if m < n { + Matrix::from_fn(n, n, |i, j| if i < m { self[(i, j)] } else { T::zero() }) + } else { + self.clone() + }; + let (s, uv, _) = svd_scaled(&a, true); + let v = uv.map(|(_, v)| v).unwrap_or_default(); + let tol = self.rank_tolerance(&s); + let rank = s.iter().filter(|&&x| x > tol).count(); + Matrix::from_fn(n, n - rank, |i, j| v[(i, rank + j)]) + } + + /// The tolerance below which a singular value counts as zero. + fn rank_tolerance(&self, s: &[T::Real]) -> T::Real { + let (m, n) = self.shape(); + let max = s.first().copied().unwrap_or(T::Real::ZERO); + T::Real::from_f64(m.max(n) as f64) * T::Real::EPSILON * max + } + + /// The coefficients of the characteristic polynomial `det(x·I - A)`, from `xⁿ` down: `1`, + /// then `-tr(A)`, …, `(-1)ⁿ·det(A)`. From the Hessenberg form, by La Budde's recurrence on + /// its leading blocks, about `n³` operations. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + /// + /// # Examples + /// + /// ``` + /// use a::math::matrix::Matrix; + /// + /// let a: Matrix = Matrix::from_rows(&[[2.0, 1.0], [1.0, 2.0]]); + /// // (x - 1)(x - 3) + /// assert_eq!(a.characteristic_polynomial(), [1.0, -4.0, 3.0]); + /// ``` + #[track_caller] + pub fn characteristic_polynomial(&self) -> Vec { + assert!(self.is_square(), "the matrix is not square"); + let n = self.rows(); + let mut h = self.clone(); + hessenberg(&mut h, false); + let at = |i: usize, j: usize| h.data[i + j * n]; + // `p[s]`: the polynomial of the leading `s × s` block, from the constant term up. + let mut p: Vec> = vec![vec![T::one()]]; + for k in 0..n { + // `(x - h_kk)·p_k`. + let prev = &p[k]; + let mut next = vec![T::zero(); k + 2]; + for (i, &c) in prev.iter().enumerate() { + next[i + 1] += c; + next[i] -= at(k, k) * c; + } + // `- Σ h_{k-m,k}·(h_{k,k-1}·…·h_{k-m+1,k-m})·p_{k-m}`. + let mut product = T::one(); + for m in 1..=k { + product *= at(k - m + 1, k - m); + let f = at(k - m, k) * product; + if f == T::zero() { + continue; + } + for (i, &c) in p[k - m].iter().enumerate() { + next[i] -= f * c; + } + } + p.push(next); + } + let mut c = p.pop().unwrap_or_default(); + c.reverse(); + c + } +} diff --git a/lib/src/math/matrix/qr.rs b/lib/src/math/matrix/qr.rs new file mode 100644 index 0000000..441400b --- /dev/null +++ b/lib/src/math/matrix/qr.rs @@ -0,0 +1,1055 @@ +//! The QR decomposition by Householder reflections. +//! +//! # The decomposition +//! +//! A matrix of at most [`SMALL`] elements, and a complex one of at most 64 columns and 32768 +//! elements, is decomposed one reflection at a time ([`unblocked`], LAPACK's `geqr2`): each column's +//! reflection taken from it, then applied to the columns right of it four at a time, each read for +//! its products with the reflection's vector and again, from the cache, for the update +//! ([`kernels::reflect`]). Larger matrices go by blocks of `nb` columns ([`block_width`]): a panel +//! of `nb` columns is decomposed and gives the triangular factor `T` of its block of reflections, +//! `H_0·…·H_nb-1 = I - V·T·V*` (the compact WY form), and the rest of the matrix takes the block +//! by three products, `C -= V·(T*·(V*·C))`, with `V` read where the panel left it, its unit +//! triangle written over `R` for the time ([`with_v`]). +//! +//! A panel is halved down to leaves of [`LEAF`] columns as LAPACK's `geqrt3` does ([`panel`]): +//! the left half, its block applied to the right half by products of blocks, the right half, and +//! the halves' factors joined, `T12 = -T1·V1*·V2·T2`. A leaf is decomposed one reflection at a time, +//! and each reflection's products with the columns left of it, which make its column of `T`, are +//! taken in the pass that applies it to those right of it. +//! +//! The block widens with the work of the update against that of the panel: 8 columns up to 384 +//! columns of `f64` (192 of `f32`, 96 complex), then 16, 24 and 32, measured with each width from +//! 8 to 64 on square matrices and ones of twice and eight times as many rows as columns. At +//! `1024 × 1024` 24 took 30.3 ms, 16 30.9, 8 32.1 and 64 32.3, and 32 as 24 in other runs; at +//! `256 × 256` 8 took 556 µs and 32 610. A matrix of four times as many rows as columns takes at +//! least 16 columns from 512 rows, 48 from 2048 and 64 from 4096: the product `V*·C` of so tall +//! a block of 32 columns or fewer ran at 41 GFLOP/s against 54 for 64, and an `8192 × 1024` +//! decomposition took 337 ms with 64 against 372 with 32. +//! +//! The decomposition keeps the blocks' `T` with the reflections: forming `Q` and the least +//! squares reuse them. +//! +//! # `Q` +//! +//! `Q` is formed from the last block back, as LAPACK's `orgqr` ([`Qr::form_q`]): each block gives +//! its own columns directly, `[I; 0] - V·T·E1*` (`E1` the block's unit triangle), and updates +//! those right of it, whose rows of the block are still zeros, by three products. The columns +//! right of the last block are those of the identity, so that their product with `V*` is a copy +//! of it. The whole `Q` of a matrix of at least four times as many rows as columns is formed in +//! one block of all the reflections, whose `T` is built from `V*·V` ([`Qr::whole_t`]): most of +//! the work is the columns right of the reflections, and in one block it is the copy and one +//! product, half of the work of applying the blocks in turn: the whole `Q` of a `1024 × 128` +//! matrix took 6.4 ms, against 12.3 by blocks of 32 before and faer's 10.9. A decomposition +//! without blocks forms `Q` one reflection at a time. +//! +//! # Least squares +//! +//! `Q*·B` takes the blocks' `T` for more than [`FOURS`] columns of `B`, by products of blocks; +//! for up to four, each column takes four reflections at a time by the diagonal blocks of their +//! blocks' `T`, `w = T*·V*·y` in one pass over the column and `y -= V·w` in another +//! ([`Qr::reflect_fours`]), half the passes of one reflection at a time. `R·X = Q*·B` is solved +//! by columns of `R` for one column of `B` or an `R` of at most [`SUBSTITUTE`] columns +//! ([`back_substitute`]), and by the blocked triangular solve otherwise. +//! +//! # Speed +//! +//! Measured on an Apple M4 Max, one core, the median of interleaved runs while other processes +//! ran, against faer 0.24 (`Par::Seq`), nalgebra 0.35 and Accelerate (`dgeqrf`, `dorgqr`, +//! `dgels`, `dormqr` with `dtrtrs`; on the matrix unit); `f64`, µs unless ms. The largest varied +//! by up to 10 % from run to run, their ratios by less. `solve(b)` is of one column, given the +//! decomposition; `qr().solve(b)` includes it. nalgebra has no whole `Q` of a non-square matrix. +//! +//! | `f64` | ours | faer | nalgebra | Accelerate | +//! | --- | --- | --- | --- | --- | +//! | `qr()` 2 × 2 | 0.05 | 0.15 | 0.07 | 0.11 | +//! | `qr()` 8 × 8 | 0.22 | 0.50 | 0.35 | 0.48 | +//! | `qr()` 32 × 32 | 2.52 | 7.18 | 4.40 | 4.20 | +//! | `qr()` 128 × 128 | 82.2 | 147 | 176 | 109 | +//! | `qr()` 512 × 512 | 4.44 ms | 5.15 ms | 13.56 ms | 2.30 ms | +//! | `qr()` 1024 × 1024 | 32.47 ms | 35.42 ms | 102.36 ms | 12.25 ms | +//! | `qr()` 2048 × 1024 | 79.24 ms | 83.14 ms | 247.62 ms | 27.22 ms | +//! | `qr()` 8192 × 1024 | 365.6 ms | 381.5 ms | 1259.7 ms | 114.7 ms | +//! | `q()` 32 × 32 | 2.18 | 5.24 | 3.56 | 4.08 | +//! | `q()` 1024 × 1024 | 28.12 ms | 44.29 ms | 95.46 ms | 11.12 ms | +//! | `q()` 8192 × 1024 | 310.3 ms | 601.5 ms | 1189.7 ms | 106.3 ms | +//! | `q_full()` 1024 × 128 | 6.39 ms | 10.85 ms | - | 8.84 ms | +//! | `q_full()` 8192 × 1024 | 2.93 s | 4.72 s | - | 3.85 s | +//! | `qr().solve(b)` 32 × 32 | 2.94 | 8.60 | 5.77 | 6.82 | +//! | `qr().solve(b)` 1024 × 1024 | 30.16 ms | 32.87 ms | 96.37 ms | 12.76 ms | +//! | `qr().solve(b)` 8192 × 1024 | 347.4 ms | 369.9 ms | 1181.6 ms | 124.2 ms | +//! | `solve(b)` 16 × 16 | 0.23 | 0.30 | 0.56 | 0.83 | +//! | `solve(b)` 1024 × 1024 | 229 | 389 | 2.00 ms | 1.06 ms | +//! | `solve(b)` 8192 × 1024 | 1.87 ms | 2.63 ms | 3.95 ms | 6.22 ms | +//! +//! `f32` and complex matrices compare alike: `qr()` of `512 × 512` took 2.18 ms in `f32` against +//! faer's 2.63 and nalgebra's 7.36, and 15.9 ms in `Complex` against 19.3 and 55.1. + +mod kernels; + +use super::blas::{Op, Side, Uplo, gemm, trsm}; +use super::{Matrix, Scalar}; +use kernels::{axpy, axpys4, dots, dots4, reflect, reflector}; + +/// The QR decomposition `A = Q·R` of an `m × n` matrix by Householder reflections: `Q` of +/// `m × min(m, n)` with orthonormal columns (unitary for complex elements), `R` of +/// `min(m, n) × n` upper triangular with a real diagonal. From [`Matrix::qr`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a: Matrix = Matrix::from_rows(&[[3.0, 1.0], [4.0, 2.0]]); +/// let qr = a.qr(); +/// let (q, r) = (qr.q(), qr.r()); +/// assert!((&q * &r - &a).as_slice().iter().all(|x| x.abs() < 1e-15)); +/// assert!((&q.transpose() * &q - Matrix::identity(2)).as_slice().iter().all(|x| x.abs() < 1e-15)); +/// assert!((r[(0, 0)].abs() - 5.0).abs() < 1e-15); +/// ``` +#[derive(Clone, Debug)] +pub struct Qr { + /// `R` on and above the diagonal, the reflections' vectors below it. + qr: Matrix, + /// The reflections' factors. + taus: Vec, + /// The triangular factors of the blocks of reflections, `H_j0·…·H_j1-1 = I - V·T·V*` for + /// the block of columns `j0..j1`, whose `T` starts at `j0 · nb` with the leading dimension + /// `nb`; empty when the decomposition was not blocked. + ts: Vec, + /// The width `nb` of the blocks, 0 when the decomposition was not blocked. + nb: usize, +} + +/// The most elements of a matrix decomposed one reflection at a time. Beyond, blocks of 8 +/// columns were faster: `f64` matrices of `64 × 64` took 14.3 µs by them against 14.6, of +/// `256 × 32` 22.7 against 25.8, while `48 × 48` took 7.3 against 7.5 and `128 × 16` 3.6 against +/// 4.2; `f32` and complex ones of the same shapes broke even at about the same size, but complex +/// ones of few columns stayed faster one reflection at a time up to `512 × 64`. +const SMALL: usize = 4096; + +/// The columns of the leaves of a panel, decomposed one reflection at a time: 4 and 16 were as +/// fast as 8 within 5 % from `64 × 64` to `8192 × 1024`, 32 up to 10 % slower. +const LEAF: usize = 8; + +/// The most columns of `B` whose `Q*·B` goes a column at a time, four reflections at a time; +/// with more, products of blocks were faster: for 8 columns of 1024 rows 916 µs against 963. +const FOURS: usize = 4; + +/// The most columns of `R` solved for by columns for any number of columns of `B`: for 4 columns +/// of 128 rows 9.4 µs against 10.2 by the triangular solve, of 192 rows 22.7 against 20.5. +const SUBSTITUTE: usize = 128; + +/// The fewest rows of a matrix whose whole `Q` is formed in one block: `16 × 4` took 0.41 µs so +/// against 0.25 one reflection at a time, `64 × 16` 6.5 against 8.8. +const ONE_BLOCK: usize = 48; + +/// The reflections whose triangular factor [`Qr::whole_t`] builds from `V*·V` directly, before +/// joining them by products of blocks. +const LEAF_T: usize = 64; + +/// The width of the blocks of the decomposition of an `m × n` matrix, 0 for none: see the +/// module's documentation. +fn block_width(m: usize, n: usize) -> usize { + let size = m * n; + if size <= SMALL || (T::IS_COMPLEX && n <= 64 && size <= 8 * SMALL) { + return 0; + } + // A complex element takes four times the work of a real one, and an `f32` half the time of + // an `f64`, at the same speed of the products of blocks. + let e = n * if T::IS_COMPLEX { + 4 + } else if core::mem::size_of::() == 4 { + 2 + } else { + 1 + }; + let nb = match e { + 0..=384 => 8, + 385..=640 => 16, + 641..=896 => 24, + _ => 32, + }; + if m >= 4 * n { + let tall = match m { + 0..=511 => 8, + 512..=2047 => 16, + 2048..=4095 => 48, + _ => 64, + }; + nb.max(tall) + } else { + nb + } +} + +impl Matrix { + /// The QR decomposition `A = Q·R`: see [`Qr`]. About `2mn² - 2n³/3` operations for + /// `m ≥ n`, most of them, for large matrices, in products of blocks of reflections. + pub fn qr(&self) -> Qr { + let (m, n) = self.shape(); + let mut qr = self.clone(); + let k = m.min(n); + let mut taus = vec![T::zero(); k]; + let nb = block_width::(m, n); + if nb == 0 { + unblocked(&mut qr.data, m, m, n, &mut taus, None, &mut []); + return Qr { + qr, + taus, + ts: Vec::new(), + nb: 0, + }; + } + let mut ts = vec![T::zero(); nb * k]; + let mut work = Work::new(nb, n); + let mut j0 = 0; + while j0 < k { + let kb = nb.min(k - j0); + let j1 = j0 + kb; + let (left, right) = qr.data.split_at_mut(j1 * m); + let block = &mut left[j0 + j0 * m..]; + panel( + block, + m, + m - j0, + kb, + &mut taus[j0..j1], + &mut ts[j0 * nb..], + nb, + &mut work, + ); + if j1 < n { + let Work { save, w, .. } = &mut work; + let t = &ts[j0 * nb..]; + with_v(block, m, kb, save, |v| { + apply_block(v, m, m - j0, kb, t, nb, &mut right[j0..], m, n - j1, w); + }); + } + j0 = j1; + } + Qr { qr, taus, ts, nb } + } +} + +impl Qr { + /// `Q`, `m × min(m, n)`, with orthonormal columns. + pub fn q(&self) -> Matrix { + self.form_q(self.taus.len()) + } + + /// The whole `m × m` unitary `Q`, whose first `min(m, n)` columns are [`q`](Qr::q). + pub fn q_full(&self) -> Matrix { + self.form_q(self.qr.rows()) + } + + /// `R`, `min(m, n) × n`, upper triangular. + pub fn r(&self) -> Matrix { + let (m, n) = self.qr.shape(); + let k = m.min(n); + let mut r = vec![T::zero(); k * n]; + for (j, col) in r.chunks_exact_mut(k.max(1)).take(n).enumerate() { + let h = k.min(j + 1); + col[..h].copy_from_slice(&self.qr.data[j * m..j * m + h]); + } + Matrix::from_column_major(k, n, r) + } + + /// The `X` of the least squares, which makes `‖A·X - B‖` the smallest, for `m ≥ n`: the + /// solution of `R·X = Q*·B`, or `None` when a diagonal element of `R` is zero, the + /// columns of `A` dependent. For a square matrix it solves `A·X = B`. + /// + /// # Panics + /// + /// Panics if the matrix has fewer rows than columns, or `B` not as many rows as it. + #[track_caller] + pub fn solve(&self, b: &Matrix) -> Option> { + let (m, n) = self.qr.shape(); + assert!(m >= n, "the matrix has fewer rows than columns"); + assert_eq!(b.rows(), m, "the right-hand side has not as many rows"); + if (0..n).any(|i| self.qr[(i, i)] == T::zero()) { + return None; + } + let cols = b.cols(); + let mut y = b.clone(); + let a = &self.qr.data; + if cols == 0 { + } else if self.nb == 0 { + // One reflection at a time, `y ← H_j*·y`. + for (j, &tau) in self.taus.iter().enumerate() { + if tau != T::zero() { + let v = &a[j * m + j + 1..(j + 1) * m]; + reflect(v, tau.conj(), &mut y.data[j..], m, cols); + } + } + } else if cols <= FOURS { + for y in y.data.chunks_exact_mut(m) { + self.reflect_fours(y); + } + } else { + self.apply_blocks(&mut y.data, cols); + } + // The first `n` rows of `y`, moved up in place. + for j in 1..cols { + y.data.copy_within(j * m..j * m + n, j * n); + } + y.data.truncate(n * cols); + let mut x = Matrix::from_column_major(n, cols, y.data); + if n == 0 { + } else if cols == 1 || n <= SUBSTITUTE { + for x in x.data.chunks_exact_mut(n) { + back_substitute(a, m, n, x); + } + } else { + trsm( + Side::Left, + Uplo::Upper, + Op::N, + false, + n, + cols, + a, + m, + &mut x.data, + n, + ); + } + Some(x) + } + + /// `Y ← Q*·Y` for the `m × cols` block `y` (leading dimension `m`), a block of reflections at + /// a time: `W = V*·Y` by the block's unit triangle and the rows below it, `W ← T*·W`, and + /// `Y -= V·W`. + fn apply_blocks(&self, y: &mut [T], cols: usize) { + let m = self.qr.rows(); + let (k, nb) = (self.taus.len(), self.nb); + let a = &self.qr.data; + let mut e1 = vec![T::zero(); nb * nb]; + let mut w = vec![T::zero(); 2 * nb * cols]; + let mut j0 = 0; + while j0 < k { + let kb = nb.min(k - j0); + let j1 = j0 + kb; + unit_lower(&a[j0 + j0 * m..], m, kb, &mut e1); + let v2 = &a[j1 + j0 * m..]; + let t = &self.ts[j0 * nb..]; + let (w1, w2) = w.split_at_mut(kb * cols); + let w2 = &mut w2[..kb * cols]; + gemm( + kb, + cols, + kb, + T::one(), + &e1, + kb, + Op::C, + &y[j0..], + m, + Op::N, + T::zero(), + w1, + kb, + ); + gemm( + kb, + cols, + m - j1, + T::one(), + v2, + m, + Op::C, + &y[j1..], + m, + Op::N, + T::one(), + w1, + kb, + ); + gemm( + kb, + cols, + kb, + T::one(), + t, + nb, + Op::C, + w1, + kb, + Op::N, + T::zero(), + w2, + kb, + ); + gemm( + kb, + cols, + kb, + -T::one(), + &e1, + kb, + Op::N, + w2, + kb, + Op::N, + T::one(), + &mut y[j0..], + m, + ); + gemm( + m - j1, + cols, + kb, + -T::one(), + v2, + m, + Op::N, + w2, + kb, + Op::N, + T::one(), + &mut y[j1..], + m, + ); + j0 = j1; + } + } + + /// `y ← Q*·y` for a column `y`, four reflections at a time by the diagonal blocks of their + /// blocks' `T`, `I - V·T·V*` of four: `w = T*·V*·y` in one pass over `y` and `y -= V·w` in + /// another, the rows below the four's triangle by kernels; the reflections left over one at a + /// time. + fn reflect_fours(&self, y: &mut [T]) { + let m = self.qr.rows(); + let (k, nb) = (self.taus.len(), self.nb); + let a = &self.qr.data; + let mut j0 = 0; + while j0 < k { + let j1 = (j0 + nb).min(k); + let t = &self.ts[j0 * nb..]; + let mut j = j0; + while j + 4 <= j1 && j + 4 <= m { + // `V`'s rows `j..j + 4`, unit lower triangular, and the rows below them. + let at = |r: usize, c: usize| a[j + r + (j + c) * m]; + let col = |c: usize| &a[(j + c) * m + j + 4..(j + c + 1) * m]; + let cols = [col(0), col(1), col(2), col(3)]; + let (top, bottom) = y[j..].split_at_mut(4); + let mut w = dots4(bottom, cols); + for (c, w) in w.iter_mut().enumerate() { + *w += top[c]; + for (r, &y) in top.iter().enumerate().skip(c + 1) { + *w += at(r, c).conj() * y; + } + } + // `w ← T*·w` for the four's block of `T`, upper triangular. + let d = j - j0; + let tt = |r: usize, c: usize| t[d + r + (d + c) * nb]; + for c in (0..4).rev() { + let mut x = T::zero(); + for (r, &w) in w.iter().enumerate().take(c + 1) { + x += tt(r, c).conj() * w; + } + w[c] = x; + } + for r in 0..4 { + let mut x = w[r]; + for (c, &w) in w.iter().enumerate().take(r) { + x += at(r, c) * w; + } + top[r] -= x; + } + axpys4(bottom, cols, w); + j += 4; + } + for j in j..j1 { + let tau = self.taus[j]; + if tau != T::zero() { + reflect( + &a[j * m + j + 1..(j + 1) * m], + tau.conj(), + &mut y[j..], + m, + 1, + ); + } + } + j0 = j1; + } + } + + /// The triangular factor of all the `k` reflections in one block, `k × k`: `S = V*·V` by + /// products of blocks, the factors of blocks of up to [`LEAF_T`] reflections from it as + /// LAPACK's `larft` builds them, `T[0..j, j] = -τ_j·T[0..j, 0..j]·S[0..j, j]`, and those + /// joined from the left by products of blocks, `T12 = -T1·S12·T2`. + fn whole_t(&self) -> Vec { + let m = self.qr.rows(); + let k = self.taus.len(); + let a = &self.qr.data; + let mut s = vec![T::zero(); k * k]; + { + let mut e = vec![T::zero(); k * k]; + unit_lower(a, m, k, &mut e); + gemm( + k, + k, + k, + T::one(), + &e, + k, + Op::C, + &e, + k, + Op::N, + T::zero(), + &mut s, + k, + ); + gemm( + k, + k, + m - k, + T::one(), + &a[k..], + m, + Op::C, + &a[k..], + m, + Op::N, + T::one(), + &mut s, + k, + ); + } + let mut t = vec![T::zero(); k * k]; + let mut y = vec![T::zero(); k * LEAF_T]; + let mut z = vec![T::zero(); k * LEAF_T]; + let mut j0 = 0; + while j0 < k { + let j1 = (j0 + LEAF_T).min(k); + let kb = j1 - j0; + for j in j0..j1 { + let tau = self.taus[j]; + t[j + j * k] = tau; + if tau == T::zero() { + continue; + } + for i in j0..j { + let mut x = T::zero(); + for p in i..j { + x += t[i + p * k] * s[p + j * k]; + } + t[i + j * k] = -tau * x; + } + } + if j0 > 0 { + let (y, z) = (&mut y[..j0 * kb], &mut z[..j0 * kb]); + gemm( + j0, + kb, + j0, + T::one(), + &t, + k, + Op::N, + &s[j0 * k..], + k, + Op::N, + T::zero(), + y, + j0, + ); + gemm( + j0, + kb, + kb, + -T::one(), + y, + j0, + Op::N, + &t[j0 + j0 * k..], + k, + Op::N, + T::zero(), + z, + j0, + ); + for (c, z) in z.chunks_exact(j0).enumerate() { + t[(j0 + c) * k..(j0 + c) * k + j0].copy_from_slice(z); + } + } + j0 = j1; + } + t + } + + /// The first `cols` columns of `Q = H_0·H_1·…·H_k-1`, `cols` either `k` or `m`, from the + /// last reflection back, as LAPACK's `orgqr`: each block of reflections gives its own columns + /// directly and updates those right of it. The matrix starts as zeros: the identity's columns + /// from `k` on are taken in by the last block, or set for the reflections one at a time. + fn form_q(&self, cols: usize) -> Matrix { + let m = self.qr.rows(); + let k = self.taus.len(); + let a = &self.qr.data; + let mut q = Matrix::from_column_major(m, cols, vec![T::zero(); m * cols]); + // The whole `Q` of a tall matrix: all the reflections in one block, whose product with + // the columns of the identity right of them, most of the work, is a copy. + let one = cols > k && m >= 4 * k && m >= ONE_BLOCK; + let nb = if one { k } else { self.nb }; + if nb == 0 || k == 0 { + for j in k..cols { + q.data[j + j * m] = T::one(); + } + for j in (0..k).rev() { + let tau = self.taus[j]; + let v = &a[j * m + j + 1..(j + 1) * m]; + let (left, right) = q.data.split_at_mut((j + 1) * m); + let rest = cols - j - 1; + if rest > 0 && tau != T::zero() { + // Row `j` of the columns right of `j` is still zero. + reflect(v, tau, &mut right[j..], m, rest); + } + left[j + j * m] = T::one() - tau; + for (q, &v) in left[j * m + j + 1..].iter_mut().zip(v) { + *q = -tau * v; + } + } + return q; + } + let built; + let ts: &[T] = if one { + built = self.whole_t(); + &built + } else { + &self.ts + }; + let mut e1 = vec![T::zero(); nb * nb]; + let mut y = vec![T::zero(); nb * nb]; + let mut w = vec![T::zero(); 2 * nb * cols]; + let mut end = k; + while end > 0 { + let j0 = (end - 1) / nb * nb; + let j1 = end; + let kb = j1 - j0; + unit_lower(&a[j0 + j0 * m..], m, kb, &mut e1); + let v2 = &a[j1 + j0 * m..]; + let t = &ts[j0 * nb..]; + let (left, right) = q.data.split_at_mut(j1 * m); + let rest = cols - j1; + // The last block, whose columns right of it are still those of the identity. + let last = j1 == k; + if rest > 0 { + // The columns right of the block, zeros in its rows: `W = T·V2*·C2`, then + // `C1 = -E1·W` and `C2 -= V2·W`. + let (w1, w2) = w.split_at_mut(kb * rest); + let w2 = &mut w2[..kb * rest]; + if last { + // `V2*·I`. + for (c, w) in w1.chunks_exact_mut(kb).enumerate() { + for (i, w) in w.iter_mut().enumerate() { + *w = v2[c + i * m].conj(); + } + } + } else { + gemm( + kb, + rest, + m - j1, + T::one(), + v2, + m, + Op::C, + &right[j1..], + m, + Op::N, + T::zero(), + w1, + kb, + ); + } + gemm( + kb, + rest, + kb, + T::one(), + t, + nb, + Op::N, + w1, + kb, + Op::N, + T::zero(), + w2, + kb, + ); + gemm( + kb, + rest, + kb, + -T::one(), + &e1, + kb, + Op::N, + w2, + kb, + Op::N, + T::zero(), + &mut right[j0..], + m, + ); + // `C2 = I - V2·W` for the last block, the identity not held yet. + gemm( + m - j1, + rest, + kb, + -T::one(), + v2, + m, + Op::N, + w2, + kb, + Op::N, + if last { T::zero() } else { T::one() }, + &mut right[j1..], + m, + ); + if last { + for c in 0..rest.min(m - j1) { + right[j1 + c + c * m] += T::one(); + } + } + } + // The block's own columns, `[I; 0] - V·T·E1*`. + gemm( + kb, + kb, + kb, + T::one(), + t, + nb, + Op::N, + &e1, + kb, + Op::C, + T::zero(), + &mut y, + kb, + ); + gemm( + kb, + kb, + kb, + -T::one(), + &e1, + kb, + Op::N, + &y, + kb, + Op::N, + T::zero(), + &mut left[j0 + j0 * m..], + m, + ); + gemm( + m - j1, + kb, + kb, + -T::one(), + v2, + m, + Op::N, + &y, + kb, + Op::N, + T::zero(), + &mut left[j1 + j0 * m..], + m, + ); + for i in j0..j1 { + left[i + i * m] += T::one(); + } + end = j0; + } + q + } +} + +/// `x ← R⁻¹·x` for the upper triangle `R` of the `n × n` block `r`, its diagonal real, and a +/// column `x`, by columns of `R`: each element of `x`, once known, taken from those above it. +fn back_substitute(r: &[T], ldr: usize, n: usize, x: &mut [T]) { + for j in (0..n).rev() { + let d = r[j + j * ldr].re(); + let xj = T::from_parts(x[j].re() / d, x[j].im() / d); + x[j] = xj; + axpy(&r[j * ldr..j * ldr + j], xj, &mut x[..j]); + } +} + +/// Scratch of the blocked decomposition. +struct Work { + /// The elements of `R` a block's unit triangle stands in for while it is used. + save: Vec, + /// Products of blocks of reflections with the rest of the matrix. + w: Vec, + /// The products of a reflection's vector with the columns of a leaf. + u: Vec, +} + +impl Work { + /// Scratch for blocks of `nb` reflections of a matrix of `n` columns. + fn new(nb: usize, n: usize) -> Work { + Work { + save: vec![T::zero(); nb * nb], + w: vec![T::zero(); 2 * nb * n.max(nb)], + u: vec![T::zero(); nb], + } + } +} + +/// Factors the `m × n` panel `a`, `m ≥ n`, into its reflections and the triangular factor `T` +/// (`ldt`, zeros below its diagonal on entry) of their block, as LAPACK's `geqrt3`: the left +/// half, then its block applied to the right half, the right half, and the two halves' `T` +/// joined by products of blocks; leaves of [`LEAF`] columns one reflection at a time. +#[allow(clippy::too_many_arguments)] +fn panel( + a: &mut [T], + lda: usize, + m: usize, + n: usize, + taus: &mut [T], + t: &mut [T], + ldt: usize, + work: &mut Work, +) { + if n <= LEAF { + unblocked(a, lda, m, n, taus, Some((t, ldt)), &mut work.u); + return; + } + let n1 = n / 2; + let n2 = n - n1; + panel(a, lda, m, n1, &mut taus[..n1], t, ldt, work); + let (l, r) = a.split_at_mut(n1 * lda); + { + let Work { save, w, .. } = &mut *work; + let t1: &[T] = t; + with_v(l, lda, n1, save, |v| { + apply_block(v, lda, m, n1, t1, ldt, r, lda, n2, w); + }); + } + let (t1, t2) = t.split_at_mut(n1 * ldt); + panel( + &mut r[n1..], + lda, + m - n1, + n2, + &mut taus[n1..], + &mut t2[n1..], + ldt, + work, + ); + // `T12 = -T1·(V1*·V2)·T2`, `V1` from row `n1` on, where `V2` starts. + let Work { save, w, .. } = work; + let (x, y) = w.split_at_mut(n1 * n2); + let y = &mut y[..n1 * n2]; + let l: &[T] = l; + with_v(&mut r[n1..], lda, n2, save, |v2| { + gemm( + n1, + n2, + m - n1, + T::one(), + &l[n1..], + lda, + Op::C, + v2, + lda, + Op::N, + T::zero(), + x, + n1, + ); + }); + gemm( + n1, + n2, + n1, + T::one(), + t1, + ldt, + Op::N, + x, + n1, + Op::N, + T::zero(), + y, + n1, + ); + gemm( + n1, + n2, + n2, + -T::one(), + y, + n1, + Op::N, + &t2[n1..], + ldt, + Op::N, + T::zero(), + x, + n1, + ); + for (t, x) in t2.chunks_mut(ldt).zip(x.chunks(n1)) { + t[..n1].copy_from_slice(x); + } +} + +/// Factors the `m × n` block `a` by `min(m, n)` reflections, one at a time, as LAPACK's +/// `geqr2`, and with `t` (`ldt`, zeros below its diagonal on entry) also the triangular factor +/// of their block, as `larft`, from the same kind of products: for the reflection of column `j`, +/// the products of its vector with the columns right of it, which apply it, and with those left +/// of it, the reflections' vectors, which make the column `j` of `T`. `u` holds `n` elements of +/// scratch when `t` is given. +fn unblocked( + a: &mut [T], + lda: usize, + m: usize, + n: usize, + taus: &mut [T], + mut t: Option<(&mut [T], usize)>, + u: &mut [T], +) { + let k = m.min(n); + for j in 0..k { + let (left, right) = a.split_at_mut(((j + 1) * lda).min(a.len())); + let col = &mut left[j * lda..j * lda + m]; + let (tau, beta) = reflector(col[j], &mut col[j + 1..]); + col[j] = beta; + taus[j] = tau; + if tau == T::zero() { + continue; + } + let v = &left[j * lda + j + 1..j * lda + m]; + let rest = n - j - 1; + if rest > 0 { + reflect(v, tau.conj(), &mut right[j..], lda, rest); + } + if let Some((t, ldt)) = &mut t { + let ldt = *ldt; + t[j + j * ldt] = tau; + if j > 0 { + // `T[0..j, j] = -τ·T[0..j, 0..j]·V[:, 0..j]*·v_j`, the products from row `j`, + // where `v_j` starts. + let s = &mut u[..j]; + dots(v, &left[j + 1..], lda, j, s); + for (i, s) in s.iter_mut().enumerate() { + *s = (left[j + i * lda] + *s).conj(); + } + for i in 0..j { + let mut x = T::zero(); + for p in i..j { + x += t[i + p * ldt] * s[p]; + } + t[i + j * ldt] = -tau * x; + } + } + } + } +} + +/// `c ← (I - V·T·V*)*·c` for the `m × n` block `c`: `V` of `m × k` with its unit lower triangle +/// held, `T` of `k × k` upper triangular with zeros below; `W = V*·c`, `W ← T*·W`, `c -= V·W`. +/// `w` holds `2·k·n` elements of scratch. +#[allow(clippy::too_many_arguments)] +fn apply_block( + v: &[T], + ldv: usize, + m: usize, + k: usize, + t: &[T], + ldt: usize, + c: &mut [T], + ldc: usize, + n: usize, + w: &mut [T], +) { + if n == 0 || k == 0 { + return; + } + let (w1, w2) = w[..2 * k * n].split_at_mut(k * n); + gemm( + k, + n, + m, + T::one(), + v, + ldv, + Op::C, + c, + ldc, + Op::N, + T::zero(), + w1, + k, + ); + gemm( + k, + n, + k, + T::one(), + t, + ldt, + Op::C, + w1, + k, + Op::N, + T::zero(), + w2, + k, + ); + gemm( + m, + n, + k, + -T::one(), + v, + ldv, + Op::N, + w2, + k, + Op::N, + T::one(), + c, + ldc, + ); +} + +/// Runs `f` on the `m × k` block `a` of reflections' vectors as `V`, its top `k × k` made the +/// unit lower triangle for the time: the elements of `R` there are kept in `save` and put back. +fn with_v( + a: &mut [T], + lda: usize, + k: usize, + save: &mut [T], + f: impl FnOnce(&[T]) -> X, +) -> X { + for j in 0..k { + let col = &mut a[j * lda..j * lda + j + 1]; + save[j * k..j * k + j + 1].copy_from_slice(col); + col[..j].fill(T::zero()); + col[j] = T::one(); + } + let x = f(a); + for j in 0..k { + a[j * lda..j * lda + j + 1].copy_from_slice(&save[j * k..j * k + j + 1]); + } + x +} + +/// The unit lower triangle of the top `k × k` of the reflections' vectors `a`, into `e`, `k × k`. +fn unit_lower(a: &[T], lda: usize, k: usize, e: &mut [T]) { + for j in 0..k { + let col = &mut e[j * k..(j + 1) * k]; + col[..j].fill(T::zero()); + col[j] = T::one(); + col[j + 1..].copy_from_slice(&a[j * lda + j + 1..j * lda + k]); + } +} diff --git a/lib/src/math/matrix/qr/kernels.rs b/lib/src/math/matrix/qr/kernels.rs new file mode 100644 index 0000000..563819a --- /dev/null +++ b/lib/src/math/matrix/qr/kernels.rs @@ -0,0 +1,391 @@ +//! The vector kernels of the unblocked steps of the QR decomposition: a column's reflection, its +//! norm from one pass of sums of squares, and the products of a reflection's vector with a few +//! columns at a time, in sums of lanes that the compiler keeps in vectors. + +use super::super::Scalar; +use super::super::householder::{norm2, reflection}; +use crate::math::complex::Float; + +/// Whether the target fuses multiply-adds in hardware; elsewhere `mul_add` is a library call. +const FMA: bool = cfg!(any(target_arch = "aarch64", target_feature = "fma")); + +/// `c + a·b`, rounded once where the target fuses multiply-adds. +#[inline(always)] +fn fma(a: T, b: T, c: T) -> T { + if !FMA { + c + a * b + } else if T::IS_COMPLEX { + let (ar, ai, br, bi) = (a.re(), a.im(), b.re(), b.im()); + let re = (-ai).mul_add(bi, ar.mul_add(br, c.re())); + let im = ai.mul_add(br, ar.mul_add(bi, c.im())); + T::from_parts(re, im) + } else { + T::from_real(a.re().mul_add(b.re(), c.re())) + } +} + +/// `c + conj(a)·b`, rounded once where the target fuses multiply-adds. +#[inline(always)] +fn fma_conj(a: T, b: T, c: T) -> T { + if !FMA { + c + a.conj() * b + } else if T::IS_COMPLEX { + let (ar, ai, br, bi) = (a.re(), a.im(), b.re(), b.im()); + let re = ai.mul_add(bi, ar.mul_add(br, c.re())); + let im = (-ai).mul_add(br, ar.mul_add(bi, c.im())); + T::from_parts(re, im) + } else { + T::from_real(a.re().mul_add(b.re(), c.re())) + } +} + +/// The sum of the lanes of an accumulator, pairwise. +#[inline(always)] +fn lanes(s: [T; L]) -> T { + let mut s = s; + let mut w = L; + while w > 1 { + w /= 2; + for l in 0..w { + let x = s[l + w]; + s[l] += x; + } + } + s[0] +} + +/// The whole chunks of `L` of `x`. +#[inline(always)] +fn chunks(x: &[T]) -> &[[T; L]] { + x.as_chunks::().0 +} + +/// The whole chunks of `L` of `x`, to write. +#[inline(always)] +fn chunks_mut(x: &mut [T]) -> &mut [[T; L]] { + x.as_chunks_mut::().0 +} + +/// `v*·a_i` for four columns `a_i` as long as `v`, in `L` lanes each. +#[inline(always)] +fn dot4(v: &[T], a: [&[T]; 4]) -> [T; 4] { + let (vb, vt) = v.as_chunks::(); + let body = v.len() - vt.len(); + let mut s = [[T::zero(); L]; 4]; + let it = vb + .iter() + .zip(chunks::(a[0])) + .zip(chunks::(a[1])); + let it = it.zip(chunks::(a[2]).iter().zip(chunks::(a[3]))); + for (((x, a0), a1), (a2, a3)) in it { + for l in 0..L { + s[0][l] = fma_conj(x[l], a0[l], s[0][l]); + s[1][l] = fma_conj(x[l], a1[l], s[1][l]); + s[2][l] = fma_conj(x[l], a2[l], s[2][l]); + s[3][l] = fma_conj(x[l], a3[l], s[3][l]); + } + } + let mut out = [T::zero(); 4]; + for ((s, a), out) in s.iter_mut().zip(a).zip(&mut out) { + for (&x, &a) in vt.iter().zip(&a[body..]) { + s[0] = fma_conj(x, a, s[0]); + } + *out = lanes(*s); + } + out +} + +/// `v*·a` for a column `a` as long as `v`, in `L` lanes. +#[inline(always)] +fn dot1(v: &[T], a: &[T]) -> T { + let (vb, vt) = v.as_chunks::(); + let body = v.len() - vt.len(); + let mut s = [T::zero(); L]; + for (x, a) in vb.iter().zip(chunks::(a)) { + for l in 0..L { + s[l] = fma_conj(x[l], a[l], s[l]); + } + } + for (&x, &a) in vt.iter().zip(&a[body..]) { + s[0] = fma_conj(x, a, s[0]); + } + lanes(s) +} + +/// `a_i -= w_i·v` for four columns `a_i` as long as `v`, by chunks of `L`. +#[inline(always)] +fn axpy4(v: &[T], a: [&mut [T]; 4], w: [T; 4]) { + let (vb, vt) = v.as_chunks::(); + let body = v.len() - vt.len(); + let n = w.map(|w| -w); + let [a0, a1, a2, a3] = a; + { + let it = vb + .iter() + .zip(chunks_mut::(a0)) + .zip(chunks_mut::(a1)); + let it = it.zip( + chunks_mut::(a2) + .iter_mut() + .zip(chunks_mut::(a3)), + ); + for (((x, a0), a1), (a2, a3)) in it { + // All four chunks read before any is written, so that they go in vectors although + // the columns might, for all the compiler knows, overlap. + let (x, mut b0, mut b1, mut b2, mut b3) = (*x, *a0, *a1, *a2, *a3); + for l in 0..L { + b0[l] = fma(x[l], n[0], b0[l]); + b1[l] = fma(x[l], n[1], b1[l]); + b2[l] = fma(x[l], n[2], b2[l]); + b3[l] = fma(x[l], n[3], b3[l]); + } + (*a0, *a1, *a2, *a3) = (b0, b1, b2, b3); + } + } + for (a, n) in [a0, a1, a2, a3].into_iter().zip(n) { + for (&x, a) in vt.iter().zip(&mut a[body..]) { + *a = fma(x, n, *a); + } + } +} + +/// `a -= w·v` for a column `a` as long as `v`, by chunks of `L`. +#[inline(always)] +fn axpy1(v: &[T], a: &mut [T], w: T) { + let (vb, vt) = v.as_chunks::(); + let body = v.len() - vt.len(); + let n = -w; + for (x, a) in vb.iter().zip(chunks_mut::(a)) { + for l in 0..L { + a[l] = fma(x[l], n, a[l]); + } + } + for (&x, a) in vt.iter().zip(&mut a[body..]) { + *a = fma(x, n, *a); + } +} + +/// `out[c] = v*·a_c` for the `cols` columns `a_c = a[c·lda..c·lda + v.len()]`. +pub(super) fn dots(v: &[T], a: &[T], lda: usize, cols: usize, out: &mut [T]) { + if T::IS_COMPLEX { + dots_by::(v, a, lda, cols, out); + } else if narrow::(v.len()) { + dots_by::(v, a, lda, cols, out); + } else { + dots_by::(v, a, lda, cols, out); + } +} + +/// Whether the type is `f32`, whose vectors hold twice the lanes of `f64`'s, and the columns of +/// `len` long enough for twice the lanes, as many vectors of sums: shorter ones would leave more +/// of their elements to the sequential tail. At 8 lanes from any length, `f32` decompositions of +/// `32 × 32` took 2.19 µs against 1.77, `Q` of `128 × 16` 1.68 against 2.36. +#[inline(always)] +fn narrow(len: usize) -> bool { + !T::IS_COMPLEX && core::mem::size_of::() == 4 && len >= 64 +} + +/// [`dots`] in `L` lanes, four columns a pass. +fn dots_by(v: &[T], a: &[T], lda: usize, cols: usize, out: &mut [T]) { + let len = v.len(); + if len == 0 { + out[..cols].fill(T::zero()); + return; + } + let col = |c: usize| &a[c * lda..c * lda + len]; + let mut c = 0; + while c + 4 <= cols { + let s = dot4::(v, [col(c), col(c + 1), col(c + 2), col(c + 3)]); + out[c..c + 4].copy_from_slice(&s); + c += 4; + } + while c < cols { + out[c] = dot1::(v, col(c)); + c += 1; + } +} + +/// `c ← c - s·(1, v)·((1, v)*·c)` for the `cols` columns `c = a[j·lda..j·lda + 1 + v.len()]`: +/// a reflection `I - τ·(1, v)·(1, v)*` applied with `s = τ`, or its conjugate transpose with +/// `s = conj(τ)`. Four columns a pass, each read for the products and again, from the cache, +/// for the update. +pub(super) fn reflect(v: &[T], s: T, a: &mut [T], lda: usize, cols: usize) { + if cols == 1 && v.len() >= LONG { + // One long column, as for a single right-hand side: more sums, which the latency of + // the multiply-adds would otherwise hold up. + let (h, t) = a[..v.len() + 1].split_first_mut().unwrap(); + if T::IS_COMPLEX { + let w = s * (*h + dot1::(v, t)); + *h -= w; + axpy1::(v, t, w); + } else if narrow::(v.len()) { + let w = s * (*h + dot1::(v, t)); + *h -= w; + axpy1::(v, t, w); + } else { + let w = s * (*h + dot1::(v, t)); + *h -= w; + axpy1::(v, t, w); + } + } else if T::IS_COMPLEX && cols == 1 && v.len() >= 8 { + // A complex multiply-add is two in a row of each part's sum, so that a shorter column + // already waits on them in two lanes. + reflect_by::(v, s, a, lda, cols); + } else if T::IS_COMPLEX { + reflect_by::(v, s, a, lda, cols); + } else if narrow::(v.len()) { + reflect_by::(v, s, a, lda, cols); + } else { + reflect_by::(v, s, a, lda, cols); + } +} + +/// [`reflect`] in `L` lanes. +fn reflect_by(v: &[T], s: T, a: &mut [T], lda: usize, cols: usize) { + let len = v.len() + 1; + let mut rest = a; + let mut c = 0; + while c + 4 <= cols { + let (a0, r) = rest.split_at_mut(lda); + let (a1, r) = r.split_at_mut(lda); + let (a2, r) = r.split_at_mut(lda); + let (a3, r) = r.split_at_mut(lda.min(r.len())); + rest = r; + let (h0, t0) = a0[..len].split_first_mut().unwrap(); + let (h1, t1) = a1[..len].split_first_mut().unwrap(); + let (h2, t2) = a2[..len].split_first_mut().unwrap(); + let (h3, t3) = a3[..len].split_first_mut().unwrap(); + let u = dot4::(v, [t0, t1, t2, t3]); + let mut w = [T::zero(); 4]; + for ((h, u), w) in [h0, h1, h2, h3].into_iter().zip(u).zip(&mut w) { + *w = s * (*h + u); + *h -= *w; + } + axpy4::(v, [t0, t1, t2, t3], w); + c += 4; + } + while c < cols { + let (a0, r) = rest.split_at_mut(lda.min(rest.len())); + rest = r; + let (h, t) = a0[..len].split_first_mut().unwrap(); + let w = s * (*h + dot1::(v, t)); + *h -= w; + axpy1::(v, t, w); + c += 1; + } +} + +/// `[v_0*·y, v_1*·y, v_2*·y, v_3*·y]` for four columns `v_i` as long as `y`. +pub(super) fn dots4(y: &[T], v: [&[T]; 4]) -> [T; 4] { + let s = if T::IS_COMPLEX { + dot4::(y, v) + } else if narrow::(y.len()) { + dot4::(y, v) + } else { + dot4::(y, v) + }; + s.map(|x| x.conj()) +} + +/// `y -= Σ w_i·v_i` for four columns `v_i` as long as `y`, `y` read and written once. +pub(super) fn axpys4(y: &mut [T], v: [&[T]; 4], w: [T; 4]) { + if T::IS_COMPLEX { + axpys4_by::(y, v, w); + } else if narrow::(y.len()) { + axpys4_by::(y, v, w); + } else { + axpys4_by::(y, v, w); + } +} + +/// [`axpys4`] by chunks of `L`. +fn axpys4_by(y: &mut [T], v: [&[T]; 4], w: [T; 4]) { + let (yb, yt) = y.as_chunks_mut::(); + let body = yb.len() * L; + let n = w.map(|w| -w); + let it = yb + .iter_mut() + .zip(chunks::(v[0])) + .zip(chunks::(v[1])); + let it = it.zip(chunks::(v[2]).iter().zip(chunks::(v[3]))); + for (((y, v0), v1), (v2, v3)) in it { + for l in 0..L { + y[l] = fma( + v3[l], + n[3], + fma(v2[l], n[2], fma(v1[l], n[1], fma(v0[l], n[0], y[l]))), + ); + } + } + for (r, y) in yt.iter_mut().enumerate() { + let r = body + r; + *y = fma( + v[3][r], + n[3], + fma(v[2][r], n[2], fma(v[1][r], n[1], fma(v[0][r], n[0], *y))), + ); + } +} + +/// The length of `v` from which [`reflect`] of one column sums in more lanes, 16 of `f64` (32 of +/// `f32` from 64) and 8 of a complex type: the latency of the multiply-adds, not their number, holds up the sums +/// of a long column, and a single right-hand side of the least squares has no other columns to +/// interleave with. +const LONG: usize = 32; + +/// `a -= w·v` for a column `a` as long as `v`. +pub(super) fn axpy(v: &[T], w: T, a: &mut [T]) { + if T::IS_COMPLEX { + axpy1::(v, a, w); + } else if narrow::(v.len()) { + axpy1::(v, a, w); + } else { + axpy1::(v, a, w); + } +} + +/// The sum of the squares of the absolute values of `x`, in eight sums. +fn squares(x: &[T]) -> T::Real { + let (body, rest) = x.as_chunks::<8>(); + let mut s = [T::Real::ZERO; 8]; + for c in body { + for l in 0..8 { + s[l] += c[l].abs2(); + } + } + let mut t = ((s[0] + s[1]) + (s[2] + s[3])) + ((s[4] + s[5]) + (s[6] + s[7])); + for v in rest { + t += v.abs2(); + } + t +} + +/// Whether a sum of squares is in the range where no square can have overflowed or underflowed +/// enough to matter, zero and NaN excluded. +#[inline(always)] +fn safe(t: R) -> bool { + t >= R::MIN_POSITIVE / R::EPSILON && t <= R::MAX * R::EPSILON +} + +/// The reflection `H = I - τ·v·v*`, `v = (1, x)`, with `H*·(alpha, x) = (β, 0)` and `β` real, as +/// [`reflector`](super::super::householder::reflector), but with the norm of `(alpha, x)` from +/// one pass of sums of squares where that is safe: overwrites `x` with the rest of `v` and +/// returns `(τ, β)`. +pub(super) fn reflector(alpha: T, x: &mut [T]) -> (T, T) { + let t = squares(x); + let norm = if safe(t) { + let a = alpha.abs2(); + if a == T::Real::ZERO || safe(a) { + (a + t).sqrt() + } else { + alpha.abs().hypot(t.sqrt()) + } + } else { + let xnorm = norm2(x); + if xnorm == T::Real::ZERO && alpha.im() == T::Real::ZERO { + return (T::zero(), alpha); + } + alpha.abs().hypot(xnorm) + }; + reflection(alpha, norm, x) +} diff --git a/lib/src/math/matrix/schur.rs b/lib/src/math/matrix/schur.rs new file mode 100644 index 0000000..923ff3f --- /dev/null +++ b/lib/src/math/matrix/schur.rs @@ -0,0 +1,892 @@ +//! The Schur decomposition: a blocked reduction to Hessenberg form, then the QR algorithm, in +//! real arithmetic for real matrices: for small ones (below order 120, or 90 for complex ones +//! with `Q`) Francis's double shifts (single shifts for complex ones), for larger ones the +//! small-bulge multishift QR algorithm with aggressive early deflation, most of its work in +//! products of blocks. + +use super::householder::{reflection, reflector}; +use super::properties::range_exponent; +use super::{Matrix, Scalar}; +use crate::math::complex::{Complex, Float}; + +mod deflate; +mod hessenberg; +mod multishift; +mod vectors; + +pub(crate) use hessenberg::hessenberg; +use multishift::multishift; +pub(crate) use vectors::eigenvectors; + +/// The Schur decomposition `A = Q·T·Q*` of a square matrix, `Q` unitary (orthogonal for real +/// elements). For complex elements `T` is upper triangular, its diagonal the eigenvalues. For +/// real ones it is the real Schur form, quasi-upper triangular: blocks of `1 × 1` for the real +/// eigenvalues and of `2 × 2` for the pairs of complex conjugate ones, each `[[a, b], [c, a]]` +/// with `b·c < 0`, its eigenvalues `a ± i·√(-b·c)`. From [`Matrix::schur`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::complex::Complex; +/// use a::math::matrix::Matrix; +/// +/// // A rotation by a quarter turn: eigenvalues ±i, one 2 × 2 block. +/// let a: Matrix = Matrix::from_rows(&[[0.0, -1.0], [1.0, 0.0]]); +/// let s = a.schur(); +/// let back = &(&s.q() * &s.t()) * &s.q().transpose(); +/// assert!((&back - &a).as_slice().iter().all(|x| x.abs() < 1e-15)); +/// let mut values = s.eigenvalues().to_vec(); +/// values.sort_by(|a, b| a.im.total_cmp(&b.im)); +/// assert_eq!(values, [Complex::new(0.0, -1.0), Complex::new(0.0, 1.0)]); +/// ``` +#[derive(Clone, Debug)] +pub struct Schur { + /// `Q`. + q: Matrix, + /// `T`. + t: Matrix, + /// The eigenvalues, in the order of the diagonal of `T`. + values: Vec>, +} + +impl Matrix { + /// The Schur decomposition `A = Q·T·Q*`: see [`Schur`]. A reduction to Hessenberg form by + /// blocks of Householder reflections, then the QR algorithm with implicit shifts, for large + /// matrices many at a time with aggressive early deflation, about `25n³` operations. A + /// matrix with a NaN or an infinity gives NaNs. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn schur(&self) -> Schur { + assert!(self.is_square(), "the matrix is not square"); + let (t, q, values) = schur(self, true, true); + Schur { + q: q.unwrap_or_default(), + t, + values, + } + } + + /// The eigenvalues, in no particular order, those of a real matrix that are not real in + /// pairs of conjugates: the diagonal of the Schur form, computed without `Q`, so about + /// `10n³` operations. A matrix with a NaN or an infinity gives NaNs. + /// + /// # Panics + /// + /// Panics if the matrix is not square. + #[track_caller] + pub fn eigenvalues(&self) -> Vec> { + assert!(self.is_square(), "the matrix is not square"); + schur(self, false, false).2 + } +} + +impl Schur { + /// `Q`, unitary. + pub fn q(&self) -> Matrix { + self.q.clone() + } + + /// `T`, upper triangular, or quasi-upper triangular for real elements. + pub fn t(&self) -> Matrix { + self.t.clone() + } + + /// The eigenvalues, in the order of the diagonal of `T`; a pair of complex conjugates of a + /// real matrix, the one with the positive imaginary part first. + pub fn eigenvalues(&self) -> &[Complex] { + &self.values + } +} + +/// The Schur form `T` of `a`, with `Q` when `want_q` is set, and the eigenvalues; only the +/// eigenvalues are right in `T` when `want_t` is not set. +#[allow(clippy::type_complexity)] +pub(crate) fn schur( + a: &Matrix, + want_t: bool, + want_q: bool, +) -> (Matrix, Option>, Vec>) { + let n = a.rows(); + if a.data.iter().any(|x| !x.is_finite()) { + let nan = T::from_parts(T::Real::NAN, T::Real::NAN); + let all = Matrix::from_fn(n, n, |_, _| nan); + let w = vec![Complex::new(T::Real::NAN, T::Real::NAN); n]; + return (all.clone(), want_q.then_some(all), w); + } + let mut h = a.clone(); + // Scaled by a power of two when the largest element is so small or so large that the + // reduction or the QR algorithm could underflow or overflow, as LAPACK's `geev`: `T` and the + // eigenvalues scaled back, exactly, and `Q` the same. + let k = range_exponent(&h.data); + if k != 0 { + let up = T::Real::TWO.powi(k); + for x in &mut h.data { + *x = x.scale(up); + } + } + let mut q = hessenberg(&mut h, want_q); + let mut w = vec![Complex::new(T::Real::ZERO, T::Real::ZERO); n]; + let z = q.as_mut().map(|q| q.data.as_mut_slice()); + let info = multishift(want_t, &mut h.data, n, z, &mut w, false); + w[..info].fill(Complex::new(T::Real::NAN, T::Real::NAN)); + if want_t && info == 0 { + // Zeros below the subdiagonal, and for complex elements on it. + for j in 0..n { + let from = if T::IS_COMPLEX { j + 1 } else { j + 2 }; + for i in from..n { + h.data[i + j * n] = T::zero(); + } + } + } + if k != 0 { + let down = T::Real::TWO.powi(-k); + for x in &mut h.data { + *x = x.scale(down); + } + for w in &mut w { + *w = Complex::new(w.re * down, w.im * down); + } + } + (h, q, w) +} + +/// The QR algorithm with one or two shifts on the upper Hessenberg `h` of order `n`, as LAPACK's +/// `lahqr`: in real arithmetic, Francis's double shifts, for real elements. The eigenvalues +/// into `w`, `h` turned into the Schur form when `want_t` is set, the transformations +/// accumulated into `z` when given. Returns the number of leading eigenvalues it didn't +/// converge to, zero on success. +fn small_qr( + h: &mut [T], + n: usize, + z: Option<&mut [T]>, + want_t: bool, + w: &mut [Complex], +) -> usize { + if T::IS_COMPLEX { + return complex_qr(h, n, z, want_t, w); + } + let mut hr: Vec = h.iter().map(|x| x.re()).collect(); + let mut zr: Option> = z.as_deref().map(|z| z.iter().map(|x| x.re()).collect()); + let info = real_qr(&mut hr, n, zr.as_deref_mut(), want_t, w); + for (x, r) in h.iter_mut().zip(hr) { + *x = T::from_real(r); + } + if let (Some(z), Some(zr)) = (z, zr) { + for (x, r) in z.iter_mut().zip(zr) { + *x = T::from_real(r); + } + } + info +} + +/// Every how many iterations without a deflation an exceptional shift is taken. +const EXCEPTIONAL: usize = 10; + +/// `(c·x + s·y, c·y - s·x)`, a plane rotation. +#[inline] +fn rotate(x: &mut R, y: &mut R, c: R, s: R) { + let t = c * *x + s * *y; + *y = c * *y - s * *x; + *x = t; +} + +/// The QR algorithm with Francis's implicit double shifts on the real upper Hessenberg `h`, +/// as LAPACK's `lahqr`: the eigenvalues into `w`, `h` turned into the real Schur form when +/// `want_t` is set, and the rotations accumulated into `z` when given. Returns the number of +/// leading eigenvalues it didn't converge to. +fn real_qr( + h: &mut [R], + n: usize, + mut z: Option<&mut [R]>, + want_t: bool, + w: &mut [Complex], +) -> usize { + let at = |i: usize, j: usize| i + j * n; + if n == 0 { + return 0; + } + let ulp = R::EPSILON; + let smlnum = R::MIN_POSITIVE * (R::from_f64(n as f64) / ulp); + let max_its = 30 * n.max(10); + let mut kdefl = 0; + // The active block is rows and columns `l..=i`. + let mut i = n - 1; + loop { + let mut l = 0; + let mut converged = false; + for _ in 0..=max_its { + // A negligible subdiagonal element splits the block. + let mut k = i; + while k > l { + let sub = h[at(k, k - 1)].abs(); + if sub <= smlnum { + break; + } + let mut tst = h[at(k - 1, k - 1)].abs() + h[at(k, k)].abs(); + if tst == R::ZERO { + if k >= 2 { + tst += h[at(k - 1, k - 2)].abs(); + } + if k + 1 < n { + tst += h[at(k + 1, k)].abs(); + } + } + // Ahues and Tisseur's conservative test. + if sub <= ulp * tst { + let up = h[at(k - 1, k)].abs(); + let (ab, ba) = (sub.max(up), sub.min(up)); + let d = (h[at(k - 1, k - 1)] - h[at(k, k)]).abs(); + let hk = h[at(k, k)].abs(); + let (aa, bb) = (hk.max(d), hk.min(d)); + let s = aa + ab; + if ba * (ab / s) <= smlnum.max(ulp * (bb * (aa / s))) { + break; + } + } + k -= 1; + } + l = k; + if l > 0 { + h[at(l, l - 1)] = R::ZERO; + } + if l + 1 >= i { + converged = true; + break; + } + kdefl += 1; + let (i1, i2) = if want_t { (0, n - 1) } else { (l, i) }; + // The shifts: the eigenvalues of the trailing 2 × 2 block, or exceptional ones. + let (h11, h12, h21, h22) = if kdefl % (2 * EXCEPTIONAL) == 0 { + let s = h[at(i, i - 1)].abs() + h[at(i - 1, i - 2)].abs(); + let h11 = R::from_f64(0.75) * s + h[at(i, i)]; + (h11, R::from_f64(-0.4375) * s, s, h11) + } else if kdefl % EXCEPTIONAL == 0 { + let s = h[at(l + 1, l)].abs() + h[at(l + 2, l + 1)].abs(); + let h11 = R::from_f64(0.75) * s + h[at(l, l)]; + (h11, R::from_f64(-0.4375) * s, s, h11) + } else { + ( + h[at(i - 1, i - 1)], + h[at(i - 1, i)], + h[at(i, i - 1)], + h[at(i, i)], + ) + }; + let s = h11.abs() + h12.abs() + h21.abs() + h22.abs(); + let (rt1r, rt1i, rt2r, rt2i); + if s == R::ZERO { + (rt1r, rt1i, rt2r, rt2i) = (R::ZERO, R::ZERO, R::ZERO, R::ZERO); + } else { + let (h11, h12, h21, h22) = (h11 / s, h12 / s, h21 / s, h22 / s); + let tr = (h11 + h22) * R::HALF; + let det = (h11 - tr) * (h22 - tr) - h12 * h21; + let rtdisc = det.abs().sqrt(); + if det >= R::ZERO { + rt1r = tr * s; + rt2r = rt1r; + rt1i = rtdisc * s; + rt2i = -rt1i; + } else { + // Two real shifts: both the one nearer `h22`. + let (a, b) = (tr + rtdisc, tr - rtdisc); + let r = if (a - h22).abs() <= (b - h22).abs() { + a * s + } else { + b * s + }; + (rt1r, rt2r, rt1i, rt2i) = (r, r, R::ZERO, R::ZERO); + } + } + // Where to start the sweep: two consecutive small subdiagonal elements. + let mut m = i - 2; + let mut v = [R::ZERO; 3]; + loop { + let h21s = h[at(m + 1, m)]; + let s = (h[at(m, m)] - rt2r).abs() + rt2i.abs() + h21s.abs(); + let h21s = h21s / s; + v[0] = h21s * h[at(m, m + 1)] + (h[at(m, m)] - rt1r) * ((h[at(m, m)] - rt2r) / s) + - rt1i * (rt2i / s); + v[1] = h21s * (h[at(m, m)] + h[at(m + 1, m + 1)] - rt1r - rt2r); + v[2] = h21s * h[at(m + 2, m + 1)]; + let s = v[0].abs() + v[1].abs() + v[2].abs(); + for x in &mut v { + *x /= s; + } + if m == l { + break; + } + let h00 = h[at(m, m - 1)].abs() * (v[1].abs() + v[2].abs()); + let h01 = v[0].abs() + * (h[at(m - 1, m - 1)].abs() + h[at(m, m)].abs() + h[at(m + 1, m + 1)].abs()); + if h00 <= ulp * h01 { + break; + } + m -= 1; + } + // The double-shift sweep, chasing the bulge down. + for k in m..i { + let nr = 3.min(i - k + 1); + if k > m { + for p in 0..nr { + v[p] = h[at(k + p, k - 1)]; + } + } + let (t1, beta) = { + let (first, rest) = v.split_at_mut(1); + real_reflector(first[0], &mut rest[..nr - 1]) + }; + v[0] = beta; + if k > m { + h[at(k, k - 1)] = v[0]; + h[at(k + 1, k - 1)] = R::ZERO; + if k + 2 < i + 1 { + h[at(k + 2, k - 1)] = R::ZERO; + } + } else if m > l { + // Rather than negating it, which goes wrong when `v` underflows. + h[at(k, k - 1)] = h[at(k, k - 1)] * (R::ONE - t1); + } + let v2 = v[1]; + let t2 = t1 * v2; + if nr == 3 { + let v3 = v[2]; + let t3 = t1 * v3; + for j in k..=i2 { + let sum = h[at(k, j)] + v2 * h[at(k + 1, j)] + v3 * h[at(k + 2, j)]; + h[at(k, j)] -= sum * t1; + h[at(k + 1, j)] -= sum * t2; + h[at(k + 2, j)] -= sum * t3; + } + for j in i1..=(k + 3).min(i) { + let sum = h[at(j, k)] + v2 * h[at(j, k + 1)] + v3 * h[at(j, k + 2)]; + h[at(j, k)] -= sum * t1; + h[at(j, k + 1)] -= sum * t2; + h[at(j, k + 2)] -= sum * t3; + } + if let Some(z) = z.as_deref_mut() { + for j in 0..n { + let sum = z[at(j, k)] + v2 * z[at(j, k + 1)] + v3 * z[at(j, k + 2)]; + z[at(j, k)] -= sum * t1; + z[at(j, k + 1)] -= sum * t2; + z[at(j, k + 2)] -= sum * t3; + } + } + } else { + for j in k..=i2 { + let sum = h[at(k, j)] + v2 * h[at(k + 1, j)]; + h[at(k, j)] -= sum * t1; + h[at(k + 1, j)] -= sum * t2; + } + for j in i1..=i { + let sum = h[at(j, k)] + v2 * h[at(j, k + 1)]; + h[at(j, k)] -= sum * t1; + h[at(j, k + 1)] -= sum * t2; + } + if let Some(z) = z.as_deref_mut() { + for j in 0..n { + let sum = z[at(j, k)] + v2 * z[at(j, k + 1)]; + z[at(j, k)] -= sum * t1; + z[at(j, k + 1)] -= sum * t2; + } + } + } + } + } + if !converged { + return i + 1; + } + if l == i { + w[i] = Complex::new(h[at(i, i)], R::ZERO); + } else { + // A 2 × 2 block: its standard form, and the rotation that gives it. + let (a, b, c, d) = ( + h[at(i - 1, i - 1)], + h[at(i - 1, i)], + h[at(i, i - 1)], + h[at(i, i)], + ); + let s = standardize(a, b, c, d); + h[at(i - 1, i - 1)] = s.a; + h[at(i - 1, i)] = s.b; + h[at(i, i - 1)] = s.c; + h[at(i, i)] = s.d; + w[i - 1] = Complex::new(s.rt1r, s.rt1i); + w[i] = Complex::new(s.rt2r, s.rt2i); + if want_t { + for j in i + 1..n { + let (x, y) = (h[at(i - 1, j)], h[at(i, j)]); + let (mut x, mut y) = (x, y); + rotate(&mut x, &mut y, s.cs, s.sn); + h[at(i - 1, j)] = x; + h[at(i, j)] = y; + } + for j in 0..i - 1 { + let (mut x, mut y) = (h[at(j, i - 1)], h[at(j, i)]); + rotate(&mut x, &mut y, s.cs, s.sn); + h[at(j, i - 1)] = x; + h[at(j, i)] = y; + } + } + if let Some(z) = z.as_deref_mut() { + for j in 0..n { + let (mut x, mut y) = (z[at(j, i - 1)], z[at(j, i)]); + rotate(&mut x, &mut y, s.cs, s.sn); + z[at(j, i - 1)] = x; + z[at(j, i)] = y; + } + } + } + kdefl = 0; + if l == 0 { + break; + } + i = l - 1; + } + 0 +} + +/// The real reflection `I - τ·v·vᵀ`, `v = (1, x)`, taking `(alpha, x)` to `(β, 0)`: overwrites +/// `x` with the rest of `v` and returns `(τ, β)`. +fn real_reflector(alpha: R, x: &mut [R]) -> (R, R) { + let big = x.iter().fold(alpha.abs(), |m, v| m.max(v.abs())); + if moderate(big) { + let squares = x.iter().fold(R::ZERO, |s, v| s + *v * *v); + if squares == R::ZERO { + return (R::ZERO, alpha); + } + let beta = -(alpha * alpha + squares).sqrt().copysign(alpha); + return (real_reflection(alpha, beta, x), beta); + } + let mut scale = R::ZERO; + for v in x.iter() { + scale = scale.max(v.abs()); + } + if scale == R::ZERO { + return (R::ZERO, alpha); + } + let mut s = R::ZERO; + for v in x.iter() { + let t = *v / scale; + s += t * t; + } + let xnorm = s.sqrt() * scale; + let beta = -alpha.hypot(xnorm).copysign(alpha); + // A norm below `MIN/ε` has lost digits to underflow: scaled up by a power of two and taken + // again, as `reflection` does. + let small = R::MIN_POSITIVE / R::EPSILON; + if beta.abs() < small { + let up = R::ONE / small; + for v in x.iter_mut() { + *v *= up; + } + let (tau, beta) = real_reflector(alpha * up, x); + return (tau, beta * small); + } + (real_reflection(alpha, beta, x), beta) +} + +/// `τ`, with `x` scaled to the rest of `v`, of the real reflection taking `(alpha, x)` to +/// `(β, 0)`: [`reflection`](super::householder::reflection) for real numbers, without its +/// overflows near the largest float. +fn real_reflection(alpha: R, beta: R, x: &mut [R]) -> R { + let d = alpha - beta; + if !d.is_finite() { + let r = R::ONE / (alpha * R::HALF - beta * R::HALF); + for v in x.iter_mut() { + *v = *v * r * R::HALF; + } + } else { + let r = R::ONE / d; + if r.is_finite() { + for v in x.iter_mut() { + *v *= r; + } + } else { + for v in x.iter_mut() { + *v /= d; + } + } + } + R::ONE - alpha / beta +} + +/// A real 2 × 2 block in standard form, its eigenvalues and the rotation to it. +struct Standard { + a: R, + b: R, + c: R, + d: R, + rt1r: R, + rt1i: R, + rt2r: R, + rt2i: R, + cs: R, + sn: R, +} + +/// The Schur form of the real block `[[a, b], [c, d]]`, as LAPACK's `lanv2`: upper triangular +/// for real eigenvalues, `[[a, b], [c, a]]` with `b·c < 0` for complex ones, by the rotation +/// `[[cs, -sn], [sn, cs]]`. +fn standardize(mut a: R, mut b: R, mut c: R, mut d: R) -> Standard { + let (mut cs, mut sn) = (R::ONE, R::ZERO); + let sign = |x: R, y: R| x.abs().copysign(y); + if c == R::ZERO { + } else if b == R::ZERO { + // Swap the rows and the columns. + cs = R::ZERO; + sn = R::ONE; + core::mem::swap(&mut a, &mut d); + b = -c; + c = R::ZERO; + } else if a - d == R::ZERO && b.is_sign_negative() != c.is_sign_negative() { + } else { + let temp = a - d; + let mut p = temp * R::HALF; + let bcmax = b.abs().max(c.abs()); + let bcmis = b.abs().min(c.abs()) * sign(R::ONE, b) * sign(R::ONE, c); + let scale = p.abs().max(bcmax); + let mut z = (p / scale) * p + (bcmax / scale) * bcmis; + if z >= R::from_f64(4.0) * R::EPSILON { + // Real eigenvalues: `z` is their half difference. + z = p + sign(scale.sqrt() * z.sqrt(), p); + a = d + z; + d -= (bcmax / z) * bcmis; + let tau = c.hypot(z); + cs = z / tau; + sn = c / tau; + b -= c; + c = R::ZERO; + } else { + // Complex or nearly equal eigenvalues: equal diagonal elements. + let sigma = b + c; + let tau = sigma.hypot(temp); + cs = (R::HALF * (R::ONE + sigma.abs() / tau)).sqrt(); + sn = -(temp * R::HALF / (tau * cs)) * sign(R::ONE, sigma); + let aa = a * cs + b * sn; + let bb = -a * sn + b * cs; + let cc = c * cs + d * sn; + let dd = -c * sn + d * cs; + a = aa * cs + cc * sn; + b = bb * cs + dd * sn; + c = -aa * sn + cc * cs; + d = -bb * sn + dd * cs; + let temp = R::HALF * (a + d); + a = temp; + d = temp; + if c != R::ZERO { + if b != R::ZERO { + if b.is_sign_negative() == c.is_sign_negative() { + // Real eigenvalues after all: upper triangular. + let sab = b.abs().sqrt(); + let sac = c.abs().sqrt(); + p = sign(sab * sac, c); + let tau = R::ONE / (b + c).abs().sqrt(); + a = temp + p; + d = temp - p; + b -= c; + c = R::ZERO; + let cs1 = sab * tau; + let sn1 = sac * tau; + let t = cs * cs1 - sn * sn1; + sn = cs * sn1 + sn * cs1; + cs = t; + } + } else { + b = -c; + c = R::ZERO; + let t = cs; + cs = -sn; + sn = t; + } + } + } + } + let (rt1i, rt2i) = if c == R::ZERO { + (R::ZERO, R::ZERO) + } else { + let r = b.abs().sqrt() * c.abs().sqrt(); + (r, -r) + }; + Standard { + a, + b, + c, + d, + rt1r: a, + rt1i, + rt2r: d, + rt2i, + cs, + sn, + } +} + +/// `|re| + |im|`. +#[inline] +fn abs1(x: C) -> C::Real { + x.re().abs() + x.im().abs() +} + +/// Whether the squares of numbers up to `big`, and their sums, neither overflow nor underflow +/// harmfully. +#[inline] +fn moderate(big: R) -> bool { + let (low, high) = if R::MANTISSA_DIGITS > 24 { + (1e-130, 1e130) + } else { + (1e-15, 1e15) + }; + R::from_f64(low) < big && big < R::from_f64(high) +} + +/// [`reflector`] for the short vectors of the QR sweeps' bulges: the squares summed directly +/// when they are [`moderate`]. +#[inline] +fn short_reflector(alpha: C, x: &mut [C]) -> (C, C) { + let zero = C::Real::ZERO; + let big = x + .iter() + .fold(alpha.re().abs().max(alpha.im().abs()), |m, v| { + m.max(v.re().abs()).max(v.im().abs()) + }); + if !moderate(big) { + return reflector(alpha, x); + } + let squares = x.iter().fold(zero, |s, v| s + v.abs2()); + if squares == zero && alpha.im() == zero { + return (C::zero(), alpha); + } + reflection(alpha, (alpha.abs2() + squares).sqrt(), x) +} + +/// The QR algorithm with single implicit shifts on the complex upper Hessenberg `h`, as +/// LAPACK's `lahqr` for complex matrices: the eigenvalues into `w`, `h` turned into the Schur +/// form when `want_t` is set, the transformations accumulated into `z` when given. Returns the +/// number of leading eigenvalues it didn't converge to. +fn complex_qr( + h: &mut [C], + n: usize, + mut z: Option<&mut [C]>, + want_t: bool, + w: &mut [Complex], +) -> usize { + let at = |i: usize, j: usize| i + j * n; + let zero = C::Real::ZERO; + if n == 0 { + return 0; + } + // Real subdiagonal elements, by a diagonal unitary similarity. + for i in 1..n { + let x = h[at(i, i - 1)]; + if x.im() != zero { + let a = x.abs(); + let sc = x.conj().scale(C::Real::ONE / a); + h[at(i, i - 1)] = C::from_real(a); + let jhi = n - 1; + for j in i..=jhi { + h[at(i, j)] *= sc; + } + for j in 0..=(i + 1).min(jhi) { + h[at(j, i)] *= sc.conj(); + } + if let Some(z) = z.as_deref_mut() { + for j in 0..n { + z[at(j, i)] *= sc.conj(); + } + } + } + } + let ulp = C::Real::EPSILON; + let smlnum = C::Real::MIN_POSITIVE * (C::Real::from_f64(n as f64) / ulp); + let max_its = 30 * n.max(10); + let mut kdefl = 0; + let mut i = n - 1; + loop { + let mut l = 0; + let mut converged = false; + for _ in 0..=max_its { + let mut k = i; + while k > l { + let sub = abs1(h[at(k, k - 1)]); + if sub <= smlnum { + break; + } + let mut tst = abs1(h[at(k - 1, k - 1)]) + abs1(h[at(k, k)]); + if tst == zero { + if k >= 2 { + tst += h[at(k - 1, k - 2)].re().abs(); + } + if k + 1 < n { + tst += h[at(k + 1, k)].re().abs(); + } + } + if h[at(k, k - 1)].re().abs() <= ulp * tst { + let up = abs1(h[at(k - 1, k)]); + let (ab, ba) = (sub.max(up), sub.min(up)); + let d = abs1(h[at(k - 1, k - 1)] - h[at(k, k)]); + let hk = abs1(h[at(k, k)]); + let (aa, bb) = (hk.max(d), hk.min(d)); + let s = aa + ab; + if ba * (ab / s) <= smlnum.max(ulp * (bb * (aa / s))) { + break; + } + } + k -= 1; + } + l = k; + if l > 0 { + h[at(l, l - 1)] = C::zero(); + } + if l >= i { + converged = true; + break; + } + kdefl += 1; + let (i1, i2) = if want_t { (0, n - 1) } else { (l, i) }; + // The shift: Wilkinson's, or an exceptional one. + let t = if kdefl % (2 * EXCEPTIONAL) == 0 { + let s = C::Real::from_f64(0.75) * h[at(i, i - 1)].re().abs(); + h[at(i, i)] + C::from_real(s) + } else if kdefl % EXCEPTIONAL == 0 { + let s = C::Real::from_f64(0.75) * h[at(l + 1, l)].re().abs(); + h[at(l, l)] + C::from_real(s) + } else { + let mut t = h[at(i, i)]; + let u = h[at(i - 1, i)].sqrt() * h[at(i, i - 1)].sqrt(); + let s = abs1(u); + if s != zero { + let x = (h[at(i - 1, i - 1)] - t).scale(C::Real::HALF); + let sx = abs1(x); + let s = s.max(sx); + let (xs, us) = (x.scale(C::Real::ONE / s), u.scale(C::Real::ONE / s)); + let mut y = (xs * xs + us * us).sqrt().scale(s); + if sx > zero { + let xr = x.scale(C::Real::ONE / sx); + if xr.re() * y.re() + xr.im() * y.im() < zero { + y = -y; + } + } + t -= u * (u / (x + y)); + } + t + }; + // Where to start: two consecutive small subdiagonal elements. + let mut m = i - 1; + let mut v: [C; 2]; + loop { + let h11 = h[at(m, m)]; + let h22 = h[at(m + 1, m + 1)]; + let h11s = h11 - t; + let h21 = h[at(m + 1, m)].re(); + let s = abs1(h11s) + h21.abs(); + let (h11s, h21) = (h11s.scale(C::Real::ONE / s), h21 / s); + v = [h11s, C::from_real(h21)]; + if m == l { + break; + } + let h10 = h[at(m, m - 1)].re(); + if h10.abs() * h21.abs() <= ulp * (abs1(h11s) * (abs1(h11) + abs1(h22))) { + break; + } + m -= 1; + } + // The single-shift sweep. + for k in m..i { + if k > m { + v = [h[at(k, k - 1)], h[at(k + 1, k - 1)]]; + } + let (t1, beta) = { + let (first, rest) = v.split_at_mut(1); + short_reflector(first[0], rest) + }; + v[0] = beta; + if k > m { + h[at(k, k - 1)] = v[0]; + h[at(k + 1, k - 1)] = C::zero(); + } + let v2 = v[1]; + let t2 = (t1 * v2).re(); + let (t1c, v2c) = (t1.conj(), v2.conj()); + for col in h[k * n..(i2 + 1) * n].chunks_exact_mut(n) { + let [x, y] = &mut col[k..k + 2] else { + unreachable!() + }; + let sum = t1c * *x + y.scale(t2); + *x -= sum; + *y -= sum * v2; + } + let turn = |a: &mut [C], rows: core::ops::Range| { + let (x, y) = a[k * n..].split_at_mut(n); + for (x, y) in x[rows.clone()].iter_mut().zip(&mut y[rows]) { + let sum = t1 * *x + y.scale(t2); + *x -= sum; + *y -= sum * v2c; + } + }; + turn(h, i1..(k + 2).min(i) + 1); + if let Some(z) = z.as_deref_mut() { + turn(z, 0..n); + } + if k == m && m > l { + // Keep `h[m, m - 1]` real, the sweep having started below it. + let mut temp = C::one() - t1; + temp = temp.scale(C::Real::ONE / temp.abs()); + h[at(m + 1, m)] *= temp.conj(); + if m + 2 <= i { + h[at(m + 2, m + 1)] *= temp; + } + for j in m..=i { + if j != m + 1 { + if i2 > j { + for c in j + 1..=i2 { + h[at(j, c)] *= temp; + } + } + for r in i1..j { + h[at(r, j)] *= temp.conj(); + } + if let Some(z) = z.as_deref_mut() { + for r in 0..n { + z[at(r, j)] *= temp.conj(); + } + } + } + } + } + } + // Keep `h[i, i - 1]` real. + let temp = h[at(i, i - 1)]; + if temp.im() != zero { + let r = temp.abs(); + h[at(i, i - 1)] = C::from_real(r); + let temp = temp.scale(C::Real::ONE / r); + if i2 > i { + for c in i + 1..=i2 { + h[at(i, c)] *= temp.conj(); + } + } + for r in i1..i { + h[at(r, i)] *= temp; + } + if let Some(z) = z.as_deref_mut() { + for r in 0..n { + z[at(r, i)] *= temp; + } + } + } + } + if !converged { + return i + 1; + } + w[i] = h[at(i, i)].to_complex(); + kdefl = 0; + if l == 0 { + break; + } + i = l - 1; + } + 0 +} diff --git a/lib/src/math/matrix/schur/deflate.rs b/lib/src/math/matrix/schur/deflate.rs new file mode 100644 index 0000000..1fa924a --- /dev/null +++ b/lib/src/math/matrix/schur/deflate.rs @@ -0,0 +1,935 @@ +//! Aggressive early deflation, as LAPACK's `laqr3`: the Schur form of a window at the bottom of +//! the active block, the eigenvalues whose part of the spike is negligible deflated, the others +//! moved to the top of the window and given back as shifts. For real elements the reordering +//! moves the `2 × 2` blocks of the real Schur form, as LAPACK's `trexc`, `laexc` and `lasy2`. + +use super::super::Scalar; +use super::super::blas::{Op, gemm}; +use super::super::householder::{Block, PANEL, reflect_left, reflect_right, reflector}; +use super::hessenberg::reduce; +use super::multishift::{multishift, nmin}; +use super::{real_reflector, rotate, small_qr, standardize}; +use crate::math::complex::{Complex, Float}; + +/// Aggressive early deflation on the window of order `nw` at the bottom of the active block +/// `ktop..=kbot` of the upper Hessenberg `h`, as LAPACK's `laqr3` (`laqr2` when `nested`, its +/// window then reduced by the QR algorithm with one or two shifts only): the window's +/// eigenvalues into `w`, those deflated at the bottom. Returns the number of shifts, the +/// undeflated eigenvalues just above the deflated ones, and the number deflated. +#[allow(clippy::too_many_arguments)] +pub(super) fn deflate( + want_t: bool, + h: &mut [T], + n: usize, + z: Option<&mut [T]>, + ktop: usize, + kbot: usize, + nw: usize, + w: &mut [Complex], + nested: bool, +) -> (usize, usize) { + let at = |i: usize, j: usize| i + j * n; + let ulp = T::Real::EPSILON; + let smlnum = T::Real::MIN_POSITIVE * (T::Real::from_f64(n as f64) / ulp); + let jw = nw.min(kbot + 1 - ktop); + if jw == 0 { + return (0, 0); + } + let kwtop = kbot + 1 - jw; + // The spike: the subdiagonal element left of the window, times `V`'s first row. + let mut s = if kwtop == ktop { + T::zero() + } else { + h[at(kwtop, kwtop - 1)] + }; + if kbot == kwtop { + w[kwtop] = h[at(kwtop, kwtop)].to_complex(); + if s.abs1() <= smlnum.max(ulp * h[at(kwtop, kwtop)].abs1()) { + if kwtop > ktop { + h[at(kwtop, kwtop - 1)] = T::zero(); + } + return (0, 1); + } + return (1, 0); + } + let tt = |i: usize, j: usize| i + j * jw; + let mut t = vec![T::zero(); jw * jw]; + for j in 0..jw { + for i in 0..=(j + 1).min(jw - 1) { + t[tt(i, j)] = h[at(kwtop + i, kwtop + j)]; + } + } + let mut v = vec![T::zero(); jw * jw]; + for i in 0..jw { + v[tt(i, i)] = T::one(); + } + let ws = &mut w[kwtop..=kbot]; + let infqr = if jw > nmin::(true) && !nested { + multishift(true, &mut t, jw, Some(&mut v), ws, true) + } else { + small_qr(&mut t, jw, Some(&mut v), true, ws) + }; + for j in 0..jw { + for i in j + 2..jw { + t[tt(i, j)] = T::zero(); + } + } + // Deflation: the eigenvalues at the bottom whose spike is negligible stay, the others go to + // the top, `ns` the undeflated so far and `ilst` where the next one goes. + let mut ns = jw; + let mut ilst = infqr; + if T::IS_COMPLEX { + for _ in infqr..jw { + let mut size = t[tt(ns - 1, ns - 1)].abs1(); + if size == T::Real::ZERO { + size = s.abs1(); + } + if s.abs1() * v[tt(0, ns - 1)].abs1() <= smlnum.max(ulp * size) { + ns -= 1; + } else { + move_triangular(&mut t, jw, &mut v, ns - 1, ilst); + ilst += 1; + } + } + } else { + let re = |t: &[T], i: usize, j: usize| t[tt(i, j)].re(); + let sr = s.re(); + while ilst < ns { + if ns == 1 || re(&t, ns - 1, ns - 2) == T::Real::ZERO { + let mut size = re(&t, ns - 1, ns - 1).abs(); + if size == T::Real::ZERO { + size = sr.abs(); + } + if (sr * v[tt(0, ns - 1)].re()).abs() <= smlnum.max(ulp * size) { + ns -= 1; + } else { + let mut ifst = ns - 1; + reorder(&mut t, jw, &mut v, &mut ifst, &mut ilst); + ilst += 1; + } + } else { + let mut size = re(&t, ns - 1, ns - 1).abs() + + re(&t, ns - 1, ns - 2).abs().sqrt() * re(&t, ns - 2, ns - 1).abs().sqrt(); + if size == T::Real::ZERO { + size = sr.abs(); + } + let spike = (sr * v[tt(0, ns - 1)].re()) + .abs() + .max((sr * v[tt(0, ns - 2)].re()).abs()); + if spike <= smlnum.max(ulp * size) { + ns -= 2; + } else { + let mut ifst = ns - 1; + reorder(&mut t, jw, &mut v, &mut ifst, &mut ilst); + ilst += 2; + } + } + } + } + if ns == 0 { + s = T::zero(); + } + if ns < jw { + // The undeflated eigenvalues largest first, which helps graded matrices. + if T::IS_COMPLEX { + for i in infqr..ns { + let mut ifst = i; + for j in i + 1..ns { + if t[tt(j, j)].abs1() > t[tt(ifst, ifst)].abs1() { + ifst = j; + } + } + if ifst != i { + move_triangular(&mut t, jw, &mut v, ifst, i); + } + } + } else { + sort_real(&mut t, jw, &mut v, infqr, ns); + } + } + // The eigenvalues from `T`. + if T::IS_COMPLEX { + for i in infqr..jw { + w[kwtop + i] = t[tt(i, i)].to_complex(); + } + } else { + let re = |i: usize, j: usize| t[tt(i, j)].re(); + let mut i = jw; + while i > infqr { + if i == infqr + 1 || re(i - 1, i - 2) == T::Real::ZERO { + w[kwtop + i - 1] = Complex::new(re(i - 1, i - 1), T::Real::ZERO); + i -= 1; + } else { + let b = standardize( + re(i - 2, i - 2), + re(i - 2, i - 1), + re(i - 1, i - 2), + re(i - 1, i - 1), + ); + w[kwtop + i - 2] = Complex::new(b.rt1r, b.rt1i); + w[kwtop + i - 1] = Complex::new(b.rt2r, b.rt2i); + i -= 2; + } + } + } + if ns < jw || s == T::zero() { + let reflected = ns > 1 && s != T::zero(); + let mut taus = Vec::new(); + if reflected { + // The spike's undeflated part back to a multiple of the first column, then the + // window's leading block back to Hessenberg form. + let mut x: Vec = (0..ns).map(|j| v[tt(0, j)].conj()).collect(); + let (tau, _) = { + let (first, rest) = x.split_at_mut(1); + reflector(first[0], rest) + }; + x[0] = T::one(); + reflect_left(tau, true, &x, ns, jw, &mut t, jw); + reflect_right(tau, &x, ns, ns, &mut t, jw); + reflect_right(tau, &x, jw, ns, &mut v, jw); + taus = vec![T::zero(); ns.saturating_sub(2)]; + reduce(&mut t, jw, ns, &mut taus); + } + if kwtop > 0 { + h[at(kwtop, kwtop - 1)] = s * v[tt(0, 0)].conj(); + } + for j in 0..jw { + for i in 0..=(j + 1).min(jw - 1) { + h[at(kwtop + i, kwtop + j)] = t[tt(i, j)]; + } + } + if reflected && ns > 2 { + // `V = V·Q`, `Q` the Hessenberg reduction's reflections, a panel at a time. + let k = ns - 2; + let mut start = 0; + while start < k { + let end = (start + PANEL).min(k); + let block = Block::new( + &t[tt(start + 1, start)..], + jw, + ns - 1 - start, + &taus[start..end], + ); + block.apply_right(jw, &mut v[(start + 1) * jw..], jw); + start = end; + } + } + // The rest of `h` and `z`: the columns of the window above it times `V`, its rows to + // the right times `V*`. + let top = if want_t { 0 } else { ktop }; + if top < kwtop { + let rows = kwtop - top; + let mut tmp = vec![T::zero(); rows * jw]; + gemm( + rows, + jw, + jw, + T::one(), + &h[at(top, kwtop)..], + n, + Op::N, + &v, + jw, + Op::N, + T::zero(), + &mut tmp, + rows, + ); + for j in 0..jw { + h[at(top, kwtop + j)..at(kwtop, kwtop + j)] + .copy_from_slice(&tmp[j * rows..(j + 1) * rows]); + } + } + if want_t && kbot + 1 < n { + let cols = n - kbot - 1; + let mut vh = vec![T::zero(); jw * jw]; + for j in 0..jw { + for i in 0..jw { + vh[tt(j, i)] = v[tt(i, j)].conj(); + } + } + let mut tmp = vec![T::zero(); jw * cols]; + gemm( + jw, + cols, + jw, + T::one(), + &vh, + jw, + Op::N, + &h[at(kwtop, kbot + 1)..], + n, + Op::N, + T::zero(), + &mut tmp, + jw, + ); + for j in 0..cols { + h[at(kwtop, kbot + 1 + j)..at(kbot + 1, kbot + 1 + j)] + .copy_from_slice(&tmp[j * jw..(j + 1) * jw]); + } + } + if let Some(z) = z { + let mut tmp = vec![T::zero(); n * jw]; + gemm( + n, + jw, + jw, + T::one(), + &z[at(0, kwtop)..], + n, + Op::N, + &v, + jw, + Op::N, + T::zero(), + &mut tmp, + n, + ); + z[at(0, kwtop)..at(0, kbot + 1)].copy_from_slice(&tmp); + } + } + (ns - infqr, jw - ns) +} + +/// The rotation `[[c, s], [-s̄, c]]`, `c` real, taking `(f, g)` to `(r, 0)`: `(c, s)`. +fn rotation(f: T, g: T) -> (T::Real, T) { + if g == T::zero() { + return (T::Real::ONE, T::zero()); + } + if f == T::zero() { + return (T::Real::ZERO, g.conj().scale(T::Real::ONE / g.abs())); + } + let (af, ag) = (f.abs(), g.abs()); + let d = af.hypot(ag); + ( + af / d, + f.scale(T::Real::ONE / af) * g.conj().scale(T::Real::ONE / d), + ) +} + +/// Swaps the diagonal elements `k` and `k + 1` of the upper triangular `t` (order `n`) by a +/// rotation, accumulated into `q`, as LAPACK's complex `trexc` (and `laexc` for two real +/// `1 × 1` blocks). +fn swap_diagonal(t: &mut [T], n: usize, q: &mut [T], k: usize) { + let at = |i: usize, j: usize| i + j * n; + let (t11, t22) = (t[at(k, k)], t[at(k + 1, k + 1)]); + let (c, s) = rotation(t[at(k, k + 1)], t22 - t11); + let sc = s.conj(); + for j in k + 2..n { + let (x, y) = (t[at(k, j)], t[at(k + 1, j)]); + t[at(k, j)] = x.scale(c) + s * y; + t[at(k + 1, j)] = y.scale(c) - sc * x; + } + let rotate_columns = |a: &mut [T], rows: usize| { + let (x, y) = a[k * n..].split_at_mut(n); + for (x, y) in x[..rows].iter_mut().zip(&mut y[..rows]) { + let (a, b) = (*x, *y); + *x = a.scale(c) + sc * b; + *y = b.scale(c) - s * a; + } + }; + rotate_columns(t, k); + rotate_columns(q, n); + t[at(k, k)] = t22; + t[at(k + 1, k + 1)] = t11; +} + +/// Moves the diagonal element `from` of the upper triangular `t` to `to` by swaps of +/// neighbours. +fn move_triangular(t: &mut [T], n: usize, q: &mut [T], from: usize, to: usize) { + if from > to { + for k in (to..from).rev() { + swap_diagonal(t, n, q, k); + } + } else { + for k in from..to { + swap_diagonal(t, n, q, k); + } + } +} + +/// The undeflated blocks `first..ns` of the real Schur form `t` sorted by decreasing absolute +/// value of the eigenvalues, by a bubble sort, which copes with a swap refused. +fn sort_real(t: &mut [T], n: usize, q: &mut [T], first: usize, ns: usize) { + let sub = |t: &[T], i: usize| t[i + (i - 1) * n].re() != T::Real::ZERO; + // The size of the block at `i` within `..end`. + let size = |t: &[T], i: usize, end: usize| if i + 1 < end && sub(t, i + 1) { 2 } else { 1 }; + let magnitude = |t: &[T], i: usize, nb: usize| { + let d = t[i + i * n].re().abs(); + if nb == 1 { + d + } else { + d + t[i + 1 + i * n].re().abs().sqrt() * t[i + (i + 1) * n].re().abs().sqrt() + } + }; + let mut end = ns; + let mut sorted = false; + while !sorted { + sorted = true; + let mut i = first; + let mut k = i + size(t, i, end); + let mut last = i; + while k < end { + let (ni, nk) = (size(t, i, end), size(t, k, end)); + if magnitude(t, i, ni) >= magnitude(t, k, nk) { + i = k; + } else { + sorted = false; + let (mut ifst, mut ilst) = (i, k); + i = if reorder(t, n, q, &mut ifst, &mut ilst) { + ilst + } else { + k + }; + } + last = i; + k = i + size(t, i, end); + } + end = last; + } +} + +/// Moves the diagonal block of the real Schur form `t` (order `n`) at row `ifst` to row `ilst` +/// by swaps of neighbouring blocks, accumulated into `q`, as LAPACK's `trexc`: both are moved to +/// the first rows of their blocks, and `ilst` ends where the block did. Returns false when a +/// swap was refused, the block then left at `ilst`. +fn reorder( + t: &mut [T], + n: usize, + q: &mut [T], + ifst: &mut usize, + ilst: &mut usize, +) -> bool { + if n <= 1 { + return true; + } + let sub = |t: &[T], i: usize| t[i + (i - 1) * n].re() != T::Real::ZERO; + if *ifst > 0 && sub(t, *ifst) { + *ifst -= 1; + } + let mut nbf = if *ifst + 1 < n && sub(t, *ifst + 1) { + 2 + } else { + 1 + }; + if *ilst > 0 && sub(t, *ilst) { + *ilst -= 1; + } + let nbl = if *ilst + 1 < n && sub(t, *ilst + 1) { + 2 + } else { + 1 + }; + if *ifst == *ilst { + return true; + } + let mut here = *ifst; + if *ifst < *ilst { + if nbf == 2 && nbl == 1 { + *ilst -= 1; + } + if nbf == 1 && nbl == 2 { + *ilst += 1; + } + while here < *ilst { + if nbf != 3 { + let next = if here + nbf + 1 < n && sub(t, here + nbf + 1) { + 2 + } else { + 1 + }; + if !swap_blocks(t, n, q, here, nbf, next) { + *ilst = here; + return false; + } + here += next; + if nbf == 2 && !sub(t, here + 1) { + nbf = 3; + } + } else { + // A block that split in two, moved one at a time. + let mut next = if here + 3 < n && sub(t, here + 3) { + 2 + } else { + 1 + }; + if !swap_blocks(t, n, q, here + 1, 1, next) { + *ilst = here; + return false; + } + if next == 1 { + swap_blocks(t, n, q, here, 1, 1); + here += 1; + } else { + if !sub(t, here + 2) { + next = 1; + } + if next == 2 { + if !swap_blocks(t, n, q, here, 1, 2) { + *ilst = here; + return false; + } + } else { + swap_blocks(t, n, q, here, 1, 1); + swap_blocks(t, n, q, here + 1, 1, 1); + } + here += 2; + } + } + } + } else { + while here > *ilst { + if nbf != 3 { + let next = if here >= 2 && sub(t, here - 1) { 2 } else { 1 }; + if !swap_blocks(t, n, q, here - next, next, nbf) { + *ilst = here; + return false; + } + here -= next; + if nbf == 2 && !sub(t, here + 1) { + nbf = 3; + } + } else { + let mut next = if here >= 2 && sub(t, here - 1) { 2 } else { 1 }; + if !swap_blocks(t, n, q, here - next, next, 1) { + *ilst = here; + return false; + } + if next == 1 { + swap_blocks(t, n, q, here, 1, 1); + here -= 1; + } else { + if !sub(t, here) { + next = 1; + } + if next == 2 { + if !swap_blocks(t, n, q, here - 1, 2, 1) { + *ilst = here; + return false; + } + } else { + swap_blocks(t, n, q, here, 1, 1); + swap_blocks(t, n, q, here - 1, 1, 1); + } + here -= 2; + } + } + } + } + *ilst = here; + true +} + +/// `I - τ·u·uᵀ` on three rows `r..r + 3` of columns `cols` (from the left) or on three columns +/// `r..r + 3` of rows `cols` (from the right) of `t`, of real elements. +fn reflect3( + t: &mut [T], + n: usize, + left: bool, + r: usize, + cols: core::ops::Range, + u: [T::Real; 3], + tau: T::Real, +) { + let (s, step) = if left { (n, 1) } else { (1, n) }; + for j in cols { + let i0 = r * step + j * s; + let (x0, x1, x2) = (t[i0], t[i0 + step], t[i0 + 2 * step]); + let sum = x0.scale(u[0]) + x1.scale(u[1]) + x2.scale(u[2]); + t[i0] = x0 - sum.scale(tau * u[0]); + t[i0 + step] = x1 - sum.scale(tau * u[1]); + t[i0 + 2 * step] = x2 - sum.scale(tau * u[2]); + } +} + +/// [`reflect3`] on a small dense real block of order 4. +fn reflect3_small( + d: &mut [[R; 4]; 4], + left: bool, + r: usize, + cols: core::ops::Range, + u: [R; 3], + tau: R, +) { + for j in cols { + let get = |d: &[[R; 4]; 4], p: usize| if left { d[r + p][j] } else { d[j][r + p] }; + let sum = get(d, 0) * u[0] + get(d, 1) * u[1] + get(d, 2) * u[2]; + for (p, &u) in u.iter().enumerate() { + let x = if left { + &mut d[r + p][j] + } else { + &mut d[j][r + p] + }; + *x -= sum * tau * u; + } + } +} + +/// Swaps the neighbouring diagonal blocks of orders `n1` and `n2` at row `j1` of the real Schur +/// form `t` (order `n`), accumulated into `q`, as LAPACK's `laexc`, the new blocks of order 2 +/// standardized. Returns false, leaving everything as it was, when the swap would change the +/// eigenvalues too much. +fn swap_blocks( + t: &mut [T], + n: usize, + q: &mut [T], + j1: usize, + n1: usize, + n2: usize, +) -> bool { + let at = |i: usize, j: usize| i + j * n; + if n1 == 0 || n2 == 0 || j1 + n1 >= n { + return true; + } + if n1 == 1 && n2 == 1 { + swap_diagonal(t, n, q, j1); + return true; + } + let (j2, j3, j4) = (j1 + 1, j1 + 2, j1 + 3); + let nd = n1 + n2; + let mut d = [[T::Real::ZERO; 4]; 4]; + let mut dnorm = T::Real::ZERO; + for (i, row) in d.iter_mut().enumerate().take(nd) { + for (j, x) in row.iter_mut().enumerate().take(nd) { + *x = t[at(j1 + i, j1 + j)].re(); + dnorm = dnorm.max(x.abs()); + } + } + let eps = T::Real::EPSILON; + let smlnum = T::Real::MIN_POSITIVE / eps; + let thresh = (T::Real::from_f64(10.0) * eps * dnorm).max(smlnum); + // `T₁₁·X - X·T₂₂ = scale·T₁₂`. + let block = |r: usize, c: usize| [[d[r][c], d[r][c + 1]], [d[r + 1][c], d[r + 1][c + 1]]]; + let (x, scale) = sylvester(n1, n2, block(0, 0), block(n1, n1), block(0, n1)); + let one = T::Real::ONE; + if n1 == 1 { + // `(scale, x₁₁, x₁₂)·H = (0, 0, *)`. + let mut u = [scale, x[0][0], x[0][1]]; + let (tau, _) = { + let (head, last) = u.split_at_mut(2); + real_reflector(last[0], head) + }; + u[2] = one; + let t11 = t[at(j1, j1)]; + reflect3_small(&mut d, true, 0, 0..3, u, tau); + reflect3_small(&mut d, false, 0, 0..3, u, tau); + if d[2][0] + .abs() + .max(d[2][1].abs()) + .max((d[2][2] - t11.re()).abs()) + > thresh + { + return false; + } + reflect3(t, n, true, j1, j1..n, u, tau); + reflect3(t, n, false, j1, 0..j2 + 1, u, tau); + t[at(j3, j1)] = T::zero(); + t[at(j3, j2)] = T::zero(); + t[at(j3, j3)] = t11; + reflect3(q, n, false, j1, 0..n, u, tau); + } else if n2 == 1 { + // `H·(-x₁₁, -x₂₁, scale) = (*, 0, 0)`. + let mut u = [-x[0][0], -x[1][0], scale]; + let (tau, _) = { + let (first, rest) = u.split_at_mut(1); + real_reflector(first[0], rest) + }; + u[0] = one; + let t33 = t[at(j3, j3)]; + reflect3_small(&mut d, true, 0, 0..3, u, tau); + reflect3_small(&mut d, false, 0, 0..3, u, tau); + if d[1][0] + .abs() + .max(d[2][0].abs()) + .max((d[0][0] - t33.re()).abs()) + > thresh + { + return false; + } + reflect3(t, n, false, j1, 0..j3 + 1, u, tau); + reflect3(t, n, true, j1, j2..n, u, tau); + t[at(j1, j1)] = t33; + t[at(j2, j1)] = T::zero(); + t[at(j3, j1)] = T::zero(); + reflect3(q, n, false, j1, 0..n, u, tau); + } else { + // Two reflections taking `(-X; scale·I)` to upper triangular. + let mut u1 = [-x[0][0], -x[1][0], scale]; + let (tau1, _) = { + let (first, rest) = u1.split_at_mut(1); + real_reflector(first[0], rest) + }; + u1[0] = one; + let temp = -tau1 * (x[0][1] + u1[1] * x[1][1]); + let mut u2 = [-temp * u1[1] - x[1][1], -temp * u1[2], scale]; + let (tau2, _) = { + let (first, rest) = u2.split_at_mut(1); + real_reflector(first[0], rest) + }; + u2[0] = one; + reflect3_small(&mut d, true, 0, 0..4, u1, tau1); + reflect3_small(&mut d, false, 0, 0..4, u1, tau1); + reflect3_small(&mut d, true, 1, 0..4, u2, tau2); + reflect3_small(&mut d, false, 1, 0..4, u2, tau2); + let worst = d[2][0] + .abs() + .max(d[2][1].abs()) + .max(d[3][0].abs()) + .max(d[3][1].abs()); + if worst > thresh { + return false; + } + reflect3(t, n, true, j1, j1..n, u1, tau1); + reflect3(t, n, false, j1, 0..j4 + 1, u1, tau1); + reflect3(t, n, true, j2, j1..n, u2, tau2); + reflect3(t, n, false, j2, 0..j4 + 1, u2, tau2); + t[at(j3, j1)] = T::zero(); + t[at(j3, j2)] = T::zero(); + t[at(j4, j1)] = T::zero(); + t[at(j4, j2)] = T::zero(); + reflect3(q, n, false, j1, 0..n, u1, tau1); + reflect3(q, n, false, j2, 0..n, u2, tau2); + } + if n2 == 2 { + standardize_block(t, n, q, j1); + } + if n1 == 2 { + standardize_block(t, n, q, j1 + n2); + } + true +} + +/// The `2 × 2` block at row `j` of `t` in standard form, the rotation applied to the rest of +/// `t` and to `q`. +fn standardize_block(t: &mut [T], n: usize, q: &mut [T], j: usize) { + let at = |i: usize, k: usize| i + k * n; + let re = |t: &[T], i: usize, k: usize| t[at(i, k)].re(); + let s = standardize( + re(t, j, j), + re(t, j, j + 1), + re(t, j + 1, j), + re(t, j + 1, j + 1), + ); + t[at(j, j)] = T::from_real(s.a); + t[at(j, j + 1)] = T::from_real(s.b); + t[at(j + 1, j)] = T::from_real(s.c); + t[at(j + 1, j + 1)] = T::from_real(s.d); + let turn = |x: &mut T, y: &mut T| { + let (mut a, mut b) = (x.re(), y.re()); + rotate(&mut a, &mut b, s.cs, s.sn); + *x = T::from_real(a); + *y = T::from_real(b); + }; + for k in j + 2..n { + let (mut x, mut y) = (t[at(j, k)], t[at(j + 1, k)]); + turn(&mut x, &mut y); + t[at(j, k)] = x; + t[at(j + 1, k)] = y; + } + for (a, rows) in [(&mut *t, j), (&mut *q, n)] { + let (x, y) = a[j * n..].split_at_mut(n); + for (x, y) in x[..rows].iter_mut().zip(&mut y[..rows]) { + turn(x, y); + } + } +} + +/// The solution `X` of `TL·X - X·TR = scale·B` for `TL` of order `n1` and `TR` of order `n2`, +/// each 1 or 2, as LAPACK's `lasy2`: Gaussian elimination with complete pivoting, the pivots +/// too small replaced, `scale ≤ 1` keeping `X` from overflowing. +#[allow(clippy::type_complexity)] +fn sylvester( + n1: usize, + n2: usize, + tl: [[R; 2]; 2], + tr: [[R; 2]; 2], + b: [[R; 2]; 2], +) -> ([[R; 2]; 2], R) { + let (zero, one) = (R::ZERO, R::ONE); + let eps = R::EPSILON; + let smlnum = R::MIN_POSITIVE / eps; + let sgn = -one; + let mut x = [[zero; 2]; 2]; + if n1 == 1 && n2 == 1 { + let mut tau1 = tl[0][0] + sgn * tr[0][0]; + let mut bet = tau1.abs(); + if bet <= smlnum { + tau1 = smlnum; + bet = smlnum; + } + let mut scale = one; + let gam = b[0][0].abs(); + if smlnum * gam > bet { + scale = one / gam; + } + x[0][0] = (b[0][0] * scale) / tau1; + return (x, scale); + } + if n1 + n2 == 3 { + // A 2 × 2 system, `tmp` column after column. + let (smin, tmp, mut btmp) = if n1 == 1 { + let big = tl[0][0] + .abs() + .max(tr[0][0].abs()) + .max(tr[0][1].abs()) + .max(tr[1][0].abs()) + .max(tr[1][1].abs()); + let tmp = [ + tl[0][0] + sgn * tr[0][0], + sgn * tr[0][1], + sgn * tr[1][0], + tl[0][0] + sgn * tr[1][1], + ]; + ((eps * big).max(smlnum), tmp, [b[0][0], b[0][1]]) + } else { + let big = tr[0][0] + .abs() + .max(tl[0][0].abs()) + .max(tl[0][1].abs()) + .max(tl[1][0].abs()) + .max(tl[1][1].abs()); + let tmp = [ + tl[0][0] + sgn * tr[0][0], + tl[1][0], + tl[0][1], + tl[1][1] + sgn * tr[0][0], + ]; + ((eps * big).max(smlnum), tmp, [b[0][0], b[1][0]]) + }; + const U12: [usize; 4] = [2, 3, 0, 1]; + const L21: [usize; 4] = [1, 0, 3, 2]; + const U22: [usize; 4] = [3, 2, 1, 0]; + const XSWAP: [bool; 4] = [false, false, true, true]; + const BSWAP: [bool; 4] = [false, true, false, true]; + let mut ipiv = 0; + for i in 1..4 { + if tmp[i].abs() > tmp[ipiv].abs() { + ipiv = i; + } + } + let mut u11 = tmp[ipiv]; + if u11.abs() <= smin { + u11 = smin; + } + let u12 = tmp[U12[ipiv]]; + let l21 = tmp[L21[ipiv]] / u11; + let mut u22 = tmp[U22[ipiv]] - u12 * l21; + if u22.abs() <= smin { + u22 = smin; + } + if BSWAP[ipiv] { + let temp = btmp[1]; + btmp[1] = btmp[0] - l21 * temp; + btmp[0] = temp; + } else { + btmp[1] -= l21 * btmp[0]; + } + let mut scale = one; + let two = R::TWO; + if two * smlnum * btmp[1].abs() > u22.abs() || two * smlnum * btmp[0].abs() > u11.abs() { + scale = R::HALF / btmp[0].abs().max(btmp[1].abs()); + btmp[0] *= scale; + btmp[1] *= scale; + } + let mut x2 = [zero; 2]; + x2[1] = btmp[1] / u22; + x2[0] = btmp[0] / u11 - (u12 / u11) * x2[1]; + if XSWAP[ipiv] { + x2.swap(0, 1); + } + x[0][0] = x2[0]; + if n1 == 1 { + x[0][1] = x2[1]; + } else { + x[1][0] = x2[1]; + } + return (x, scale); + } + // 2 × 2 by 2 × 2: the equivalent system of order 4. + let mut smin = zero; + for row in tr.iter().chain(tl.iter()) { + for v in row { + smin = smin.max(v.abs()); + } + } + let smin = (eps * smin).max(smlnum); + let mut t16 = [[zero; 4]; 4]; + t16[0][0] = tl[0][0] + sgn * tr[0][0]; + t16[1][1] = tl[1][1] + sgn * tr[0][0]; + t16[2][2] = tl[0][0] + sgn * tr[1][1]; + t16[3][3] = tl[1][1] + sgn * tr[1][1]; + t16[0][1] = tl[0][1]; + t16[1][0] = tl[1][0]; + t16[2][3] = tl[0][1]; + t16[3][2] = tl[1][0]; + t16[0][2] = sgn * tr[1][0]; + t16[1][3] = sgn * tr[1][0]; + t16[2][0] = sgn * tr[0][1]; + t16[3][1] = sgn * tr[0][1]; + let mut btmp = [b[0][0], b[1][0], b[0][1], b[1][1]]; + let mut jpiv = [0usize; 4]; + for i in 0..3 { + let mut xmax = zero; + let (mut ipsv, mut jpsv) = (i, i); + for (ip, row) in t16.iter().enumerate().skip(i) { + for (jp, v) in row.iter().enumerate().skip(i) { + if v.abs() >= xmax { + xmax = v.abs(); + ipsv = ip; + jpsv = jp; + } + } + } + if ipsv != i { + t16.swap(ipsv, i); + btmp.swap(ipsv, i); + } + if jpsv != i { + for row in &mut t16 { + row.swap(jpsv, i); + } + } + jpiv[i] = jpsv; + if t16[i][i].abs() < smin { + t16[i][i] = smin; + } + for j in i + 1..4 { + t16[j][i] /= t16[i][i]; + btmp[j] -= t16[j][i] * btmp[i]; + for k in i + 1..4 { + t16[j][k] -= t16[j][i] * t16[i][k]; + } + } + } + if t16[3][3].abs() < smin { + t16[3][3] = smin; + } + let mut scale = one; + let eight = R::from_f64(8.0); + if (0..4).any(|i| eight * smlnum * btmp[i].abs() > t16[i][i].abs()) { + let big = btmp.iter().fold(zero, |m, v| m.max(v.abs())); + scale = (one / eight) / big; + for v in &mut btmp { + *v *= scale; + } + } + let mut tmp = [zero; 4]; + for k in (0..4).rev() { + let temp = one / t16[k][k]; + tmp[k] = btmp[k] * temp; + for j in k + 1..4 { + tmp[k] -= (temp * t16[k][j]) * tmp[j]; + } + } + for k in (0..3).rev() { + if jpiv[k] != k { + tmp.swap(k, jpiv[k]); + } + } + x[0][0] = tmp[0]; + x[1][0] = tmp[1]; + x[0][1] = tmp[2]; + x[1][1] = tmp[3]; + (x, scale) +} diff --git a/lib/src/math/matrix/schur/hessenberg.rs b/lib/src/math/matrix/schur/hessenberg.rs new file mode 100644 index 0000000..91fbbaf --- /dev/null +++ b/lib/src/math/matrix/schur/hessenberg.rs @@ -0,0 +1,416 @@ +//! The reduction to upper Hessenberg form by Householder reflections, blocked as LAPACK's +//! `gehrd`: a panel of reflections at a time, built with `A·V·T` as LAPACK's `lahr2`, so that +//! most of the work on the rest of the matrix is products of blocks. + +use super::super::blas::{Op, gemm}; +use super::super::householder::{form_q, reflect_left, reflect_right, reflector}; +use super::super::{Matrix, Scalar}; + +/// The columns of a panel. +const PANEL: usize = 32; + +/// The last columns, reduced one at a time: there the panels' products are too small to pay. +const UNBLOCKED: usize = 32; + +/// Reduces `a` to upper Hessenberg form `H = Q*·A·Q` in place, and gives `Q` when `want_q` is +/// set. +pub(crate) fn hessenberg(a: &mut Matrix, want_q: bool) -> Option> { + let n = a.rows(); + let mut taus = vec![T::zero(); n.saturating_sub(2)]; + reduce(&mut a.data, n, n, &mut taus); + let q = want_q.then(|| { + let mut q = Matrix::identity(n); + if n > 2 { + let sub = form_q(&a.data[1..], n, n - 1, &taus, n - 1); + for j in 0..n - 1 { + q.data[(j + 1) * n + 1..(j + 2) * n].copy_from_slice(sub.column(j)); + } + } + q + }); + for j in 0..n.saturating_sub(2) { + a.data[j * n + j + 2..(j + 1) * n].fill(T::zero()); + } + q +} + +/// Reduces the leading `ihi × ihi` block of the `n × n` matrix `a` to upper Hessenberg form, the +/// rows below it taken to be zero in its columns: `A = Q·H·Q*` with `Q = H₁·H₂·…·H_{ihi-2}`, +/// each reflection's factor in `taus` and its vector below the subdiagonal of its column, the +/// columns after `ihi` multiplied by `Q*` from the left. +pub(crate) fn reduce(a: &mut [T], n: usize, ihi: usize, taus: &mut [T]) { + let mut p = 0; + let mut t = vec![T::zero(); PANEL * PANEL]; + let mut y = Vec::new(); + while p + UNBLOCKED + 2 <= ihi { + let ib = PANEL.min(ihi - p - 1); + y.resize(ihi * ib, T::zero()); + panel(a, n, ihi, p, ib, &mut taus[p..p + ib], &mut t, &mut y); + // `A·(I - V·T·V*) = A - Y·V*`: the columns after the panel in one product, the vectors' + // last unit element put in for it. + let e = a[p + ib + (p + ib - 1) * n]; + a[p + ib + (p + ib - 1) * n] = T::one(); + let (v, rest) = a.split_at_mut((p + ib) * n); + let v = &v[p + ib + p * n..]; + gemm( + ihi, + ihi - p - ib, + ib, + -T::one(), + &y, + ihi, + Op::N, + v, + n, + Op::C, + T::one(), + rest, + n, + ); + a[p + ib + (p + ib - 1) * n] = e; + // The panel's own columns above the reflections' rows: `Y·V₁*`, `V₁` unit lower + // triangular. + for j in 0..ib - 1 { + for l in 0..=j { + let f = if l == j { + T::one() + } else { + a[p + 1 + j + (p + l) * n].conj() + }; + let column = &mut a[(p + 1 + j) * n..(p + 1 + j) * n + p + 1]; + for (x, y) in column.iter_mut().zip(&y[l * ihi..l * ihi + p + 1]) { + *x -= *y * f; + } + } + } + // `(I - V·T·V*)*` from the left on the rows of the reflections, the columns after. + let at = p + 1 + p * n; + let (v, c) = a.split_at_mut((p + ib) * n); + block_left( + &v[at..], + n, + ihi - p - 1, + ib, + &t, + PANEL, + &mut c[p + 1..], + n, + n - p - ib, + ); + p += ib; + } + // The rest one reflection at a time, as LAPACK's `gehd2`. + for j in p..ihi.saturating_sub(2) { + let (tau, beta) = { + let alpha = a[j + 1 + j * n]; + reflector(alpha, &mut a[j * n + j + 2..j * n + ihi]) + }; + a[j + 1 + j * n] = T::one(); + taus[j] = tau; + let (left, right) = a.split_at_mut((j + 1) * n); + let v = &left[j * n + j + 1..j * n + ihi]; + // `A·H` on the rows of the block, then `H*·A` on the rows of the reflection. + reflect_right(tau, v, ihi, ihi - j - 1, right, n); + reflect_left(tau, true, v, ihi - j - 1, n - j - 1, &mut right[j + 1..], n); + a[j + 1 + j * n] = beta; + } +} + +/// `C = (I - V·T·V*)*·C = C - V·T*·V*·C` for the `m × cols` block `c`: `V` of `m × k` unit +/// lower trapezoidal, its vectors below the diagonal of `v`, and `T` of `k × k` upper +/// triangular. The conjugate transposes are made explicitly, so that each product takes its +/// blocks as they are. +#[allow(clippy::too_many_arguments)] +pub(super) fn block_left( + v: &[T], + ldv: usize, + m: usize, + k: usize, + t: &[T], + ldt: usize, + c: &mut [T], + ldc: usize, + cols: usize, +) { + if cols == 0 || k == 0 { + return; + } + let mut vh = vec![T::zero(); k * m]; + let mut full = vec![T::zero(); m * k]; + for j in 0..k { + vh[j + j * k] = T::one(); + full[j + j * m] = T::one(); + for i in j + 1..m { + let x = v[i + j * ldv]; + vh[j + i * k] = x.conj(); + full[i + j * m] = x; + } + } + let mut th = vec![T::zero(); k * k]; + for j in 0..k { + for l in 0..=j { + th[j + l * k] = t[l + j * ldt].conj(); + } + } + let mut w = vec![T::zero(); k * cols]; + gemm( + k, + cols, + m, + T::one(), + &vh, + k, + Op::N, + c, + ldc, + Op::N, + T::zero(), + &mut w, + k, + ); + let mut tw = vec![T::zero(); k * cols]; + gemm( + k, + cols, + k, + T::one(), + &th, + k, + Op::N, + &w, + k, + Op::N, + T::zero(), + &mut tw, + k, + ); + gemm( + m, + cols, + k, + -T::one(), + &full, + m, + Op::N, + &tw, + k, + Op::N, + T::one(), + c, + ldc, + ); +} + +/// Reduces the `nb` columns of `a` from column `p`, rows below `ihi` zero, as LAPACK's `lahr2`: +/// the reflections `I - V·T·V*` that zero each column below its subdiagonal, applied to the +/// panel only, with `T` (`nb × nb`, upper triangular, columns `PANEL` apart) and +/// `Y = A·V·T` (`ihi × nb`) for the rest. The vectors' last unit element is left as `β`. +#[allow(clippy::too_many_arguments)] +fn panel( + a: &mut [T], + n: usize, + ihi: usize, + p: usize, + nb: usize, + taus: &mut [T], + t: &mut [T], + y: &mut [T], +) { + // The reflections' rows start at `k`; `Y` and `T` are `ihi` and `PANEL` apart. + let k = p + 1; + let ld = ihi; + let at = |i: usize, j: usize| i + j * n; + let mut e = T::zero(); + let mut w = vec![T::zero(); nb]; + for i in 0..nb { + let col = p + i; + if i > 0 { + // The column of `A - Y·V*`, rows `k..`. + let w = &mut w[..i]; + for (j, w) in w.iter_mut().enumerate() { + *w = -a[at(k + i - 1, p + j)].conj(); + } + gemv(&mut a[at(k, col)..at(ihi, col)], &y[k..], ld, w); + // Then `(I - V·T·V*)*` from the left: `w = T*·V*·b`, `b -= V·w`, with `V₁` the + // unit lower triangle of rows `k..k + i` and `V₂` the rows below. + w.copy_from_slice(&a[at(k, col)..at(k + i, col)]); + for j in 0..i { + let mut s = w[j]; + for r in j + 1..i { + s += a[at(k + r, p + j)].conj() * w[r]; + } + w[j] = s; + } + gemv_c(w, &a[at(k + i, p)..], n, &a[at(k + i, col)..at(ihi, col)]); + for j in (0..i).rev() { + let mut s = T::zero(); + for l in 0..=j { + s += t[l + j * PANEL].conj() * w[l]; + } + w[j] = -s; + } + let (left, right) = a.split_at_mut(at(0, col)); + gemv(&mut right[k + i..ihi], &left[at(k + i, p)..], n, w); + for r in (0..i).rev() { + let mut s = w[r]; + for j in 0..r { + s += left[at(k + r, p + j)] * w[j]; + } + w[r] = s; + } + for (b, w) in right[k..k + i].iter_mut().zip(w.iter()) { + *b += *w; + } + a[at(k + i - 1, p + i - 1)] = e; + } + let (tau, beta) = { + let alpha = a[at(k + i, col)]; + reflector(alpha, &mut a[at(k + i + 1, col)..at(ihi, col)]) + }; + taus[i] = tau; + e = beta; + a[at(k + i, col)] = T::one(); + // `Y[k.., i] = A[k.., col + 1..]·v`, the vector's element `c` paired with column `c`. + let (left, right) = a.split_at_mut(at(0, col + 1)); + let v = &left[at(k + i, col)..at(ihi, col)]; + let yi = &mut y[k + i * ld..ihi + i * ld]; + yi.fill(T::zero()); + gemv(yi, &right[k..], n, v); + // `T[0..i, i] = V₂*·v`, then `Y[k.., i] = τ·(Y[k.., i] - Y[k.., 0..i]·T[0..i, i])`. + let ti = &mut t[i * PANEL..i * PANEL + i]; + ti.fill(T::zero()); + gemv_c(ti, &left[at(k + i, p)..], n, v); + let w = &mut w[..i]; + for (w, t) in w.iter_mut().zip(ti.iter()) { + *w = -*t; + } + let (done, yi) = y.split_at_mut(i * ld); + let yi = &mut yi[k..ihi]; + if i > 0 { + gemv(yi, &done[k..], ld, w); + } + for y in yi.iter_mut() { + *y *= tau; + } + // `T[0..i, i] = -τ·T[0..i, 0..i]·T[0..i, i]`. + for j in 0..i { + let mut s = T::zero(); + for l in j..i { + s += t[j + l * PANEL] * t[l + i * PANEL]; + } + t[j + i * PANEL] = -(tau * s); + } + t[i + i * PANEL] = tau; + } + a[at(k + nb - 1, p + nb - 1)] = e; + // `Y[0..k, ..] = A[0..k, k..]·V·T`: `V₁` from the panel's rows `k..k + nb`, `V₂` below. + for j in 0..nb { + y[j * ld..j * ld + k].copy_from_slice(&a[at(0, k + j)..at(k, k + j)]); + } + for j in 0..nb { + for r in j + 1..nb { + let f = a[at(k + r, p + j)]; + let (left, right) = y.split_at_mut(r * ld); + for (x, y) in left[j * ld..j * ld + k].iter_mut().zip(&right[..k]) { + *x += *y * f; + } + } + } + if ihi > k + nb { + gemm( + k, + nb, + ihi - k - nb, + T::one(), + &a[at(0, k + nb)..], + n, + Op::N, + &a[at(k + nb, p)..], + n, + Op::N, + T::one(), + y, + ld, + ); + } + for j in (0..nb).rev() { + let f = t[j + j * PANEL]; + for x in &mut y[j * ld..j * ld + k] { + *x *= f; + } + for l in 0..j { + let f = t[l + j * PANEL]; + let (left, right) = y.split_at_mut(j * ld); + for (x, y) in right[..k].iter_mut().zip(&left[l * ld..l * ld + k]) { + *x += *y * f; + } + } + } +} + +/// `y += A·x`, for `A` of `y.len() × x.len()` with the leading dimension `lda`, four columns at +/// a time so that each pass over `y` does four multiply-adds. +pub(super) fn gemv(y: &mut [T], a: &[T], lda: usize, x: &[T]) { + let m = y.len(); + let mut c = 0; + while c + 4 <= x.len() { + let (x0, x1, x2, x3) = (x[c], x[c + 1], x[c + 2], x[c + 3]); + let a0 = &a[c * lda..c * lda + m]; + let a1 = &a[(c + 1) * lda..(c + 1) * lda + m]; + let a2 = &a[(c + 2) * lda..(c + 2) * lda + m]; + let a3 = &a[(c + 3) * lda..(c + 3) * lda + m]; + for i in 0..m { + y[i] += a0[i] * x0 + a1[i] * x1 + a2[i] * x2 + a3[i] * x3; + } + c += 4; + } + while c < x.len() { + let x0 = x[c]; + for (y, a) in y.iter_mut().zip(&a[c * lda..c * lda + m]) { + *y += *a * x0; + } + c += 1; + } +} + +/// `w += A*·x`, for `A` of `x.len() × w.len()` with the leading dimension `lda`: dot products +/// four columns at a time, each in four partial sums so that they don't wait on each other. +pub(super) fn gemv_c(w: &mut [T], a: &[T], lda: usize, x: &[T]) { + let m = x.len(); + let body = m - m % 4; + let mut j = 0; + while j + 4 <= w.len() { + let cols: [&[T]; 4] = core::array::from_fn(|c| &a[(j + c) * lda..(j + c) * lda + m]); + let mut s = [[T::zero(); 4]; 4]; + for r in (0..body).step_by(4) { + for (s, col) in s.iter_mut().zip(&cols) { + for l in 0..4 { + s[l] += col[r + l].conj() * x[r + l]; + } + } + } + for (c, (s, col)) in s.iter().zip(&cols).enumerate() { + let mut t = (s[0] + s[1]) + (s[2] + s[3]); + for r in body..m { + t += col[r].conj() * x[r]; + } + w[j + c] += t; + } + j += 4; + } + while j < w.len() { + let col = &a[j * lda..j * lda + m]; + let mut s = [T::zero(); 4]; + for r in (0..body).step_by(4) { + for l in 0..4 { + s[l] += col[r + l].conj() * x[r + l]; + } + } + let mut t = (s[0] + s[1]) + (s[2] + s[3]); + for r in body..m { + t += col[r].conj() * x[r]; + } + w[j] += t; + j += 1; + } +} diff --git a/lib/src/math/matrix/schur/multishift.rs b/lib/src/math/matrix/schur/multishift.rs new file mode 100644 index 0000000..a9744a4 --- /dev/null +++ b/lib/src/math/matrix/schur/multishift.rs @@ -0,0 +1,748 @@ +//! The small-bulge multishift QR algorithm with aggressive early deflation of Braman, Byers and +//! Mathias, as LAPACK's `laqr0` and `laqr5`: each sweep chases a chain of many bulges, two +//! shifts each, down the diagonal, a few columns at a time, the reflections of each step +//! gathered into a small orthogonal (unitary) matrix and applied to the rest of the matrix and +//! to `Z` as products of blocks. One code serves real and complex elements: for real ones the +//! shifts come in pairs of conjugates or of reals, so that the arithmetic stays real. + +use super::super::Scalar; +use super::super::blas::{Op, gemm}; +use super::deflate::deflate; +use super::{short_reflector, small_qr, standardize}; +use crate::math::complex::{Complex, Float}; + +/// Below this order the QR algorithm with one or two shifts is used, LAPACK's `nmin` of 75: +/// measured on Apple's M4, 120 (for complex elements with `Z`, 90), as Francis's double shifts +/// for real elements and the single shifts without `Z` keep up to there. +pub(super) fn nmin(want_z: bool) -> usize { + if T::IS_COMPLEX && want_z { 90 } else { 120 } +} + +/// A sweep is skipped when the deflation window deflated more than this percentage of itself. +const NIBBLE: usize = 14; + +/// After this many iterations without a deflation the deflation window grows. +const KEXNW: usize = 5; + +/// Every this many iterations without a deflation the shifts are exceptional ones. +const KEXSH: usize = 6; + +/// Sweeps on matrices of at least this order gather their reflections into a block, which +/// pays for small ones too when the products by `Z` are long. +const ACCUMULATE: usize = 200; + +/// The number of shifts of a sweep and the order of the deflation window, for a matrix of +/// order `n`, with or without `Z`: LAPACK's `iparmq` choices up to order 150, beyond it +/// `n/16` shifts, at most `iparmq`'s or, without `Z`, half of them, and windows half as +/// large again. Measured on Apple's M4, fewer shifts than `iparmq`'s (tuned for older machines) +/// pay, except where the products by `Z` are long. +fn parameters(n: usize, want_z: bool) -> (usize, usize) { + let lapack = if n >= 6000 { + 256 + } else if n >= 3000 { + 128 + } else if n >= 590 { + 64 + } else if n >= 150 { + 10.max(n / (n as f64).log2().round() as usize) + } else if n >= 60 { + 10 + } else if n >= 30 { + 4 + } else { + 2 + }; + let ns = if n < 150 { + lapack + } else { + (n / 16) + .min(if want_z { lapack } else { lapack / 2 }) + .max(10) + }; + let ns = (ns - ns % 2).max(2); + let nw = if n < 150 { ns } else { 3 * ns / 2 }; + (ns, nw) +} + +/// `|re| + |im|` of a shift. +fn abs1(x: Complex) -> R { + x.re.abs() + x.im.abs() +} + +/// The Schur form of the upper Hessenberg `h` of order `n`, as LAPACK's `laqr0`, or `laqr4` +/// when `nested`, whose deflation windows don't recurse again: the eigenvalues into `w`, `h` +/// turned into the Schur form when `want_t` is set (otherwise only the eigenvalues are right), +/// and the transformations accumulated into the columns of `z`, of order `n` too, when given. +/// Returns the number of leading eigenvalues it didn't converge to, zero on success. +pub(super) fn multishift( + want_t: bool, + h: &mut [T], + n: usize, + mut z: Option<&mut [T]>, + w: &mut [Complex], + nested: bool, +) -> usize { + let nmin = nmin::(z.is_some()); + if n < nmin { + return small_qr(h, n, z, want_t, w); + } + let at = |i: usize, j: usize| i + j * n; + let (ns, nwr) = parameters(n, z.is_some()); + let nwr = nwr.max(2).min((n - 1) / 3); + let nsr = ns.min((n - 3) / 6); + let nsr = (nsr - nsr % 2).max(2); + let nwmax = (n - 1) / 3; + let nsmax = (n - 3) / 6; + let nsmax = nsmax - nsmax % 2; + let accumulate = z.is_some() || n >= ACCUMULATE; + let itmax = 30.max(2 * KEXSH) * n; + let mut nw = nwmax; + let mut ndfl = 1; + let mut ndec: isize = -1; + // The active block ends before `end`. + let mut end = n; + for _ in 0..itmax { + if end == 0 { + return 0; + } + let kbot = end - 1; + let mut ktop = kbot; + while ktop > 0 && h[at(ktop, ktop - 1)] != T::zero() { + ktop -= 1; + } + // The deflation window: the recommended size, or after a while without deflations + // larger and then gradually smaller. + let nh = kbot - ktop + 1; + let nwupbd = nh.min(nwmax); + nw = if ndfl < KEXNW { + nwupbd.min(nwr) + } else { + nwupbd.min(2 * nw) + }; + if nw < nwmax { + if nw + 1 >= nh { + nw = nh; + } else { + let kwtop = kbot + 1 - nw; + if h[at(kwtop, kwtop - 1)].abs1() > h[at(kwtop - 1, kwtop - 2)].abs1() { + nw += 1; + } + } + } + if ndfl < KEXNW { + ndec = -1; + } else if ndec >= 0 || nw >= nwupbd { + ndec += 1; + if (nw as isize) - ndec < 2 { + ndec = 0; + } + nw -= ndec as usize; + } + let (ls, ld) = deflate(want_t, h, n, z.as_deref_mut(), ktop, kbot, nw, w, nested); + end -= ld; + // A sweep, unless the deflations were many, or the rest is small. + if ld == 0 || (100 * ld <= nw * NIBBLE && end - ktop > nmin.min(nwmax)) { + let kbot = end - 1; + let mut ks = end - ls; + let mut ns = nsmax.min(nsr).min(2.max(kbot - ktop)); + ns -= ns % 2; + if ndfl % KEXSH == 0 { + exceptional(h, n, ktop, kbot, ns, w); + ks = kbot + 1 - ns; + } else { + if kbot + 1 - ks <= ns / 2 { + // Too few shifts from the window: the eigenvalues of the trailing block. + ks = kbot + 1 - ns; + let mut sub = vec![T::zero(); ns * ns]; + for j in 0..ns { + for i in 0..=(j + 1).min(ns - 1) { + sub[i + j * ns] = h[at(ks + i, ks + j)]; + } + } + let info = if ns > nmin && !nested { + multishift(false, &mut sub, ns, None, &mut w[ks..=kbot], true) + } else { + small_qr(&mut sub, ns, None, false, &mut w[ks..=kbot]) + }; + ks += info; + if ks >= kbot { + trailing_pair(h, n, kbot, w); + ks = kbot - 1; + } + } + if kbot + 1 - ks > ns { + // The largest first, conjugates staying together. + let mut last = kbot; + let mut sorted = false; + while !sorted && last > ks { + sorted = true; + for i in ks..last { + if abs1(w[i]) < abs1(w[i + 1]) { + sorted = false; + w.swap(i, i + 1); + } + } + last -= 1; + } + } + if !T::IS_COMPLEX { + pair_up(&mut w[ks..=kbot]); + } + } + if kbot + 1 - ks == 2 { + // Of two shifts both real, or of a complex one's, the one nearer the corner. + let corner = h[at(kbot, kbot)].to_complex(); + if T::IS_COMPLEX || w[kbot].im == T::Real::ZERO { + if abs1(w[kbot] - corner) < abs1(w[kbot - 1] - corner) { + w[kbot - 1] = w[kbot]; + } else { + w[kbot] = w[kbot - 1]; + } + } + } + ns = ns.min(kbot + 1 - ks); + ns -= ns % 2; + ks = kbot + 1 - ns; + let mut s = w[ks..=kbot].to_vec(); + sweep( + want_t, + h, + n, + z.as_deref_mut(), + ktop, + kbot, + &mut s, + accumulate, + ); + } + ndfl = if ld > 0 { 1 } else { ndfl + 1 }; + } + end +} + +/// Exceptional shifts into `w[kbot + 1 - ns..=kbot]`, from the subdiagonal near the corner. +fn exceptional( + h: &[T], + n: usize, + ktop: usize, + kbot: usize, + ns: usize, + w: &mut [Complex], +) { + let at = |i: usize, j: usize| i + j * n; + let ks = kbot + 1 - ns; + let wilk1 = T::Real::from_f64(0.75); + if T::IS_COMPLEX { + let mut i = kbot; + while i > ks { + let x = h[at(i, i)] + T::from_real(wilk1 * h[at(i, i - 1)].abs1()); + w[i] = x.to_complex(); + w[i - 1] = w[i]; + if i < ks + 3 { + break; + } + i -= 2; + } + } else { + let low = (ks + 1).max(ktop + 2); + let mut i = kbot; + while i >= low { + let ss = h[at(i, i - 1)].abs() + h[at(i - 1, i - 2)].abs(); + let aa = wilk1 * ss + h[at(i, i)].re(); + let s = standardize(aa, ss, T::Real::from_f64(-0.4375) * ss, aa); + w[i - 1] = Complex::new(s.rt1r, s.rt1i); + w[i] = Complex::new(s.rt2r, s.rt2i); + if i < low + 2 { + break; + } + i -= 2; + } + if ks == ktop { + w[ks + 1] = Complex::new(h[at(ks + 1, ks + 1)].re(), T::Real::ZERO); + w[ks] = w[ks + 1]; + } + } +} + +/// The eigenvalues of the trailing `2 × 2` block into `w[kbot - 1..=kbot]`, the shifts when the +/// QR algorithm failed on a trailing block. +fn trailing_pair(h: &[T], n: usize, kbot: usize, w: &mut [Complex]) { + let at = |i: usize, j: usize| i + j * n; + let (a, b, c, d) = ( + h[at(kbot - 1, kbot - 1)], + h[at(kbot - 1, kbot)], + h[at(kbot, kbot - 1)], + h[at(kbot, kbot)], + ); + if T::IS_COMPLEX { + let s = a.abs1() + b.abs1() + c.abs1() + d.abs1(); + if s == T::Real::ZERO { + w[kbot - 1] = Complex::new(T::Real::ZERO, T::Real::ZERO); + w[kbot] = w[kbot - 1]; + return; + } + let r = T::Real::ONE / s; + let (a, b, c, d) = (a.scale(r), b.scale(r), c.scale(r), d.scale(r)); + let tr = (a + d).scale(T::Real::HALF); + let det = (a - tr) * (d - tr) - b * c; + let root = (-det).sqrt(); + w[kbot - 1] = (tr + root).scale(s).to_complex(); + w[kbot] = (tr - root).scale(s).to_complex(); + } else { + let s = standardize(a.re(), b.re(), c.re(), d.re()); + w[kbot - 1] = Complex::new(s.rt1r, s.rt1i); + w[kbot] = Complex::new(s.rt2r, s.rt2i); + } +} + +/// Orders real shifts so that each consecutive pair is two reals or two conjugates, those +/// already next to each other. +fn pair_up(s: &mut [Complex]) { + let mut i = s.len(); + while i >= 3 { + if s[i - 1].im != -s[i - 2].im { + s[i - 3..i].rotate_right(1); + } + i -= 2; + } +} + +/// A multiple of the first column of `(H - s₁·I)·(H - s₂·I)`, `H` the block of order `nb`, 2 or +/// 3, at row and column `r` of `h`, as LAPACK's `laqr1`: real for real elements, the shifts +/// being two reals or two conjugates. +fn first_column( + h: &[T], + n: usize, + r: usize, + nb: usize, + s1: Complex, + s2: Complex, +) -> [T; 3] { + let at = |i: usize, j: usize| r + i + (r + j) * n; + let zero = T::Real::ZERO; + if T::IS_COMPLEX { + let (s1, s2) = (T::from_parts(s1.re, s1.im), T::from_parts(s2.re, s2.im)); + let h11 = h[at(0, 0)]; + let h21 = h[at(1, 0)]; + let h31 = if nb == 3 { h[at(2, 0)] } else { T::zero() }; + let s = (h11 - s2).abs1() + h21.abs1() + h31.abs1(); + if s == zero { + return [T::zero(); 3]; + } + let rs = T::Real::ONE / s; + let (h21s, h31s) = (h21.scale(rs), h31.scale(rs)); + let mut v0 = (h11 - s1) * (h11 - s2).scale(rs) + h[at(0, 1)] * h21s; + let mut v1 = h21s * (h11 + h[at(1, 1)] - s1 - s2); + let mut v2 = T::zero(); + if nb == 3 { + v0 += h[at(0, 2)] * h31s; + v1 += h[at(1, 2)] * h31s; + v2 = h31s * (h11 + h[at(2, 2)] - s1 - s2) + h21s * h[at(2, 1)]; + } + [v0, v1, v2] + } else { + let g = |i: usize, j: usize| h[at(i, j)].re(); + let (h11, h21) = (g(0, 0), g(1, 0)); + let h31 = if nb == 3 { g(2, 0) } else { zero }; + let s = (h11 - s2.re).abs() + s2.im.abs() + h21.abs() + h31.abs(); + if s == zero { + return [T::zero(); 3]; + } + let (h21s, h31s) = (h21 / s, h31 / s); + let sum = s1.re + s2.re; + let mut v0 = (h11 - s1.re) * ((h11 - s2.re) / s) - s1.im * (s2.im / s) + g(0, 1) * h21s; + let mut v1 = h21s * (h11 + g(1, 1) - sum); + let mut v2 = zero; + if nb == 3 { + v0 += g(0, 2) * h31s; + v1 += g(1, 2) * h31s; + v2 = h31s * (h11 + g(2, 2) - sum) + h21s * g(2, 1); + } + [T::from_real(v0), T::from_real(v1), T::from_real(v2)] + } +} + +/// Sets the subdiagonal `h[k + 1, k]` to zero when it is negligible both by the usual test and +/// by Ahues and Tisseur's, for the vigilant deflation inside a sweep. +#[allow(clippy::too_many_arguments)] +fn vigilant( + h: &mut [T], + n: usize, + k: usize, + ktop: usize, + kbot: usize, + smlnum: T::Real, + ulp: T::Real, +) { + let at = |i: usize, j: usize| i + j * n; + if h[at(k + 1, k)] == T::zero() { + return; + } + let mut tst1 = h[at(k, k)].abs1() + h[at(k + 1, k + 1)].abs1(); + if tst1 == T::Real::ZERO { + for d in 1..=3 { + if k >= ktop + d { + tst1 += h[at(k, k - d)].abs1(); + } + } + for d in 2..=4 { + if k + d <= kbot { + tst1 += h[at(k + d, k + 1)].abs1(); + } + } + } + let sub = h[at(k + 1, k)].abs1(); + if sub <= smlnum.max(ulp * tst1) { + let up = h[at(k, k + 1)].abs1(); + let (h12, h21) = (sub.max(up), sub.min(up)); + let d = (h[at(k, k)] - h[at(k + 1, k + 1)]).abs1(); + let hk = h[at(k + 1, k + 1)].abs1(); + let (h11, h22) = (hk.max(d), hk.min(d)); + let scl = h11 + h12; + let tst2 = h22 * (h11 / scl); + if tst2 == T::Real::ZERO || h21 * (h12 / scl) <= smlnum.max(ulp * tst2) { + h[at(k + 1, k)] = T::zero(); + } + } +} + +/// A reflection `I - τ·v·v*`, `v = (1, v₂, v₃)`, as `[τ, v₂, v₃]`. +type Reflection = [T; 3]; + +/// `x - τ·(x·v)·v*` on the elements `x₁..x₃` of a row, the reflection from the right. +#[inline(always)] +fn right3(r: Reflection, x1: &mut T, x2: &mut T, x3: &mut T) { + let [tau, v2, v3] = r; + let sum = tau * (*x1 + v2 * *x2 + v3 * *x3); + *x1 -= sum; + *x2 -= sum * v2.conj(); + *x3 -= sum * v3.conj(); +} + +/// `x - conj(τ)·v·(v*·x)` on the elements `x₁..x₃` of a column, the reflection's conjugate +/// transpose from the left. +#[inline(always)] +fn left3(r: Reflection, x1: &mut T, x2: &mut T, x3: &mut T) { + let [tau, v2, v3] = r; + let sum = tau.conj() * (*x1 + v2.conj() * *x2 + v3.conj() * *x3); + *x1 -= sum; + *x2 -= sum * v2; + *x3 -= sum * v3; +} + +/// Three columns `c..c + 3` of the column-major `a` with the leading dimension `ld`, to change. +#[inline(always)] +fn columns3(a: &mut [T], ld: usize, c: usize) -> (&mut [T], &mut [T], &mut [T]) { + let (x, rest) = a[c * ld..].split_at_mut(ld); + let (y, rest) = rest.split_at_mut(ld); + (x, y, &mut rest[..ld]) +} + +/// Applies the reflection from the right to rows `rows` of columns `c..c + 3`. +#[inline(always)] +fn right3_columns( + r: Reflection, + a: &mut [T], + ld: usize, + c: usize, + rows: core::ops::Range, +) { + let (x, y, z) = columns3(a, ld, c); + for ((x, y), z) in x[rows.clone()] + .iter_mut() + .zip(&mut y[rows.clone()]) + .zip(&mut z[rows]) + { + right3(r, x, y, z); + } +} + +/// One sweep of the small-bulge multishift QR algorithm on the active block `ktop..=kbot` of the +/// upper Hessenberg `h`, with the shifts `s`, as LAPACK's `laqr5`: chains of bulges two shifts +/// each, chased `2·(bulges)` columns at a time, the reflections of each step gathered into `U` +/// and applied to the far parts of `h` and to `z` by products of blocks when `accumulate` is +/// set. +#[allow(clippy::too_many_arguments)] +fn sweep( + want_t: bool, + h: &mut [T], + n: usize, + mut z: Option<&mut [T]>, + ktop: usize, + kbot: usize, + s: &mut [Complex], + accumulate: bool, +) { + let at = |i: usize, j: usize| i + j * n; + if s.len() < 2 || ktop >= kbot { + return; + } + if !T::IS_COMPLEX { + // Pairs from the start, an odd one out last. + let mut i = 0; + while i + 2 < s.len() { + if s[i].im != -s[i + 1].im { + s[i..i + 3].rotate_left(1); + } + i += 2; + } + } + let ns = s.len() - s.len() % 2; + let ulp = T::Real::EPSILON; + let smlnum = T::Real::MIN_POSITIVE * (T::Real::from_f64(n as f64) / ulp); + if ktop + 2 <= kbot { + h[at(ktop + 2, ktop)] = T::zero(); + } + let nbmps = ns / 2; + let kdu = 4 * nbmps; + let mut v: Vec> = vec![[T::zero(); 3]; nbmps]; + let mut u = if accumulate { + vec![T::zero(); kdu * kdu] + } else { + Vec::new() + }; + let (top, bot) = (ktop as isize, kbot as isize); + let step = 2 * nbmps as isize; + // Each chunk moves the chain from `incol` (perhaps before the block, the bulges not yet in) + // `step` columns down; `U`'s row and column `c` is `h`'s `incol + 1 + c`. + let mut incol = top - step + 1; + while incol <= bot - 2 { + let ndcol = incol + kdu as isize; + let jtop = if accumulate { + ktop.max(incol.max(0) as usize) + } else if want_t { + 0 + } else { + ktop + }; + let jbot = if accumulate { + ndcol.min(bot) as usize + } else if want_t { + n - 1 + } else { + kbot + }; + if accumulate { + u.fill(T::zero()); + for i in 0..kdu { + u[i + i * kdu] = T::one(); + } + } + let u_first = (top - incol - 1).max(0) as usize; + for krcol in incol..=(incol + step - 1).min(bot - 2) { + // Bulges `mtop..mbot` are in the block; bulge `m` acts on rows and columns + // `r..r + 3`, `r = krcol + 1 + 2m`, made from column `r - 1`. + let mtop = ((top - krcol).max(0) / 2) as usize; + let mbot = (nbmps as isize).min((bot - krcol - 1) / 2) as usize; + let row = |m: usize| (krcol + 1 + 2 * m as isize) as usize; + if mbot < nbmps && row(mbot) + 1 == kbot { + // A last bulge with room for only a 2 × 2 reflection. + let m = mbot; + let r = row(m); + let (tau, v2) = if r == ktop { + let x = first_column(h, n, r, 2, s[2 * m], s[2 * m + 1]); + let mut rest = [x[1]]; + let (tau, _) = short_reflector(x[0], &mut rest); + (tau, rest[0]) + } else { + let mut rest = [h[at(r + 1, r - 1)]]; + let (tau, beta) = short_reflector(h[at(r, r - 1)], &mut rest); + h[at(r, r - 1)] = beta; + h[at(r + 1, r - 1)] = T::zero(); + (tau, rest[0]) + }; + v[m] = [tau, v2, T::zero()]; + let (x, y) = h[r * n..].split_at_mut(n); + for (x, y) in x[jtop..=kbot].iter_mut().zip(&mut y[jtop..=kbot]) { + let sum = tau * (*x + v2 * *y); + *x -= sum; + *y -= sum * v2.conj(); + } + let (tc, v2c) = (tau.conj(), v2.conj()); + for j in r..=jbot { + let sum = tc * (h[at(r, j)] + v2c * h[at(r + 1, j)]); + h[at(r, j)] -= sum; + h[at(r + 1, j)] -= sum * v2; + } + if r > ktop { + vigilant(h, n, r - 1, ktop, kbot, smlnum, ulp); + } + let (target, ld, c, rows) = if accumulate { + ( + Some(&mut u[..]), + kdu, + (r as isize - incol - 1) as usize, + u_first..kdu, + ) + } else { + (z.as_deref_mut(), n, r, 0..n) + }; + if let Some(target) = target { + let (x, y) = target[c * ld..].split_at_mut(ld); + for (x, y) in x[rows.clone()].iter_mut().zip(&mut y[rows]) { + let sum = tau * (*x + v2 * *y); + *x -= sum; + *y -= sum * v2.conj(); + } + } + } + // The 3 × 3 bulges, from the bottom one up: each moved down a column, applied from + // the right and to its next column from the left (what the deflation test needs), + // the rest of the left side after. + for m in (mtop..mbot).rev() { + let r = row(m); + if r == ktop { + let x = first_column(h, n, r, 3, s[2 * m], s[2 * m + 1]); + let mut rest = [x[1], x[2]]; + let (tau, _) = short_reflector(x[0], &mut rest); + v[m] = [tau, rest[0], rest[1]]; + } else { + // The previous step's reflection on the row below the bulge, whose first + // two elements are zero. + let [tau, v2, v3] = v[m]; + let sum = tau * v3 * h[at(r + 2, r + 1)]; + h[at(r + 2, r - 1)] = -sum; + h[at(r + 2, r)] = -(sum * v2.conj()); + h[at(r + 2, r + 1)] -= sum * v3.conj(); + let mut rest = [h[at(r + 1, r - 1)], h[at(r + 2, r - 1)]]; + let (tau, beta) = short_reflector(h[at(r, r - 1)], &mut rest); + let moved = [tau, rest[0], rest[1]]; + if h[at(r + 2, r - 1)] != T::zero() + || h[at(r + 2, r)] != T::zero() + || h[at(r + 2, r + 1)] == T::zero() + { + h[at(r, r - 1)] = beta; + v[m] = moved; + } else { + // The bulge collapsed: a fresh one from the shifts, if its fill is + // negligible. + let x = first_column(h, n, r, 3, s[2 * m], s[2 * m + 1]); + let mut rest = [x[1], x[2]]; + let (tau, _) = short_reflector(x[0], &mut rest); + let fresh = [tau, rest[0], rest[1]]; + let sum = + tau.conj() * (h[at(r, r - 1)] + rest[0].conj() * h[at(r + 1, r - 1)]); + let fill = + (h[at(r + 1, r - 1)] - sum * rest[0]).abs1() + (sum * rest[1]).abs1(); + let near = h[at(r - 1, r - 1)].abs1() + + h[at(r, r)].abs1() + + h[at(r + 1, r + 1)].abs1(); + if fill > ulp * near { + h[at(r, r - 1)] = beta; + v[m] = moved; + } else { + h[at(r, r - 1)] -= sum; + v[m] = fresh; + } + } + h[at(r + 1, r - 1)] = T::zero(); + h[at(r + 2, r - 1)] = T::zero(); + } + let reflection = v[m]; + right3_columns(reflection, h, n, r, jtop..kbot.min(r + 2) + 1); + let (mut x1, mut x2, mut x3) = (h[at(r, r)], h[at(r + 1, r)], h[at(r + 2, r)]); + left3(reflection, &mut x1, &mut x2, &mut x3); + (h[at(r, r)], h[at(r + 1, r)], h[at(r + 2, r)]) = (x1, x2, x3); + if r > ktop { + vigilant(h, n, r - 1, ktop, kbot, smlnum, ulp); + } + } + for m in (mtop..mbot).rev() { + let r = row(m); + let reflection = v[m]; + for j in ktop.max(r + 1)..=jbot { + let column = &mut h[j * n + r..j * n + r + 3]; + let [x1, x2, x3] = column else { unreachable!() }; + left3(reflection, x1, x2, x3); + } + } + if accumulate { + for m in (mtop..mbot).rev() { + let r = row(m); + let c = (r as isize - incol - 1) as usize; + let i2 = u_first.max(2 * m); + let i4 = kdu.min((krcol - incol) as usize + 2 * (mbot - 1) + 5); + right3_columns(v[m], &mut u, kdu, c, i2..i4); + } + } else if let Some(z) = z.as_deref_mut() { + for m in (mtop..mbot).rev() { + right3_columns(v[m], z, n, row(m), 0..n); + } + } + } + if accumulate { + // `U` on the far parts: `U*` on the rows of the window to their right, `U` on the + // columns of the window above it and in `Z`. + let (jtop, jbot) = if want_t { (0, n - 1) } else { (ktop, kbot) }; + let k1 = u_first; + let nu = kdu - (ndcol - bot).max(0) as usize - k1; + let h0 = (incol + 1 + k1 as isize) as usize; + let mut uw = vec![T::zero(); nu * nu]; + let mut uh = vec![T::zero(); nu * nu]; + for j in 0..nu { + for i in 0..nu { + let x = u[k1 + i + (k1 + j) * kdu]; + uw[i + j * nu] = x; + uh[j + i * nu] = x.conj(); + } + } + let c0 = (ndcol.min(bot) + 1) as usize; + if c0 <= jbot { + let cols = jbot + 1 - c0; + let mut tmp = vec![T::zero(); nu * cols]; + let src = &h[h0 + c0 * n..]; + gemm( + nu, + cols, + nu, + T::one(), + &uh, + nu, + Op::N, + src, + n, + Op::N, + T::zero(), + &mut tmp, + nu, + ); + for j in 0..cols { + h[h0 + (c0 + j) * n..h0 + (c0 + j) * n + nu] + .copy_from_slice(&tmp[j * nu..(j + 1) * nu]); + } + } + let r1 = ktop.max(incol.max(0) as usize); + if jtop < r1 { + times_u(&mut h[h0 * n..], n, jtop..r1, &uw, nu); + } + if let Some(z) = z.as_deref_mut() { + times_u(&mut z[h0 * n..], n, 0..n, &uw, nu); + } + } + incol += step; + } +} + +/// `A[rows, 0..k] = A[rows, 0..k]·U`, `U` of `k × k`, by a product into a copy. +fn times_u(a: &mut [T], ld: usize, rows: core::ops::Range, u: &[T], k: usize) { + let m = rows.len(); + let mut tmp = vec![T::zero(); m * k]; + gemm( + m, + k, + k, + T::one(), + &a[rows.start..], + ld, + Op::N, + u, + k, + Op::N, + T::zero(), + &mut tmp, + m, + ); + for j in 0..k { + a[j * ld + rows.start..j * ld + rows.end].copy_from_slice(&tmp[j * m..(j + 1) * m]); + } +} diff --git a/lib/src/math/matrix/schur/vectors.rs b/lib/src/math/matrix/schur/vectors.rs new file mode 100644 index 0000000..f9ad5a3 --- /dev/null +++ b/lib/src/math/matrix/schur/vectors.rs @@ -0,0 +1,610 @@ +//! The eigenvectors of a matrix from its Schur form `A = Q·T·Q*`: those of `T` by back +//! substitution, as LAPACK's `trevc`, then `Q` times them by products of blocks. For real +//! elements the substitution stays in real arithmetic on the quasi-triangular `T`, a pair of +//! complex conjugate eigenvectors held as its real and imaginary parts. The substitution is +//! blocked: a block of eigenvectors at a time, solved a block of rows at a time, the solved +//! rows' share of the rows above one product of blocks; a vector that needs scaling against +//! overflow is solved again alone, as LAPACK does it. + +use super::super::Scalar; +use super::super::blas::{Op, gemm}; +use super::super::householder::norm2; +use crate::math::complex::{Complex, Float}; + +/// The columns of a block of the product by `Q`. +const BLOCK: usize = 64; + +/// The eigenvectors solved together, and the rows of a block of their substitution. +const SOLVE: usize = 64; + +/// The unit eigenvectors of `A = Q·T·Q*`, as the columns of an `n × n` matrix in the order of +/// `T`'s diagonal, given `T` (the real Schur form, with standard `2 × 2` blocks, for real +/// elements) and `Q`, column-major. A pair of complex conjugate eigenvalues of a real matrix has +/// the conjugate eigenvectors, the one of the positive imaginary part first. +pub(crate) fn eigenvectors(t: &[T], q: &[T], n: usize) -> Vec> { + let zero = T::Real::ZERO; + if t.iter().chain(q).any(|x| !x.is_finite()) { + return vec![Complex::new(T::Real::NAN, T::Real::NAN); n * n]; + } + let x = Triangle::new(t, n).solve(); + let v = times_q(q, &x, n); + let column = |j: usize| &v[j * n..(j + 1) * n]; + let mut out = vec![Complex::new(zero, zero); n * n]; + let mut k = 0; + while k < n { + if second(t, n, k + 1) { + let (re, im) = (column(k), column(k + 1)); + let norm = norm2(re).hypot(norm2(im)); + let r = if norm > zero { + T::Real::ONE / norm + } else { + T::Real::ONE + }; + for i in 0..n { + let (a, b) = (re[i].re() * r, im[i].re() * r); + out[i + k * n] = Complex::new(a, b); + out[i + (k + 1) * n] = Complex::new(a, -b); + } + k += 2; + } else { + let norm = norm2(column(k)); + let r = if norm > zero { + T::Real::ONE / norm + } else { + T::Real::ONE + }; + for (o, v) in out[k * n..(k + 1) * n].iter_mut().zip(column(k)) { + *o = v.scale(r).to_complex(); + } + k += 1; + } + } + out +} + +/// `Q·X` for the upper triangular `X`, a block of columns at a time, each the product by the +/// columns of `Q` it needs. +fn times_q(q: &[T], x: &[T], n: usize) -> Vec { + let mut v = vec![T::zero(); n * n]; + let mut c0 = 0; + while c0 < n { + let c1 = (c0 + BLOCK).min(n); + let (xs, vs) = (&x[c0 * n..], &mut v[c0 * n..]); + gemm( + n, + c1 - c0, + c1, + T::one(), + q, + n, + Op::N, + xs, + n, + Op::N, + T::zero(), + vs, + n, + ); + c0 = c1; + } + v +} + +/// Whether row `k` of the real Schur form `t` is the second of a `2 × 2` block, never for +/// complex elements. +fn second(t: &[T], n: usize, k: usize) -> bool { + !T::IS_COMPLEX && k > 0 && k < n && t[k + (k - 1) * n] != T::zero() +} + +/// The (quasi-)triangular `T` of the Schur form, with what its back substitutions share. +struct Triangle<'a, T: Scalar> { + /// `T`, column-major. + t: &'a [T], + /// The order. + n: usize, + /// The 1-norms of the columns above the diagonal, bounding the growth of an update. + norms: Vec, + /// The unit roundoff. + ulp: T::Real, + /// The smallest pivot, `n` units of roundoff above underflow. + smlnum: T::Real, + /// The largest element a vector may reach before it is scaled down. + bignum: T::Real, +} + +impl<'a, T: Scalar> Triangle<'a, T> { + /// The triangle `t` of order `n`. + fn new(t: &'a [T], n: usize) -> Self { + let ulp = T::Real::EPSILON; + let smlnum = T::Real::MIN_POSITIVE * (T::Real::from_f64(n.max(1) as f64) / ulp); + let norms = (0..n) + .map(|j| { + t[j * n..j * n + j] + .iter() + .fold(T::Real::ZERO, |s, x| s + x.abs1()) + }) + .collect(); + Triangle { + t, + n, + norms, + ulp, + smlnum, + bignum: (T::Real::ONE - ulp) / smlnum, + } + } + + /// Element `(i, j)` of `T`. + #[inline(always)] + fn at(&self, i: usize, j: usize) -> T { + self.t[i + j * self.n] + } + + /// The columns of the eigenvalue at `k`: 2 for a pair of complex ones of a real matrix. + fn width(&self, k: usize) -> usize { + if second(self.t, self.n, k + 1) { 2 } else { 1 } + } + + /// The eigenvectors of `T` as LAPACK's `trevc`: in column `k` for an eigenvalue, but for a + /// pair of complex ones of a real matrix at `k, k + 1` the real part of the first one's in + /// column `k` and the imaginary part in column `k + 1`; upper triangular, each vector + /// scaled to a largest element of 1, `|re| + |im|` for the complex ones of a real matrix. + fn solve(&self) -> Vec { + let (n, zero) = (self.n, T::Real::ZERO); + let mut x = vec![T::zero(); n * n]; + let mut redo = vec![false; n]; + let mut c0 = 0; + while c0 < n { + let mut c1 = (c0 + SOLVE).min(n); + if second(self.t, n, c1) { + c1 += 1; + } + let mut k = c0; + while k < c1 { + self.start(&mut x, k, c0); + redo[k] = !self.rows(&mut x, k, c0, k, false); + k += self.width(k); + } + // The rows above, a block at a time: the solved rows' share in one product, then + // the block's own substitution. + let mut hi = c0; + while hi > 0 { + let mut lo = hi.saturating_sub(SOLVE); + if second(self.t, n, lo) { + lo -= 1; + } + let (rows, cols) = (hi - lo, c1 - c0); + let mut part = vec![T::zero(); rows * cols]; + for j in 0..cols { + part[j * rows..(j + 1) * rows] + .copy_from_slice(&x[lo + (c0 + j) * n..hi + (c0 + j) * n]); + } + let (tb, xb) = (&self.t[lo + hi * n..], &x[hi + c0 * n..]); + gemm( + rows, + cols, + c1 - hi, + -T::one(), + tb, + n, + Op::N, + xb, + n, + Op::N, + T::one(), + &mut part, + rows, + ); + for j in 0..cols { + x[lo + (c0 + j) * n..hi + (c0 + j) * n] + .copy_from_slice(&part[j * rows..(j + 1) * rows]); + } + let mut k = c0; + while k < c1 { + if !redo[k] { + redo[k] = !self.rows(&mut x, k, lo, hi, false); + } + k += self.width(k); + } + hi = lo; + } + c0 = c1; + } + // The vectors that needed scaling, or overflowed in the products, solved again with + // the scaling; then each scaled to a largest element of 1. + let mut k = 0; + while k < n { + let w = self.width(k); + let cols = k * n..(k + w) * n; + let largest = |x: &[T]| { + (0..=k + w - 1).fold(zero, |m, i| { + let s = if w == 2 { + x[i].abs1() + x[i + n].abs1() + } else { + x[i].abs1() + }; + if s.is_finite() { + m.max(s) + } else { + T::Real::INFINITY + } + }) + }; + if redo[k] || largest(&x[cols.clone()]) > self.bignum { + x[cols.clone()].fill(T::zero()); + self.start(&mut x, k, 0); + self.rows(&mut x, k, 0, k, true); + } + let m = largest(&x[cols.clone()]); + if m > zero && m.is_finite() { + let r = T::Real::ONE / m; + for v in &mut x[cols] { + *v = v.scale(r); + } + } + k += w; + } + x + } + + /// The eigenvalue at `k`: `(wr, wi)`, `wi` positive for a pair of a real matrix, and the + /// smallest pivot of its substitution. + fn shift(&self, k: usize) -> (T, T::Real, T::Real) { + let lambda = self.at(k, k); + if self.width(k) == 2 { + let p = k + 1; + let wr = self.at(p, p).re(); + let wi = self.at(p, k).re().abs().sqrt() * self.at(k, p).re().abs().sqrt(); + (lambda, wi, (self.ulp * (wr.abs() + wi)).max(self.smlnum)) + } else { + ( + lambda, + T::Real::ZERO, + (self.ulp * lambda.abs1()).max(self.smlnum), + ) + } + } + + /// Sets the columns of the eigenvalue at `k` to its part on the diagonal, 1 for a single + /// one, and in rows `from..` above it the right-hand side of the substitution, that part's + /// share; the rows above `from` get it with the rest of the solved rows' shares. + fn start(&self, x: &mut [T], k: usize, from: usize) { + let n = self.n; + if self.width(k) == 1 { + x[k + k * n] = T::one(); + for i in from..k { + x[i + k * n] = -self.at(i, k); + } + return; + } + // `x = re + i·im` for `λ = wr + i·wi`: `(T_b - λ·I)·(x_k, x_p) = 0` on the block. + let p = k + 1; + let (_, wi, _) = self.shift(k); + let (up, down) = (self.at(k, p).re(), self.at(p, k).re()); + let (rk, ip) = if up.abs() >= down.abs() { + (T::Real::ONE, wi / up) + } else { + (-wi / down, T::Real::ONE) + }; + let (re, im) = x[k * n..].split_at_mut(n); + re[k] = T::from_real(rk); + re[p] = T::zero(); + im[k] = T::zero(); + im[p] = T::from_real(ip); + for i in from..k { + re[i] = self.at(i, k).scale(-rk); + im[i] = self.at(i, p).scale(-ip); + } + } + + /// The substitution of rows `lo..hi` of the columns of the eigenvalue at `k`, the rows + /// below already solved and their share subtracted: `(T - λ·I)·x = b` a diagonal block of + /// `T` at a time, from the bottom, each block's solution taken from the rows above it. When + /// the solution grows near overflow, the vector is scaled down if `careful`, or else the + /// substitution stops and gives false. + fn rows(&self, x: &mut [T], k: usize, lo: usize, hi: usize, careful: bool) -> bool { + let (n, one) = (self.n, T::Real::ONE); + let (lambda, wi, smin) = self.shift(k); + let w = self.width(k); + let (head, tail) = x[k * n..].split_at_mut(n); + let (re, mut im) = (head, if w == 2 { Some(&mut tail[..n]) } else { None }); + let last = k + w - 1; + let mut j = hi; + while j > lo { + let j2 = j - 1; + let j1 = if j2 > lo && second(self.t, n, j2) { + j2 - 1 + } else { + j2 + }; + let nb = j2 - j1 + 1; + if T::IS_COMPLEX { + let mut d = self.at(j2, j2) - lambda; + if d.abs1() < smin { + d = T::from_real(smin); + } + let xi = re[j2] / d; + re[j2] = xi; + let size = xi.abs1(); + if size > self.bignum { + if !careful { + return false; + } + let r = one / size; + for v in &mut re[..=last] { + *v = v.scale(r); + } + } + } else { + let g = |i: usize, c: usize| self.at(i, c).re(); + let a = [[g(j1, j1), g(j1, j2)], [g(j2, j1), g(j2, j2)]]; + let b = match &im { + Some(im) => [[re[j1].re(), im[j1].re()], [re[j2].re(), im[j2].re()]], + None => [[re[j1].re(), T::Real::ZERO], [re[j2].re(), T::Real::ZERO]], + }; + let (mut s, mut scale, xnorm) = small_solve(nb, w, smin, a, b, lambda.re(), wi); + if xnorm > one && self.norms[j1].max(self.norms[j2]) > self.bignum / xnorm { + let r = one / xnorm; + for v in s.iter_mut().flatten() { + *v *= r; + } + scale *= r; + } + if scale != one { + if !careful { + return false; + } + for v in re[..=last] + .iter_mut() + .chain(im.iter_mut().flat_map(|im| im[..=last].iter_mut())) + { + *v = v.scale(scale); + } + } + for (p, row) in [j1, j2].into_iter().enumerate().take(nb) { + re[row] = T::from_real(s[p][0]); + if let Some(im) = &mut im { + im[row] = T::from_real(s[p][1]); + } + } + } + for row in j1..=j2 { + let column = &self.t[lo + row * n..j1 + row * n]; + let f = re[row]; + for (x, t) in re[lo..j1].iter_mut().zip(column) { + *x -= *t * f; + } + if let Some(im) = &mut im { + let f = im[row]; + for (x, t) in im[lo..j1].iter_mut().zip(column) { + *x -= *t * f; + } + } + } + j = j1; + } + true + } +} + +/// `(a + i·b) / (c + i·d)` by Smith's algorithm, which doesn't overflow needlessly. +fn divide(a: R, b: R, c: R, d: R) -> (R, R) { + if d.abs() <= c.abs() { + let e = d / c; + let f = c + d * e; + ((a + b * e) / f, (b - a * e) / f) + } else { + let e = c / d; + let f = d + c * e; + ((b + a * e) / f, (-a + b * e) / f) + } +} + +/// The solution of `(A - w·I)·X = scale·B` for the real `A` of order `na` (1 or 2) and +/// `w = wr + i·wi`, `X` and `B` with `nw` columns (1, or 2 for the real and imaginary parts), as +/// LAPACK's `laln2`: Gaussian elimination with complete pivoting, a pivot below `smin` replaced +/// by it, and `scale ≤ 1` keeping `X` from overflowing. Returns `X`, `scale` and the largest +/// row sum of `|X|`. +#[allow(clippy::type_complexity)] +fn small_solve( + na: usize, + nw: usize, + smin: R, + a: [[R; 2]; 2], + b: [[R; 2]; 2], + wr: R, + wi: R, +) -> ([[R; 2]; 2], R, R) { + let (zero, one) = (R::ZERO, R::ONE); + let smlnum = R::TWO * R::MIN_POSITIVE; + let bignum = one / smlnum; + let smini = smin.max(smlnum); + let mut scale = one; + let mut x = [[zero; 2]; 2]; + if na == 1 { + if nw == 1 { + let mut csr = a[0][0] - wr; + let mut cnorm = csr.abs(); + if cnorm < smini { + csr = smini; + cnorm = smini; + } + let bnorm = b[0][0].abs(); + if cnorm < one && bnorm > one && bnorm > bignum * cnorm { + scale = one / bnorm; + } + x[0][0] = (b[0][0] * scale) / csr; + return (x, scale, x[0][0].abs()); + } + let (mut csr, mut csi) = (a[0][0] - wr, -wi); + let mut cnorm = csr.abs() + csi.abs(); + if cnorm < smini { + csr = smini; + csi = zero; + cnorm = smini; + } + let bnorm = b[0][0].abs() + b[0][1].abs(); + if cnorm < one && bnorm > one && bnorm > bignum * cnorm { + scale = one / bnorm; + } + (x[0][0], x[0][1]) = divide(scale * b[0][0], scale * b[0][1], csr, csi); + return (x, scale, x[0][0].abs() + x[0][1].abs()); + } + // `C = A - wr·I`, column after column, and the order of the pivoting for each choice of + // the first pivot. + let cr = [a[0][0] - wr, a[1][0], a[0][1], a[1][1] - wr]; + const PIVOT: [[usize; 4]; 4] = [[0, 1, 2, 3], [1, 0, 3, 2], [2, 3, 0, 1], [3, 2, 1, 0]]; + const RSWAP: [bool; 4] = [false, true, false, true]; + const ZSWAP: [bool; 4] = [false, false, true, true]; + if nw == 1 { + let mut cmax = zero; + let mut icmax = 0; + for (j, c) in cr.iter().enumerate() { + if c.abs() > cmax { + cmax = c.abs(); + icmax = j; + } + } + if cmax < smini { + let bnorm = b[0][0].abs().max(b[1][0].abs()); + if smini < one && bnorm > one && bnorm > bignum * smini { + scale = one / bnorm; + } + let temp = scale / smini; + x[0][0] = temp * b[0][0]; + x[1][0] = temp * b[1][0]; + return (x, scale, temp * bnorm); + } + let p = PIVOT[icmax]; + let ur11 = cr[icmax]; + let (cr21, ur12, cr22) = (cr[p[1]], cr[p[2]], cr[p[3]]); + let ur11r = one / ur11; + let lr21 = ur11r * cr21; + let mut ur22 = cr22 - ur12 * lr21; + if ur22.abs() < smini { + ur22 = smini; + } + let (br1, mut br2) = if RSWAP[icmax] { + (b[1][0], b[0][0]) + } else { + (b[0][0], b[1][0]) + }; + br2 -= lr21 * br1; + let bbnd = (br1 * (ur22 * ur11r)).abs().max(br2.abs()); + if bbnd > one && ur22.abs() < one && bbnd >= bignum * ur22.abs() { + scale = one / bbnd; + } + let xr2 = (br2 * scale) / ur22; + let xr1 = (scale * br1) * ur11r - xr2 * (ur11r * ur12); + if ZSWAP[icmax] { + x[0][0] = xr2; + x[1][0] = xr1; + } else { + x[0][0] = xr1; + x[1][0] = xr2; + } + let mut xnorm = xr1.abs().max(xr2.abs()); + if xnorm > one && cmax > one && xnorm > bignum / cmax { + let temp = cmax / bignum; + x[0][0] *= temp; + x[1][0] *= temp; + xnorm *= temp; + scale *= temp; + } + return (x, scale, xnorm); + } + let ci = [-wi, zero, zero, -wi]; + let mut cmax = zero; + let mut icmax = 0; + for j in 0..4 { + if cr[j].abs() + ci[j].abs() > cmax { + cmax = cr[j].abs() + ci[j].abs(); + icmax = j; + } + } + if cmax < smini { + let bnorm = (b[0][0].abs() + b[0][1].abs()).max(b[1][0].abs() + b[1][1].abs()); + if smini < one && bnorm > one && bnorm > bignum * smini { + scale = one / bnorm; + } + let temp = scale / smini; + for (xr, br) in x.iter_mut().zip(&b) { + xr[0] = temp * br[0]; + xr[1] = temp * br[1]; + } + return (x, scale, temp * bnorm); + } + let p = PIVOT[icmax]; + let (ur11, ui11) = (cr[icmax], ci[icmax]); + let (cr21, ci21) = (cr[p[1]], ci[p[1]]); + let (ur12, ui12) = (cr[p[2]], ci[p[2]]); + let (cr22, ci22) = (cr[p[3]], ci[p[3]]); + let (ur11r, ui11r, lr21, li21, ur12s, ui12s, mut ur22, mut ui22); + if icmax == 0 || icmax == 3 { + // The off-diagonal elements of the pivoted `C` are real. + if ur11.abs() > ui11.abs() { + let temp = ui11 / ur11; + ur11r = one / (ur11 * (one + temp * temp)); + ui11r = -temp * ur11r; + } else { + let temp = ur11 / ui11; + ui11r = -one / (ui11 * (one + temp * temp)); + ur11r = -temp * ui11r; + } + lr21 = cr21 * ur11r; + li21 = cr21 * ui11r; + ur12s = ur12 * ur11r; + ui12s = ur12 * ui11r; + ur22 = cr22 - ur12 * lr21; + ui22 = ci22 - ur12 * li21; + } else { + // The diagonal elements of the pivoted `C` are real. + ur11r = one / ur11; + ui11r = zero; + lr21 = cr21 * ur11r; + li21 = ci21 * ur11r; + ur12s = ur12 * ur11r; + ui12s = ui12 * ur11r; + ur22 = cr22 - ur12 * lr21 + ui12 * li21; + ui22 = -ur12 * li21 - ui12 * lr21; + } + let u22abs = ur22.abs() + ui22.abs(); + if u22abs < smini { + ur22 = smini; + ui22 = zero; + } + let (mut br1, mut bi1, mut br2, mut bi2) = if RSWAP[icmax] { + (b[1][0], b[1][1], b[0][0], b[0][1]) + } else { + (b[0][0], b[0][1], b[1][0], b[1][1]) + }; + br2 = br2 - lr21 * br1 + li21 * bi1; + bi2 = bi2 - li21 * br1 - lr21 * bi1; + let bbnd = ((br1.abs() + bi1.abs()) * (u22abs * (ur11r.abs() + ui11r.abs()))) + .max(br2.abs() + bi2.abs()); + if bbnd > one && u22abs < one && bbnd >= bignum * u22abs { + scale = one / bbnd; + br1 *= scale; + bi1 *= scale; + br2 *= scale; + bi2 *= scale; + } + let (xr2, xi2) = divide(br2, bi2, ur22, ui22); + let xr1 = ur11r * br1 - ui11r * bi1 - ur12s * xr2 + ui12s * xi2; + let xi1 = ui11r * br1 + ur11r * bi1 - ui12s * xr2 - ur12s * xi2; + if ZSWAP[icmax] { + x = [[xr2, xi2], [xr1, xi1]]; + } else { + x = [[xr1, xi1], [xr2, xi2]]; + } + let mut xnorm = (xr1.abs() + xi1.abs()).max(xr2.abs() + xi2.abs()); + if xnorm > one && cmax > one && xnorm > bignum / cmax { + let temp = cmax / bignum; + for v in x.iter_mut().flatten() { + *v *= temp; + } + xnorm *= temp; + scale *= temp; + } + (x, scale, xnorm) +} diff --git a/lib/src/math/matrix/svd.rs b/lib/src/math/matrix/svd.rs new file mode 100644 index 0000000..1299a41 --- /dev/null +++ b/lib/src/math/matrix/svd.rs @@ -0,0 +1,229 @@ +//! The singular value decomposition: a reduction to a real bidiagonal matrix by Householder +//! reflections in panels (after one to triangular by QR for a tall matrix), then divide and +//! conquer on the bidiagonal matrix, whose singular vectors the reflections turn into those of the +//! matrix; the singular values alone by the dqds algorithm. Small matrices are reduced without +//! panels, their vectors by the implicit QR algorithm. + +mod bidiagonal; +mod dc; +mod dqds; +mod qr; +mod secular; +mod small; + +use super::properties::range_exponent; +use super::{Matrix, Scalar}; +use crate::math::complex::Float; +use bidiagonal::{apply_adjoint_right, reduce, triangularize}; + +/// The singular value decomposition `A = U·Σ·V*` of an `m × n` matrix, `k = min(m, n)`: `U` of +/// `m × k` and `V` of `n × k` with orthonormal columns, `Σ` the diagonal of the singular values, +/// real, not negative, in descending order. From [`Matrix::svd`]. +/// +/// # Examples +/// +/// ``` +/// use a::math::matrix::Matrix; +/// +/// let a: Matrix = Matrix::from_rows(&[[3.0, 0.0], [4.0, 5.0]]); +/// let svd = a.svd(); +/// let s = svd.singular_values(); +/// assert!((s[0] - 45f64.sqrt()).abs() < 1e-14 && (s[1] - 5f64.sqrt()).abs() < 1e-14); +/// let sigma = Matrix::from_diagonal(s); +/// let back = &(&svd.u() * &sigma) * &svd.v().transpose(); +/// assert!((&back - &a).as_slice().iter().all(|x| x.abs() < 1e-14)); +/// ``` +#[derive(Clone, Debug)] +pub struct Svd { + /// `U`, `m × k`. + u: Matrix, + /// The singular values, descending. + s: Vec, + /// `V`, `n × k`. + v: Matrix, +} + +impl Svd { + /// `U`, `m × min(m, n)`, the left singular vectors as orthonormal columns. + pub fn u(&self) -> Matrix { + self.u.clone() + } + + /// The singular values, real, not negative, in descending order. + pub fn singular_values(&self) -> &[T::Real] { + &self.s + } + + /// `V`, `n × min(m, n)`, the right singular vectors as orthonormal columns. + pub fn v(&self) -> Matrix { + self.v.clone() + } +} + +impl Matrix { + /// The singular value decomposition `A = U·Σ·V*`: see [`Svd`]. A reduction to a real + /// bidiagonal matrix by Householder reflections from both sides, in panels whose updates of + /// the rest of the matrix are products of blocks, then divide and conquer on the bidiagonal + /// matrix, Gu and Eisenstat's, and the reflections applied to its singular vectors in blocks: + /// about `9n³` operations for a square matrix, most of them in products of blocks. Up to + /// about a hundred columns (88 to 112 by the type), where that takes longer, the reduction + /// is made one column and row at a time, `U` and `V` are multiplied out from the reflections, + /// and the implicit QR algorithm's rotations are applied to them. A matrix with at least + /// `2.5` times as many rows as columns is first reduced to triangular by QR. A matrix with a + /// NaN or an infinity gives NaNs. + pub fn svd(&self) -> Svd { + let (s, uv) = svd(self, true); + let (u, v) = uv.unwrap_or_default(); + Svd { u, s, v } + } + + /// The singular values, real, not negative, in descending order, without `U` and `V`: + /// much less work than [`svd`](Matrix::svd). Those of the bidiagonal matrix come from the + /// dqds algorithm, to high relative accuracy, so that a bidiagonal matrix has even its + /// smallest singular values to nearly full precision; QR goes first from `1.75` times as + /// many rows as columns. + pub fn singular_values(&self) -> Vec { + svd(self, false).0 + } +} + +/// The singular values of `a`, and `U` and `V` when `want_uv` is set. +#[allow(clippy::type_complexity)] +pub(crate) fn svd( + a: &Matrix, + want_uv: bool, +) -> (Vec, Option<(Matrix, Matrix)>) { + let (mut s, uv, k) = svd_scaled(a, want_uv); + if k != 0 { + let down = T::Real::TWO.powi(-k); + for x in &mut s { + *x *= down; + } + } + (s, uv) +} + +/// [`svd`] with the singular values of `2^k·a` and `k`, a power of two that brings the largest +/// element into the range where products of elements neither overflow nor underflow: for what +/// depends only on the singular values' ratios (ranks, null spaces, condition numbers), which +/// stay finite where `a`'s own singular values would overflow or underflow. +#[allow(clippy::type_complexity)] +pub(crate) fn svd_scaled( + a: &Matrix, + want_uv: bool, +) -> (Vec, Option<(Matrix, Matrix)>, i32) { + let (m, n) = a.shape(); + // A wide matrix by its conjugate transpose, `A* = V·Σ·U*`. + let wide = m < n; + let (rows, cols) = if wide { (n, m) } else { (m, n) }; + let swap = |uv: Option<(Matrix, Matrix)>| { + if wide { uv.map(|(u, v)| (v, u)) } else { uv } + }; + if a.data.iter().any(|x| !x.is_finite()) { + let nan = T::from_parts(T::Real::NAN, T::Real::NAN); + let uv = want_uv.then(|| { + ( + Matrix::from_fn(rows, cols, |_, _| nan), + Matrix::from_fn(cols, cols, |_, _| nan), + ) + }); + return (vec![T::Real::NAN; cols], swap(uv), 0); + } + // Scaled by a power of two when the largest element is so small or so large that products + // of elements could underflow or overflow, as LAPACK's drivers do. + let k = range_exponent(&a.data); + let mut x = if wide { + adjoint(&a.data, m, n).data + } else { + a.data.clone() + }; + if k != 0 { + let up = T::Real::TWO.powi(k); + for x in &mut x { + *x = x.scale(up); + } + } + let (s, uv) = decompose(x, rows, cols, want_uv); + (s, swap(uv), k) +} + +/// [`svd`] of the `m × n` column-major `a`, `m ≥ n`, its elements finite and of a moderate size. +#[allow(clippy::type_complexity)] +fn decompose( + mut a: Vec, + m: usize, + n: usize, + want_uv: bool, +) -> (Vec, Option<(Matrix, Matrix)>) { + // QR first from `m ≥ 2.5n` with the vectors, from `m ≥ 1.75n` without, where it starts to + // pay. + if n > 0 + && if want_uv { + 2 * m >= 5 * n + } else { + 4 * m >= 7 * n + } + { + // `A = Q·R`, and `R = U·Σ·V*`: fewer operations than the reduction of `A` itself. + let taus = triangularize(&mut a, m, n); + let mut r = vec![T::zero(); n * n]; + for j in 0..n { + r[j * n..j * n + j + 1].copy_from_slice(&a[j * m..j * m + j + 1]); + } + let (s, uv) = decompose(r, n, n, want_uv); + let uv = uv.map(|(ur, v)| { + // `U* = [U_R*, 0]·Q*`. + let mut x = vec![T::zero(); n * m]; + for j in 0..n { + for (i, x) in x[j * n..(j + 1) * n].iter_mut().enumerate() { + *x = ur.data[j + i * n].conj(); + } + } + apply_adjoint_right(&a, m, m, &taus, &mut x, n); + (adjoint(&x, n, m), v) + }); + return (s, uv); + } + if n <= small::largest::(want_uv) { + return small::decompose(a, m, n, want_uv); + } + let (mut d, mut e, reduced) = reduce(a, m, n); + if !want_uv { + if !dqds::singular_values(&mut d, &e) { + e.push(T::Real::ZERO); + qr::bidiagonal_qr::<_, qr::Columns<_>, qr::Columns<_>>(&mut d, &mut e, None, None); + } + return (d, None); + } + let mut ub = vec![T::Real::ZERO; n * n]; + let mut vb = vec![T::Real::ZERO; n * n]; + dc::bidiagonal_svd(&mut d, &mut e, &mut ub, &mut vb, T::gemm_real); + // `U* = [U_B*, 0]·Q*` and `V* = V_B*·P*`, the reflections applied from the right. + let mut x = vec![T::zero(); n * m]; + let mut y = vec![T::zero(); n * n]; + for j in 0..n { + for i in 0..n { + x[i + j * n] = T::from_real(ub[j + i * n]); + y[i + j * n] = T::from_real(vb[j + i * n]); + } + } + reduced.apply_q_right(&mut x, n); + reduced.apply_p_right(&mut y, n); + (d, Some((adjoint(&x, n, m), adjoint(&y, n, n)))) +} + +/// The conjugate transpose of the `rows × cols` column-major `x`, in tiles. +fn adjoint(x: &[T], rows: usize, cols: usize) -> Matrix { + const TILE: usize = 16; + let mut a = Matrix::zeros(cols, rows); + for j0 in (0..rows).step_by(TILE) { + for i0 in (0..cols).step_by(TILE) { + for j in j0..(j0 + TILE).min(rows) { + for i in i0..(i0 + TILE).min(cols) { + a.data[i + j * cols] = x[j + i * rows].conj(); + } + } + } + } + a +} diff --git a/lib/src/math/matrix/svd/bidiagonal.rs b/lib/src/math/matrix/svd/bidiagonal.rs new file mode 100644 index 0000000..a3caad3 --- /dev/null +++ b/lib/src/math/matrix/svd/bidiagonal.rs @@ -0,0 +1,521 @@ +//! The reduction of a matrix to a real upper bidiagonal one, `B = Q*·A·P`, in panels as LAPACK's +//! `gebrd` and `labrd`: the reflections of a panel are applied to the rest of the matrix at once, +//! by two products of blocks, and within the panel the rest is only read, once per column, by +//! a pass that computes both of its products with vectors. The reduction to triangular that +//! comes first for tall matrices, and the products of the reflections with the singular vectors, +//! by blocks. + +use super::super::Scalar; +use super::super::blas::{Op, gemm}; +use super::super::householder::{Block, norm2, reflection}; +use crate::math::complex::Float; + +/// The width of the panels of the reduction. +const NB: usize = 16; + +/// The reflections applied together to the singular vectors. +const BLOCK: usize = 32; + +/// The width of the panels of the reduction to triangular. +const QB: usize = 16; + +/// The reduction of an `m × n` matrix, `m ≥ n`, to the real upper bidiagonal `B = Q*·A·P`: +/// `Q = H₀·H₁·…·H_{n-1}` and `P = G₀·G₁·…·G_{n-2}`, products of reflections. +pub(super) struct Bidiagonal { + /// `m × n`, the vectors of the reflections `H_j` below the diagonal. + a: Vec, + /// The rows. + m: usize, + /// The columns. + n: usize, + /// `n × n`, the vector of `G_j` in column `j` from row `j + 1`, its leading one held. + w: Vec, + /// The factors of `H_j`. + tauq: Vec, + /// The factors of `G_j`. + taup: Vec, +} + +/// Reduces the `m × n` column-major `a`, `m ≥ n`: the diagonal and the superdiagonal of `B`, +/// and the reflections. +pub(super) fn reduce( + mut a: Vec, + m: usize, + n: usize, +) -> (Vec, Vec, Bidiagonal) { + let mut d = vec![T::Real::ZERO; n]; + let mut e = vec![T::Real::ZERO; n.saturating_sub(1)]; + let mut tauq = vec![T::zero(); n]; + let mut taup = vec![T::zero(); n.saturating_sub(1)]; + let mut w = vec![T::zero(); n * n]; + let nb = NB.min(n); + // `X` (`m × nb`) and `Y` (`n × nb`) of a panel: the reflections of the panel change the rest + // of the matrix by `-V·Y* - X·W*`, `V` and `W` the vectors of the left and right ones. + let mut x = vec![T::zero(); m * nb]; + let mut y = vec![T::zero(); n * nb]; + let mut p = 0; + while p < n { + let ib = nb.min(n - p); + let mut panel = Panel { + a: &mut a, + m, + n, + p, + x: &mut x, + y: &mut y, + w: &mut w, + }; + for i in 0..ib { + let (dq, tq, ep) = panel.step(i); + d[p + i] = dq; + tauq[p + i] = tq; + if let Some((ev, tp)) = ep { + e[p + i] = ev; + taup[p + i] = tp; + } + } + let q = p + ib; + if q < n { + // The rest: `A₂₂ -= V₂·Y₂* + X₂·W₂*`. + let (left, right) = a.split_at_mut(q * m); + let c = &mut right[q..]; + let minus = -T::one(); + gemm( + m - q, + n - q, + ib, + minus, + &left[q + p * m..], + m, + Op::N, + &y[q..], + n, + Op::C, + T::one(), + c, + m, + ); + gemm( + m - q, + n - q, + ib, + minus, + &x[q..], + m, + Op::N, + &w[q + p * n..], + n, + Op::C, + T::one(), + c, + m, + ); + } + p = q; + } + ( + d, + e, + Bidiagonal { + a, + m, + n, + w, + tauq, + taup, + }, + ) +} + +/// The state of the reduction of the panel from column `p`. +struct Panel<'a, T> { + /// The matrix, `m × n`. + a: &'a mut [T], + /// The rows. + m: usize, + /// The columns. + n: usize, + /// The first column of the panel. + p: usize, + /// `X`, `m × nb`. + x: &'a mut [T], + /// `Y`, `n × nb`. + y: &'a mut [T], + /// The vectors of the right reflections, `n × n`. + w: &'a mut [T], +} + +impl Panel<'_, T> { + /// Reduces column and row `c = p + i`: the diagonal element and the factor of the left + /// reflection, and when there is one, the superdiagonal element and the factor of the right. + #[allow(clippy::type_complexity)] + fn step(&mut self, i: usize) -> (T::Real, T, Option<(T::Real, T)>) { + let (m, n, p) = (self.m, self.n, self.p); + let c = p + i; + let ldy = n; + let mut t = vec![T::zero(); i + 1]; + // Column `c`, rows `c..`: `-= V·conj(Y[c, ..i]) + X·conj(W[c, ..i])`. + { + let (left, right) = self.a.split_at_mut(c * m); + let col = &mut right[c..m]; + if i > 0 { + for (l, t) in t[..i].iter_mut().enumerate() { + *t = -self.y[c + l * ldy].conj(); + } + gemv_n(m - c, i, &left[c + p * m..], m, &t[..i], col); + for (l, t) in t[..i].iter_mut().enumerate() { + *t = -self.w[c + (p + l) * n].conj(); + } + gemv_n(m - c, i, &self.x[c..], m, &t[..i], col); + } + } + let (tauq, beta) = { + let (head, tail) = self.a[c * m + c..(c + 1) * m].split_at_mut(1); + reflector(head[0], tail) + }; + self.a[c + c * m] = T::one(); + if c + 1 == n { + return (beta.re(), tauq, None); + } + let nc = n - c - 1; + let a = &*self.a; + let v = &a[c + c * m..(c + 1) * m]; + // `t2 = V*·v` and `t3 = X*·v`, for `Y[.., i] = τ·(A*·v - Y·t2 - W·t3)`. + let mut t2 = vec![T::zero(); i]; + let mut t3 = vec![T::zero(); i]; + gemv_c(m - c, i, &a[c + p * m..], m, v, &mut t2); + gemv_c(m - c, i, &self.x[c..], m, v, &mut t3); + let mut corr = vec![T::zero(); nc]; + gemv_n(nc, i, &self.y[c + 1..], ldy, &t2, &mut corr); + gemv_n(nc, i, &self.w[c + 1 + p * n..], n, &t3, &mut corr); + // Row `c` conjugated, all but `-Y[.., i]`: `conj(A[c, ..]) - Y·conj(V[c, ..]) - + // W·conj(X[c, ..])`. + let mut r: Vec = (0..nc).map(|j| a[c + (c + 1 + j) * m].conj()).collect(); + for (l, t) in t[..i].iter_mut().enumerate() { + *t = -a[c + (p + l) * m].conj(); + } + gemv_n(nc, i, &self.y[c + 1..], ldy, &t[..i], &mut r); + for (l, t) in t[..i].iter_mut().enumerate() { + *t = -self.x[c + l * m].conj(); + } + gemv_n(nc, i, &self.w[c + 1 + p * n..], n, &t[..i], &mut r); + // One pass over the rest: `Y[.., i]`, the row, and `z = A₂₂·r` with it. + let mut z = vec![T::zero(); m - c - 1]; + let yi = &mut self.y[c + 1 + i * ldy..c + 1 + i * ldy + nc]; + fused( + &a[c + (c + 1) * m..], + m, + m - c, + nc, + v, + tauq, + &corr, + yi, + &mut r, + &mut z, + ); + let alpha = r[0]; + let (taup, beta2) = { + let (head, tail) = r.split_at_mut(1); + reflector(head[0], tail) + }; + let wc = &mut self.w[c + 1 + c * n..(c + 1) * n]; + wc[0] = T::one(); + wc[1..].copy_from_slice(&r[1..]); + // `X[.., i] = τ·(A·w - V·(Y*·w) - X·(W*·w))`, `A·w = (z - β·A[.., c + 1])/(α - β)`. + let xi = c + 1 + i * m; + if taup == T::zero() { + self.x[xi..xi + m - c - 1].fill(T::zero()); + } else { + let s = T::one() / (alpha - beta2); + let a1 = &a[c + 1 + (c + 1) * m..(c + 2) * m]; + let mut xc: Vec = z + .iter() + .zip(a1) + .map(|(&z, &a)| (z - a * beta2) * s) + .collect(); + let wc = &self.w[c + 1 + c * n..(c + 1) * n]; + let mut t4 = vec![T::zero(); i + 1]; + let mut t5 = vec![T::zero(); i]; + gemv_c(nc, i + 1, &self.y[c + 1..], ldy, wc, &mut t4); + gemv_c(nc, i, &self.w[c + 1 + p * n..], n, wc, &mut t5); + for t in t4.iter_mut().chain(t5.iter_mut()) { + *t = -*t; + } + gemv_n(m - c - 1, i + 1, &a[c + 1 + p * m..], m, &t4, &mut xc); + gemv_n(m - c - 1, i, &self.x[c + 1..], m, &t5, &mut xc); + for (x, &v) in self.x[xi..xi + m - c - 1].iter_mut().zip(&xc) { + *x = v * taup; + } + } + (beta.re(), tauq, Some((beta2.re(), taup))) + } +} + +/// Reduces the `m × n` column-major `a`, `m ≥ n`, to upper triangular by Householder +/// reflections, as LAPACK's `geqrf`: `R` on and above the diagonal, the vectors below it, and the +/// factors returned. Panels of `QB` columns are reduced with the products by vectors of the +/// reduction to bidiagonal, and their reflections applied to the rest of the matrix in blocks. +pub(super) fn triangularize(a: &mut [T], m: usize, n: usize) -> Vec { + let mut taus = vec![T::zero(); n]; + let mut p = 0; + while p < n { + let q = (p + QB).min(n); + for j in p..q { + let (tau, beta) = { + let (head, tail) = a[j * m + j..(j + 1) * m].split_at_mut(1); + reflector(head[0], tail) + }; + a[j + j * m] = beta; + taus[j] = tau; + if tau == T::zero() || j + 1 == q { + continue; + } + // The rest of the panel: `c -= conj(τ)·v·(v*·c)`, two columns at a time. + let ct = tau.conj(); + let (left, right) = a.split_at_mut((j + 1) * m); + let v = &left[j * m + j + 1..]; + let mut cols = right[j..].chunks_mut(m).take(q - j - 1); + loop { + match (cols.next(), cols.next()) { + (Some(c0), Some(c1)) => { + let (s0, s1) = dot2_c(&c0[1..m - j], &c1[1..m - j], v); + let w0 = ct * (c0[0] + s0.conj()); + let w1 = ct * (c1[0] + s1.conj()); + c0[0] -= w0; + c1[0] -= w1; + let it = c0[1..m - j].iter_mut().zip(c1[1..m - j].iter_mut()); + for ((x0, x1), &v) in it.zip(v) { + *x0 -= v * w0; + *x1 -= v * w1; + } + } + (Some(c0), None) => { + let w0 = ct * (c0[0] + dot_c(&c0[1..m - j], v).conj()); + c0[0] -= w0; + for (x0, &v) in c0[1..m - j].iter_mut().zip(v) { + *x0 -= v * w0; + } + break; + } + _ => break, + } + } + } + if q < n { + let block = Block::new(&a[p + p * m..], m, m - p, &taus[p..q]); + block.apply_left(true, n - q, &mut a[p + q * m..], m); + } + p = q; + } + taus +} + +/// The reflection `H = I - τ·v·v*`, `v = (1, x)`, with `H*·(alpha, x) = (β, 0)` and `β` real, as +/// [`reflector`](super::super::householder::reflector) but with the norm of `x` from one pass of +/// sums of squares when they can neither overflow nor underflow. +pub(super) fn reflector(alpha: T, x: &mut [T]) -> (T, T) { + let xnorm = norm(x); + if xnorm == T::Real::ZERO && alpha.im() == T::Real::ZERO { + return (T::zero(), alpha); + } + reflection(alpha, alpha.abs().hypot(xnorm), x) +} + +/// The 2-norm of `x`: the root of the sum of the squares in eight sums, or [`norm2`]'s scaled +/// one when the sum is out of the range where no square can have overflowed or underflowed +/// enough to matter, zero included. +fn norm(x: &[T]) -> T::Real { + let (chunks, rest) = x.as_chunks::<8>(); + let mut s = [T::Real::ZERO; 8]; + for c in chunks { + for l in 0..8 { + s[l] += c[l].abs2(); + } + } + let mut t = ((s[0] + s[1]) + (s[2] + s[3])) + ((s[4] + s[5]) + (s[6] + s[7])); + for v in rest { + t += v.abs2(); + } + let small = T::Real::MIN_POSITIVE / T::Real::EPSILON; + if t >= small && t <= T::Real::MAX * T::Real::EPSILON { + t.sqrt() + } else { + norm2(x) + } +} + +/// `y += A·x`, `A` of `rows × cols` with the leading dimension `lda`. +fn gemv_n(rows: usize, cols: usize, a: &[T], lda: usize, x: &[T], y: &mut [T]) { + let y = &mut y[..rows]; + let mut j = 0; + while j + 4 <= cols { + let (x0, x1, x2, x3) = (x[j], x[j + 1], x[j + 2], x[j + 3]); + let a0 = &a[j * lda..j * lda + rows]; + let a1 = &a[(j + 1) * lda..(j + 1) * lda + rows]; + let a2 = &a[(j + 2) * lda..(j + 2) * lda + rows]; + let a3 = &a[(j + 3) * lda..(j + 3) * lda + rows]; + for ((((y, &a0), &a1), &a2), &a3) in y.iter_mut().zip(a0).zip(a1).zip(a2).zip(a3) { + *y += a0 * x0 + a1 * x1 + a2 * x2 + a3 * x3; + } + j += 4; + } + while j < cols { + let x0 = x[j]; + for (y, &a0) in y.iter_mut().zip(&a[j * lda..j * lda + rows]) { + *y += a0 * x0; + } + j += 1; + } +} + +/// `y = A*·x`, `A` of `rows × cols` with the leading dimension `lda`. +fn gemv_c(rows: usize, cols: usize, a: &[T], lda: usize, x: &[T], y: &mut [T]) { + for (j, y) in y[..cols].iter_mut().enumerate() { + *y = dot_c(&a[j * lda..j * lda + rows], &x[..rows]); + } +} + +/// `a*·x`, in eight sums, which the compiler keeps in vector registers. +#[inline] +fn dot_c(a: &[T], x: &[T]) -> T { + let (xc, xr) = x.as_chunks::<8>(); + let (ac, ar) = a[..x.len()].as_chunks::<8>(); + let mut s = [T::zero(); 8]; + for (a, x) in ac.iter().zip(xc) { + for l in 0..8 { + s[l] += a[l].conj() * x[l]; + } + } + let mut t = ((s[0] + s[1]) + (s[2] + s[3])) + ((s[4] + s[5]) + (s[6] + s[7])); + for (a, x) in ar.iter().zip(xr) { + t += a.conj() * *x; + } + t +} + +/// `(a*·x, b*·x)`, each in eight sums, the loads of `x` shared. +#[inline] +fn dot2_c(a: &[T], b: &[T], x: &[T]) -> (T, T) { + let n = x.len(); + let (xc, xr) = x.as_chunks::<8>(); + let (ac, ar) = a[..n].as_chunks::<8>(); + let (bc, br) = b[..n].as_chunks::<8>(); + let (mut s, mut t) = ([T::zero(); 8], [T::zero(); 8]); + for ((a, b), x) in ac.iter().zip(bc).zip(xc) { + for l in 0..8 { + s[l] += a[l].conj() * x[l]; + } + for l in 0..8 { + t[l] += b[l].conj() * x[l]; + } + } + let mut u = ((s[0] + s[1]) + (s[2] + s[3])) + ((s[4] + s[5]) + (s[6] + s[7])); + let mut w = ((t[0] + t[1]) + (t[2] + t[3])) + ((t[4] + t[5]) + (t[6] + t[7])); + for ((a, b), x) in ar.iter().zip(br).zip(xr) { + u += a.conj() * *x; + w += b.conj() * *x; + } + (u, w) +} + +/// The pass over the rest of the matrix, `A` of `rows × cols` from row and column `c` (its +/// first column `c + 1`): for each column `j`, `y_j = τ·(A[.., j]*·v - corr_j)`, +/// `r_j -= y_j`, then `z += A[1.., j]·r_j`. +#[allow(clippy::too_many_arguments)] +fn fused( + a: &[T], + lda: usize, + rows: usize, + cols: usize, + v: &[T], + tau: T, + corr: &[T], + y: &mut [T], + r: &mut [T], + z: &mut [T], +) { + let v = &v[..rows]; + let z = &mut z[..rows - 1]; + let mut j = 0; + while j + 4 <= cols { + let c0 = &a[j * lda..j * lda + rows]; + let c1 = &a[(j + 1) * lda..(j + 1) * lda + rows]; + let c2 = &a[(j + 2) * lda..(j + 2) * lda + rows]; + let c3 = &a[(j + 3) * lda..(j + 3) * lda + rows]; + let ((s0, s1), (s2, s3)) = (dot2_c(c0, c1, v), dot2_c(c2, c3, v)); + let s = [s0, s1, s2, s3]; + let mut rr = [T::zero(); 4]; + for q in 0..4 { + let yj = tau * (s[q] - corr[j + q]); + y[j + q] = yj; + r[j + q] -= yj; + rr[q] = r[j + q]; + } + let it = z + .iter_mut() + .zip(&c0[1..]) + .zip(&c1[1..]) + .zip(&c2[1..]) + .zip(&c3[1..]); + for ((((z, &a0), &a1), &a2), &a3) in it { + *z += (a0 * rr[0] + a1 * rr[1]) + (a2 * rr[2] + a3 * rr[3]); + } + j += 4; + } + while j < cols { + let c0 = &a[j * lda..j * lda + rows]; + let yj = tau * (dot_c(c0, v) - corr[j]); + y[j] = yj; + r[j] -= yj; + let rj = r[j]; + for (z, &a0) in z.iter_mut().zip(&c0[1..]) { + *z += a0 * rj; + } + j += 1; + } +} + +impl Bidiagonal { + /// `x = x·Q*`, `x` of `rows × m`. + pub fn apply_q_right(&self, x: &mut [T], rows: usize) { + apply_adjoint_right(&self.a, self.m, self.m, &self.tauq, x, rows); + } + + /// `x = x·P*`, `x` of `rows × n`. + pub fn apply_p_right(&self, x: &mut [T], rows: usize) { + let n = self.n; + if n > 1 { + apply_adjoint_right(&self.w[1..], n, n - 1, &self.taup, &mut x[rows..], rows); + } + } +} + +/// `x = x·(H₀·H₁·…·H_{k-1})*` for the reflections whose vectors are below the diagonal of the +/// `m × k` block `a` and whose factors are `taus`, `x` of `rows × m`: by blocks of `BLOCK` +/// reflections from the last, `x·(I - V·T·V*)* = x - (x·V)·T*·V*`, whose products all take their +/// left operand as it is held. +pub(super) fn apply_adjoint_right( + a: &[T], + lda: usize, + m: usize, + taus: &[T], + x: &mut [T], + rows: usize, +) { + let k = taus.len(); + let mut end = k; + while end > 0 { + let start = (end - 1) / BLOCK * BLOCK; + let mut block = Block::new(&a[start + start * lda..], lda, m - start, &taus[start..end]); + let b = block.k; + let t = block.t.clone(); + for i in 0..b { + for j in 0..b { + block.t[i + j * b] = t[j + i * b].conj(); + } + } + block.apply_right(rows, &mut x[start * rows..], rows); + end = start; + } +} diff --git a/lib/src/math/matrix/svd/dc.rs b/lib/src/math/matrix/svd/dc.rs new file mode 100644 index 0000000..b559864 --- /dev/null +++ b/lib/src/math/matrix/svd/dc.rs @@ -0,0 +1,420 @@ +//! The singular value decomposition of a real upper bidiagonal matrix by divide and conquer, as +//! LAPACK's `bdsdc` and `lasd0`–`lasd3`: the matrix split in halves at a row, the halves solved, +//! and their decompositions merged through a broken arrow matrix whose singular vectors update +//! theirs in products of blocks. + +use super::qr::{Columns, bidiagonal_qr}; +use super::secular::arrow; +use crate::math::complex::Float; + +/// `c = β·c + α·a·b` on real column-major blocks: `(m, n, k, α, a, lda, b, ldb, β, c, ldc)`. +pub(super) type Gemm = fn(usize, usize, usize, R, &[R], usize, &[R], usize, R, &mut [R], usize); + +/// Problems up to this order are solved by the implicit QR algorithm. +pub(super) const LEAF: usize = 25; + +/// The decomposition `B = U·Σ·Vᵀ` of the `n × n` upper bidiagonal `B` with the diagonal `d` and +/// the superdiagonal `e`: `d` turned into the singular values, descending, and `u` and `v`, of +/// `n × n`, column-major, set to `U` and `V`. `false` when the implicit QR algorithm of a small +/// problem did not converge. The products of blocks are `mm`'s. +pub(super) fn bidiagonal_svd( + d: &mut [R], + e: &mut [R], + u: &mut [R], + v: &mut [R], + mm: Gemm, +) -> bool { + let n = d.len(); + u.fill(R::ZERO); + v.fill(R::ZERO); + for j in 0..n { + u[j + j * n] = R::ONE; + v[j + j * n] = R::ONE; + } + if n <= LEAF { + let mut e = e.to_vec(); + e.resize(n, R::ZERO); + let mut cu = Columns { + data: u, + rows: n, + ld: n, + }; + let mut cv = Columns { + data: v, + rows: n, + ld: n, + }; + return bidiagonal_qr(d, &mut e, Some(&mut cu), Some(&mut cv)); + } + // Scaled to the largest element, and split where the superdiagonal is negligible. + let scale = d + .iter() + .chain(e.iter()) + .fold(R::ZERO, |s, &x| s.max(x.abs())); + if scale == R::ZERO { + return true; + } + for x in d.iter_mut().chain(e.iter_mut()) { + *x /= scale; + } + let tiny = R::from_f64(0.9) * R::EPSILON * R::HALF; + let mut ok = true; + let mut order = Vec::with_capacity(n); + let mut start = 0; + for i in 0..n { + if i + 1 < n && e[i].abs() >= tiny { + continue; + } + if i + 1 < n { + e[i] = R::ZERO; + } + let size = i + 1 - start; + let (o, good) = solve(start, size, 0, d, e, u, v, n, mm); + ok &= good; + order.extend(o.into_iter().map(|j| start + j)); + start = i + 1; + } + for x in d.iter_mut() { + *x *= scale; + } + // Descending, the columns of `U` and `V` with them. + order.sort_by(|&a, &b| { + d[b].partial_cmp(&d[a]) + .unwrap_or(core::cmp::Ordering::Equal) + }); + let (u0, v0, d0) = (u.to_vec(), v.to_vec(), d.to_vec()); + for (j, &o) in order.iter().enumerate() { + d[j] = d0[o]; + u[j * n..(j + 1) * n].copy_from_slice(&u0[o * n..(o + 1) * n]); + v[j * n..(j + 1) * n].copy_from_slice(&v0[o * n..(o + 1) * n]); + } + ok +} + +/// Solves the `n × (n + sqre)` upper bidiagonal block from row and column `f`, with the diagonal +/// `d[f..f + n]` and the superdiagonal `e[f..f + n - 1 + sqre]`, into the diagonal blocks of `u` +/// (`n × n`) and `v` (`(n + sqre) × (n + sqre)`), which hold the identity there on entry: the +/// singular values into `d[f..f + n]`, and their order, ascending, returned with whether all +/// converged. With `sqre`, column `n` of the block of `v` is the null vector. +#[allow(clippy::too_many_arguments)] +fn solve( + f: usize, + n: usize, + sqre: usize, + d: &mut [R], + e: &mut [R], + u: &mut [R], + v: &mut [R], + ld: usize, + mm: Gemm, +) -> (Vec, bool) { + if n <= LEAF { + return leaf(f, n, sqre, d, e, u, v, ld); + } + let nl = n / 2; + let nr = n - nl - 1; + let (left, a) = solve(f, nl, 1, d, e, u, v, ld, mm); + let (right, b) = solve(f + nl + 1, nr, sqre, d, e, u, v, ld, mm); + let order = merge(f, nl, nr, sqre, d, e, u, v, ld, &left, &right, mm); + (order, a && b) +} + +/// [`solve`] by the implicit QR algorithm, a column past a square block first rotated away. +#[allow(clippy::too_many_arguments)] +fn leaf( + f: usize, + n: usize, + sqre: usize, + d: &mut [R], + e: &mut [R], + u: &mut [R], + v: &mut [R], + ld: usize, +) -> (Vec, bool) { + let mut s = d[f..f + n].to_vec(); + let mut t = vec![R::ZERO; n]; + t[..n - 1].copy_from_slice(&e[f..f + n - 1]); + let mut cv = Columns { + data: &mut v[f + f * ld..], + rows: n + sqre, + ld, + }; + if sqre == 1 { + // Rotations from the right chase the element of the extra column up and out. + let mut g = e[f + n - 1]; + for k in (0..n).rev() { + let (r, cs, sn) = super::qr::rotg(s[k], g); + s[k] = r; + if k > 0 { + g = -sn * t[k - 1]; + t[k - 1] *= cs; + } + cv.rotate(k, n, cs, sn); + } + } + let mut cu = Columns { + data: &mut u[f + f * ld..], + rows: n, + ld, + }; + let ok = bidiagonal_qr(&mut s, &mut t, Some(&mut cu), Some(&mut cv)); + d[f..f + n].copy_from_slice(&s); + ((0..n).rev().collect(), ok) +} + +/// Where an entry's vectors come from: the left half, the right, or both after a rotation of +/// deflation mixed them. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Side { + /// Rows of the left half only. + Left, + /// Rows of both halves. + Both, + /// Rows of the right half only. + Right, +} + +/// Merges the solved halves of the block of [`solve`], `nl` rows above row `f + nl` and `nr` +/// below, whose ascending orders are `left` and `right`, as LAPACK's `lasd1`: returns the order +/// of the block's singular values. +#[allow(clippy::too_many_arguments)] +fn merge( + f: usize, + nl: usize, + nr: usize, + sqre: usize, + d: &mut [R], + e: &[R], + u: &mut [R], + v: &mut [R], + ld: usize, + left: &[usize], + right: &[usize], + mm: Gemm, +) -> Vec { + let n = nl + nr + 1; + let m = n + sqre; + let mid = f + nl; + // Scaled to the largest element. + let mut scale = d[mid].abs().max(e[mid].abs()); + for j in (f..f + n).filter(|&j| j != mid) { + scale = scale.max(d[j].abs()); + } + if scale == R::ZERO { + scale = R::ONE; + } + let (alpha, beta) = (d[mid] / scale, e[mid] / scale); + // The entries of the arrow, ascending: the value, the element of `z`, the column of the + // old vectors in the block, the side. + let mut ent: Vec<(R, R, usize, Side)> = Vec::with_capacity(n - 1); + { + let (mut a, mut b) = (left.iter().peekable(), right.iter().peekable()); + let lval = |c: usize| d[f + c] / scale; + let rval = |c: usize| d[mid + 1 + c] / scale; + loop { + let take_left = match (a.peek(), b.peek()) { + (Some(&&x), Some(&&y)) => lval(x) <= rval(y), + (Some(_), None) => true, + (None, Some(_)) => false, + (None, None) => break, + }; + if take_left { + let c = *a.next().unwrap_or(&0); + ent.push((lval(c), alpha * v[mid + (f + c) * ld], c, Side::Left)); + } else { + let c = *b.next().unwrap_or(&0); + let col = nl + 1 + c; + ent.push(( + rval(c), + beta * v[mid + 1 + (f + col) * ld], + col, + Side::Right, + )); + } + } + } + let za = alpha * v[mid + mid * ld]; + let zb = if sqre == 1 { + beta * v[mid + 1 + (f + n) * ld] + } else { + R::ZERO + }; + // Deflation, as LAPACK's `lasd2`: an entry whose `z` is negligible keeps its vectors, and of + // two entries with values too close a rotation makes the first's `z` zero. + let tol = R::from_f64(8.0) * R::EPSILON * R::HALF; + let dmax = ent.last().map_or(R::ZERO, |x| x.0); + let tol = tol * alpha.abs().max(beta.abs()).max(dmax); + let mut kept: Vec = Vec::with_capacity(n); + let mut deflated: Vec = Vec::new(); + for j in 0..ent.len() { + if ent[j].1.abs() <= tol { + deflated.push(j); + continue; + } + if let Some(&p) = kept.last() + && (ent[j].0 - ent[p].0).abs() <= tol + { + let (zp, zj) = (ent[p].1, ent[j].1); + let r = zj.hypot(zp); + let (c, s) = (zj / r, -zp / r); + ent[j].1 = r; + ent[p].1 = R::ZERO; + let (cp, cj) = (f + ent[p].2, f + ent[j].2); + Columns { + data: &mut u[f..], + rows: n, + ld, + } + .rotate(cp, cj, c, s); + Columns { + data: &mut v[f..], + rows: m, + ld, + } + .rotate(cp, cj, c, s); + if ent[j].3 != ent[p].3 { + ent[j].3 = Side::Both; + } + kept.pop(); + deflated.push(p); + } + kept.push(j); + } + deflated.sort_unstable(); + if let Some(&j) = kept.first() + && ent[j].0.abs() <= tol * R::HALF + { + ent[j].0 = tol * R::HALF; + } + // The first column of the arrow: the two null vectors of the halves rotated together. + let (z0, c, s) = if sqre == 1 { + let r = za.hypot(zb); + if r <= tol { + (tol, R::ONE, R::ZERO) + } else { + (r, za / r, zb / r) + } + } else if za.abs() <= tol { + (tol, R::ONE, R::ZERO) + } else { + (za, R::ONE, R::ZERO) + }; + // The kept entries in the order of the products: the left ones, the first column, both, the + // right ones, so that each half of the rows takes one block of columns. + let k = kept.len() + 1; + let count = |side: Side| kept.iter().filter(|&&j| ent[j].3 == side).count(); + let (nla, nb) = (count(Side::Left), count(Side::Both)); + let mut next = [0, nla + 1, nla + 1 + nb]; + let mut pos = vec![nla; k]; + for (p, &j) in pos[1..].iter_mut().zip(&kept) { + let g = ent[j].3 as usize; + *p = next[g]; + next[g] += 1; + } + let mut dk = vec![R::ZERO; k]; + let mut zk = vec![z0; k]; + for (i, &j) in kept.iter().enumerate() { + dk[i + 1] = ent[j].0; + zk[i + 1] = ent[j].1; + } + // The old vectors, gathered in that order. + let mut uw = vec![R::ZERO; n * k]; + let mut vw = vec![R::ZERO; m * k]; + for (&p, &j) in pos[1..].iter().zip(&kept) { + let col = f + ent[j].2; + uw[p * n..(p + 1) * n].copy_from_slice(&u[f + col * ld..f + n + col * ld]); + vw[p * m..(p + 1) * m].copy_from_slice(&v[f + col * ld..f + m + col * ld]); + } + uw[nl + nla * n] = R::ONE; + let vnull: Vec = (0..m) + .map(|r| { + let x = v[f + r + mid * ld]; + let y = if sqre == 1 { + v[f + r + (f + n) * ld] + } else { + R::ZERO + }; + vw[r + nla * m] = c * x + s * y; + c * y - s * x + }) + .collect(); + let defl: Vec<(R, Vec, Vec)> = deflated + .iter() + .map(|&j| { + let col = f + ent[j].2; + let uc = u[f + col * ld..f + n + col * ld].to_vec(); + let vc = v[f + col * ld..f + m + col * ld].to_vec(); + (ent[j].0, uc, vc) + }) + .collect(); + // The arrow's decomposition, and the products that update the vectors. + let mut sigma = vec![R::ZERO; k]; + let mut ua = vec![R::ZERO; k * k]; + let mut va = vec![R::ZERO; k * k]; + arrow(&dk, &zk, &pos, &mut sigma, &mut ua, &mut va); + let (top, bottom) = (nla + 1 + nb, k - nla); + let (one, zero) = (R::ONE, R::ZERO); + let ub = &mut u[f + f * ld..]; + mm(nl, k, top, one, &uw, n, &ua, k, zero, ub, ld); + for i in 0..k { + ub[nl + i * ld] = ua[nla + i * k]; + } + let ub = &mut u[mid + 1 + f * ld..]; + mm( + nr, + k, + bottom, + one, + &uw[nl + 1 + nla * n..], + n, + &ua[nla..], + k, + zero, + ub, + ld, + ); + let vb = &mut v[f + f * ld..]; + mm(nl + 1, k, top, one, &vw, m, &va, k, zero, vb, ld); + let vb = &mut v[mid + 1 + f * ld..]; + let rows = nr + sqre; + mm( + rows, + k, + bottom, + one, + &vw[nl + 1 + nla * m..], + m, + &va[nla..], + k, + zero, + vb, + ld, + ); + for (i, (_, uc, vc)) in defl.iter().enumerate() { + let col = f + k + i; + u[f + col * ld..f + n + col * ld].copy_from_slice(uc); + v[f + col * ld..f + m + col * ld].copy_from_slice(vc); + } + if sqre == 1 { + v[f + (f + n) * ld..f + m + (f + n) * ld].copy_from_slice(&vnull); + } + // The values, and their order: the arrow's ascending, merged with the deflated ones. + for (i, &x) in sigma.iter().enumerate() { + d[f + i] = x * scale; + } + for (i, (x, _, _)) in defl.iter().enumerate() { + d[f + k + i] = *x * scale; + } + let mut order = Vec::with_capacity(n); + let (mut a, mut b) = (0, k); + while a < k || b < n { + if b == n || (a < k && d[f + a] <= d[f + b]) { + order.push(a); + a += 1; + } else { + order.push(b); + b += 1; + } + } + order +} diff --git a/lib/src/math/matrix/svd/dqds.rs b/lib/src/math/matrix/svd/dqds.rs new file mode 100644 index 0000000..4324900 --- /dev/null +++ b/lib/src/math/matrix/svd/dqds.rs @@ -0,0 +1,934 @@ +//! The singular values of a real bidiagonal matrix by the differential quotient-difference +//! algorithm with shifts, dqds, as LAPACK's `lasq1`–`lasq6`: to high relative accuracy, and +//! with no square roots in its inner loop. The qd array `z` is indexed from 1 as in LAPACK, its +//! element 0 unused. + +use crate::math::complex::Float; + +/// The singular values of the upper bidiagonal matrix with the diagonal `d` and the +/// superdiagonal `e`, into `d`, descending; `false`, `d` untouched, when dqds did not converge. +pub(super) fn singular_values(d: &mut [R], e: &[R]) -> bool { + let n = d.len(); + if n == 0 { + return true; + } + if n == 1 { + d[0] = d[0].abs(); + return true; + } + if n == 2 { + let (min, max) = las2(d[0], e[0], d[1]); + d[0] = max; + d[1] = min; + return true; + } + let mut sigmx = e.iter().fold(R::ZERO, |s, &x| s.max(x.abs())); + if sigmx == R::ZERO { + for d in d.iter_mut() { + *d = d.abs(); + } + d.sort_by(|a, b| b.partial_cmp(a).unwrap_or(core::cmp::Ordering::Equal)); + return true; + } + for x in d.iter() { + sigmx = sigmx.max(x.abs()); + } + // `q₁, e₁, q₂, …, q_n`, scaled so that the largest square is near the top of the range, then + // squared. + let eps = R::EPSILON; + let safmin = R::MIN_POSITIVE; + let scale = (eps / safmin).sqrt(); + let mut z = vec![R::ZERO; 4 * n + 1]; + for i in 0..n { + z[2 * i + 1] = d[i].abs(); + if i + 1 < n { + z[2 * i + 2] = e[i].abs(); + } + } + rescale(&mut z[1..2 * n], sigmx, scale); + for x in &mut z[1..2 * n] { + *x = *x * *x; + } + z[2 * n] = R::ZERO; + if !lasq2(n, &mut z) { + return false; + } + for i in 0..n { + d[i] = z[i + 1].sqrt(); + } + rescale(d, scale, sigmx); + true +} + +/// `x · (to / from)` without overflow or underflow on the way, as LAPACK's `lascl`. +fn rescale(x: &mut [R], from: R, to: R) { + let small = R::MIN_POSITIVE; + let big = R::ONE / small; + let (mut from, mut to) = (from, to); + loop { + let from1 = from * small; + let (mul, done); + if from1 == from { + // `from` is infinite. + mul = to / from; + done = true; + } else { + let to1 = to / big; + if to1 == to { + // `to` is zero or infinite. + mul = to; + done = true; + from = R::ONE; + } else if from1.abs() > to.abs() && to != R::ZERO { + mul = small; + done = false; + from = from1; + } else if to1.abs() > from.abs() { + mul = big; + done = false; + to = to1; + } else { + mul = to / from; + done = true; + } + } + for x in x.iter_mut() { + *x *= mul; + } + if done { + return; + } + } +} + +/// The singular values `(min, max)` of the upper triangular `[[f, g], [0, h]]`, as LAPACK's +/// `las2`. +fn las2(f: R, g: R, h: R) -> (R, R) { + let (one, two) = (R::ONE, R::TWO); + let (fa, ga, ha) = (f.abs(), g.abs(), h.abs()); + let (fhmn, fhmx) = (fa.min(ha), fa.max(ha)); + if fhmn == R::ZERO { + let max = if fhmx == R::ZERO { + ga + } else { + let (lo, hi) = (fhmx.min(ga), fhmx.max(ga)); + hi * (one + (lo / hi) * (lo / hi)).sqrt() + }; + return (R::ZERO, max); + } + if ga < fhmx { + let as_ = one + fhmn / fhmx; + let at = (fhmx - fhmn) / fhmx; + let au = (ga / fhmx) * (ga / fhmx); + let c = two / ((as_ * as_ + au).sqrt() + (at * at + au).sqrt()); + (fhmn * c, fhmx / c) + } else { + let au = fhmx / ga; + if au == R::ZERO { + ((fhmn * fhmx) / ga, ga) + } else { + let as_ = one + fhmn / fhmx; + let at = (fhmx - fhmn) / fhmx; + let c = one + / ((one + (as_ * au) * (as_ * au)).sqrt() + (one + (at * au) * (at * au)).sqrt()); + let min = (fhmn * c) * au; + (min + min, ga / (c + c)) + } + } +} + +/// The state of dqds that LAPACK passes between `lasq2`, `lasq3` and `lasq4`. +struct State { + /// The first row of the unreduced block. + i0: usize, + /// Its last row. + n0: usize, + /// Ping (0) or pong (1) half of the array; 2 after a flip, which skips the tests of + /// deflation once. + pp: usize, + /// The smallest `d` of the last transform, or minus the shift to take. + dmin: R, + /// The accumulated shift. + sigma: R, + /// The lower part of `sigma`. + desig: R, + /// The largest `q`. + qmax: R, + /// The type of the last shift. + ttype: i32, + /// `dmin` without the last element. + dmin1: R, + /// `dmin` without the last two. + dmin2: R, + /// The last `d`. + dn: R, + /// The `d` before. + dn1: R, + /// The `d` before that. + dn2: R, + /// The factor of the shift of type 6. + g: R, + /// The shift. + tau: R, + /// The transforms done. + iter: usize, +} + +/// The eigenvalues of the symmetric positive definite tridiagonal matrix whose `L·D·Lᵀ` +/// factorization `z[1..2n]` holds as `q₁, e₁, q₂, …`, as LAPACK's `lasq2`: into `z[1..=n]`, +/// descending. `z` has `4n + 1` elements. +fn lasq2(n: usize, z: &mut [R]) -> bool { + let (zero, half) = (R::ZERO, R::HALF); + let four = R::TWO + R::TWO; + let eps = R::EPSILON * half; + let safmin = R::MIN_POSITIVE; + let tol = eps * R::from_f64(100.0); + let tol2 = tol * tol; + let cbias = R::from_f64(1.5); + // Sums of the q's and e's, and diagonality. + let mut d = zero; + let mut e = zero; + let mut k = 1; + while k <= 2 * (n - 1) { + if !(z[k] >= zero && z[k + 1] >= zero) { + return false; + } + d += z[k]; + e += z[k + 1]; + k += 2; + } + if z[2 * n - 1] < zero || z[2 * n - 1].is_nan() { + return false; + } + d += z[2 * n - 1]; + if e == zero { + for k in 2..=n { + z[k] = z[2 * k - 1]; + } + z[1..=n].sort_by(|a, b| b.partial_cmp(a).unwrap_or(core::cmp::Ordering::Equal)); + return true; + } + if d + e == zero { + z[1..=n].fill(zero); + return true; + } + // `z = (q₁, qq₁, e₁, ee₁, q₂, …)`. + let mut k = 2 * n; + while k >= 2 { + z[2 * k] = zero; + z[2 * k - 1] = z[k]; + z[2 * k - 2] = zero; + z[2 * k - 3] = z[k - 1]; + k -= 2; + } + let mut s = State { + i0: 1, + n0: n, + pp: 0, + dmin: zero, + sigma: zero, + desig: zero, + qmax: zero, + ttype: 0, + dmin1: zero, + dmin2: zero, + dn: zero, + dn1: zero, + dn2: zero, + g: zero, + tau: zero, + iter: 2, + }; + // Reversed when the end is much larger, so that the small values come out at the bottom. + if cbias * z[4 * s.i0 - 3] < z[4 * s.n0 - 3] { + let ipn4 = 4 * (s.i0 + s.n0); + let mut i4 = 4 * s.i0; + while i4 <= 2 * (s.i0 + s.n0 - 1) { + z.swap(i4 - 3, ipn4 - i4 - 3); + z.swap(i4 - 1, ipn4 - i4 - 5); + i4 += 4; + } + } + // Two dqd transforms with Li's test of splitting. + let mut pp = 0; + for _ in 0..2 { + let mut d = z[4 * s.n0 + pp - 3]; + let mut i4 = 4 * (s.n0 - 1) + pp; + while i4 >= 4 * s.i0 + pp { + if z[i4 - 1] <= tol2 * d { + z[i4 - 1] = -zero; + d = z[i4 - 3]; + } else { + d = z[i4 - 3] * (d / (d + z[i4 - 1])); + } + i4 -= 4; + } + let mut d = z[4 * s.i0 + pp - 3]; + let mut i4 = 4 * s.i0 + pp; + while i4 <= 4 * (s.n0 - 1) + pp { + let j = i4 - 2 * pp; + z[j - 2] = d + z[i4 - 1]; + if z[i4 - 1] <= tol2 * d { + z[i4 - 1] = -zero; + z[j - 2] = d; + z[j] = zero; + d = z[i4 + 1]; + } else if safmin * z[i4 + 1] < z[j - 2] && safmin * z[j - 2] < z[i4 + 1] { + let t = z[i4 + 1] / z[j - 2]; + z[j] = z[i4 - 1] * t; + d *= t; + } else { + z[j] = z[i4 + 1] * (z[i4 - 1] / z[j - 2]); + d = z[i4 + 1] * (d / z[j - 2]); + } + i4 += 4; + } + z[4 * s.n0 - pp - 2] = d; + let mut qmax = z[4 * s.i0 - pp - 2]; + let mut i4 = 4 * s.i0 - pp + 2; + while i4 <= 4 * s.n0 - pp - 2 { + qmax = qmax.max(z[i4]); + i4 += 4; + } + s.qmax = qmax; + pp = 1 - pp; + } + s.iter = 2; + for _ in 0..=n { + if s.n0 < 1 { + break; + } + // `e(n0)` holds minus the shift of the block that split off below. + s.desig = zero; + s.sigma = if s.n0 == n { zero } else { -z[4 * s.n0 - 1] }; + if s.sigma < zero { + return false; + } + // The last unreduced block, its largest q and a Gershgorin bound. + let mut emax = zero; + let mut qmin = z[4 * s.n0 - 3]; + s.qmax = qmin; + let mut i4 = 4 * s.n0; + let mut found = false; + while i4 >= 8 { + if z[i4 - 5] <= zero { + found = true; + break; + } + if qmin >= four * emax { + qmin = qmin.min(z[i4 - 3]); + emax = emax.max(z[i4 - 5]); + } + s.qmax = s.qmax.max(z[i4 - 7] + z[i4 - 5]); + i4 -= 4; + } + if !found { + i4 = 4; + } + s.i0 = i4 / 4; + s.pp = 0; + if s.n0 - s.i0 > 1 { + let mut dee = z[4 * s.i0 - 3]; + let mut deemin = dee; + let mut kmin = s.i0; + let mut i4 = 4 * s.i0 + 1; + while i4 <= 4 * s.n0 - 3 { + dee = z[i4] * (dee / (dee + z[i4 - 2])); + if dee <= deemin { + deemin = dee; + kmin = i4.div_ceil(4); + } + i4 += 4; + } + if (kmin - s.i0) * 2 < s.n0 - kmin && deemin <= half * z[4 * s.n0 - 3] { + let ipn4 = 4 * (s.i0 + s.n0); + s.pp = 2; + let mut i4 = 4 * s.i0; + while i4 <= 2 * (s.i0 + s.n0 - 1) { + z.swap(i4 - 3, ipn4 - i4 - 3); + z.swap(i4 - 2, ipn4 - i4 - 2); + z.swap(i4 - 1, ipn4 - i4 - 5); + z.swap(i4, ipn4 - i4 - 4); + i4 += 4; + } + } + } + // Minus the initial shift. + s.dmin = -zero.max(qmin - R::TWO * qmin.sqrt() * emax.sqrt()); + let nbig = 100 * (s.n0 - s.i0 + 1); + let mut done = false; + for _ in 0..nbig { + if s.i0 > s.n0 { + done = true; + break; + } + lasq3(&mut s, z, eps, tol2); + s.pp = 1 - s.pp; + // Splits, when an e is very small. + if s.pp == 0 + && s.n0 >= s.i0 + 3 + && (z[4 * s.n0] <= tol2 * s.qmax || z[4 * s.n0 - 1] <= tol2 * s.sigma) + { + let mut splt = s.i0 - 1; + s.qmax = z[4 * s.i0 - 3]; + let mut emin = z[4 * s.i0 - 1]; + let mut oldemn = z[4 * s.i0]; + let mut i4 = 4 * s.i0; + while i4 <= 4 * (s.n0 - 3) { + if z[i4] <= tol2 * z[i4 - 3] || z[i4 - 1] <= tol2 * s.sigma { + z[i4 - 1] = -s.sigma; + splt = i4 / 4; + s.qmax = zero; + emin = z[i4 + 3]; + oldemn = z[i4 + 4]; + } else { + s.qmax = s.qmax.max(z[i4 + 1]); + emin = emin.min(z[i4 - 1]); + oldemn = oldemn.min(z[i4]); + } + i4 += 4; + } + z[4 * s.n0 - 1] = emin; + z[4 * s.n0] = oldemn; + s.i0 = splt + 1; + } + } + if !done && s.i0 <= s.n0 { + return false; + } + } + if s.n0 >= 1 { + return false; + } + for k in 2..=n { + z[k] = z[4 * k - 3]; + } + z[1..=n].sort_by(|a, b| b.partial_cmp(a).unwrap_or(core::cmp::Ordering::Equal)); + true +} + +/// Deflates converged values from the bottom of the block, chooses a shift and takes a dqds +/// transform, as LAPACK's `lasq3`. +fn lasq3(s: &mut State, z: &mut [R], eps: R, tol2: R) { + let (zero, half, one, two) = (R::ZERO, R::HALF, R::ONE, R::TWO); + let cbias = R::from_f64(1.5); + let tol = eps * R::from_f64(100.0); + let n0in = s.n0; + // Deflation. + loop { + if s.n0 < s.i0 { + return; + } + let nn = 4 * s.n0 + s.pp; + let mut one_value = s.n0 == s.i0; + let mut two_values = s.n0 == s.i0 + 1; + if !one_value && !two_values { + // `e(n0 - 1)` negligible: one value. + if !(z[nn - 5] > tol2 * (s.sigma + z[nn - 3]) + && z[nn - 2 * s.pp - 4] > tol2 * z[nn - 7]) + { + one_value = true; + } else if !(z[nn - 9] > tol2 * s.sigma && z[nn - 2 * s.pp - 8] > tol2 * z[nn - 11]) { + // `e(n0 - 2)` negligible: two. + two_values = true; + } + } + if one_value { + z[4 * s.n0 - 3] = z[4 * s.n0 + s.pp - 3] + s.sigma; + s.n0 -= 1; + continue; + } + if two_values { + if z[nn - 3] > z[nn - 7] { + z.swap(nn - 3, nn - 7); + } + let t = half * ((z[nn - 7] - z[nn - 3]) + z[nn - 5]); + if z[nn - 5] > z[nn - 3] * tol2 && t != zero { + let mut u = z[nn - 3] * (z[nn - 5] / t); + if u <= t { + u = z[nn - 3] * (z[nn - 5] / (t * (one + (one + u / t).sqrt()))); + } else { + u = z[nn - 3] * (z[nn - 5] / (t + t.sqrt() * (t + u).sqrt())); + } + let t = z[nn - 7] + (u + z[nn - 5]); + z[nn - 3] *= z[nn - 7] / t; + z[nn - 7] = t; + } + z[4 * s.n0 - 7] = z[nn - 7] + s.sigma; + z[4 * s.n0 - 3] = z[nn - 3] + s.sigma; + s.n0 -= 2; + continue; + } + break; + } + if s.pp == 2 { + s.pp = 0; + } + // Reversed when warranted. + if (s.dmin <= zero || s.n0 < n0in) && cbias * z[4 * s.i0 + s.pp - 3] < z[4 * s.n0 + s.pp - 3] { + let ipn4 = 4 * (s.i0 + s.n0); + let mut j4 = 4 * s.i0; + while j4 <= 2 * (s.i0 + s.n0 - 1) { + z.swap(j4 - 3, ipn4 - j4 - 3); + z.swap(j4 - 2, ipn4 - j4 - 2); + z.swap(j4 - 1, ipn4 - j4 - 5); + z.swap(j4, ipn4 - j4 - 4); + j4 += 4; + } + let (i0, n0, pp) = (s.i0, s.n0, s.pp); + if n0 - i0 <= 4 { + z[4 * n0 + pp - 1] = z[4 * i0 + pp - 1]; + z[4 * n0 - pp] = z[4 * i0 - pp]; + } + s.dmin2 = s.dmin2.min(z[4 * n0 + pp - 1]); + z[4 * n0 + pp - 1] = z[4 * n0 + pp - 1] + .min(z[4 * i0 + pp - 1]) + .min(z[4 * i0 + pp + 3]); + z[4 * n0 - pp] = z[4 * n0 - pp].min(z[4 * i0 - pp]).min(z[4 * i0 - pp + 4]); + s.qmax = s.qmax.max(z[4 * i0 + pp - 3]).max(z[4 * i0 + pp + 1]); + s.dmin = -zero; + } + lasq4(s, z, n0in); + // dqds until `dmin` is not negative. + let mut safe = false; + let mut tries = 0; + loop { + lasq5(s, z, eps); + s.iter += 1; + tries += 1; + if s.dmin >= zero && s.dmin1 >= zero { + break; + } else if s.dmin < zero + && s.dmin1 > zero + && z[4 * (s.n0 - 1) - s.pp] < tol * (s.sigma + s.dn1) + && s.dn.abs() < tol * s.sigma + { + // Convergence hidden by a negative `dn`. + z[4 * (s.n0 - 1) - s.pp + 2] = zero; + s.dmin = zero; + break; + } else if s.dmin < zero && tries < 30 { + // The shift too large: another. + if s.ttype < -22 { + s.tau = zero; + } else if s.dmin1 > zero { + s.tau = (s.tau + s.dmin) * (one - two * eps); + s.ttype -= 11; + } else { + s.tau *= R::from_f64(0.25); + s.ttype -= 12; + } + } else if s.dmin.is_nan() { + if s.tau == zero { + safe = true; + break; + } + s.tau = zero; + } else { + // Possible underflow. + safe = true; + break; + } + } + if safe { + lasq6(s, z); + s.iter += 1; + s.tau = zero; + } + if s.tau < s.sigma { + s.desig += s.tau; + let t = s.sigma + s.desig; + s.desig -= t - s.sigma; + s.sigma = t; + } else { + let t = s.sigma + s.tau; + s.desig = s.sigma + (s.desig - (t - s.tau)); + s.sigma = t; + } +} + +/// Chooses the shift, as LAPACK's `lasq4`. +fn lasq4(s: &mut State, z: &[R], n0in: usize) { + let (zero, half, one, two) = (R::ZERO, R::HALF, R::ONE, R::TWO); + let cnst1 = R::from_f64(0.563); + let cnst2 = R::from_f64(1.01); + let cnst3 = R::from_f64(1.05); + let qurtr = R::from_f64(0.25); + let third = R::from_f64(0.333); + let hundrd = R::from_f64(100.0); + if s.dmin <= zero { + s.tau = -s.dmin; + s.ttype = -1; + return; + } + let (i0, n0, pp) = (s.i0, s.n0, s.pp); + let nn = 4 * n0 + pp; + let (dmin, dmin1, dmin2, dn, dn1, dn2) = (s.dmin, s.dmin1, s.dmin2, s.dn, s.dn1, s.dn2); + // The shift, set as the cases go so that an early return keeps the safe value. + let mut t; + if n0in == n0 { + if dmin == dn || dmin == dn1 { + let b1 = z[nn - 3].sqrt() * z[nn - 5].sqrt(); + let b2 = z[nn - 7].sqrt() * z[nn - 9].sqrt(); + let a2 = z[nn - 7] + z[nn - 5]; + if dmin == dn && dmin1 == dn1 { + // Cases 2 and 3. + let gap2 = dmin2 - a2 - dmin2 * qurtr; + let gap1 = if gap2 > zero && gap2 > b2 { + a2 - dn - (b2 / gap2) * b2 + } else { + a2 - dn - (b1 + b2) + }; + if gap1 > zero && gap1 > b1 { + t = (dn - (b1 / gap1) * b1).max(half * dmin); + s.ttype = -2; + } else { + t = zero; + if dn > b1 { + t = dn - b1; + } + if a2 > b1 + b2 { + t = t.min(a2 - (b1 + b2)); + } + t = t.max(third * dmin); + s.ttype = -3; + } + } else { + // Case 4. + s.ttype = -4; + t = qurtr * dmin; + let (gam, mut a2, b2, np); + if dmin == dn { + gam = dn; + a2 = zero; + if z[nn - 5] > z[nn - 7] { + s.tau = t; + return; + } + b2 = z[nn - 5] / z[nn - 7]; + np = nn as isize - 9; + } else { + let np0 = nn - 2 * pp; + gam = dn1; + if z[np0 - 4] > z[np0 - 2] { + s.tau = t; + return; + } + a2 = z[np0 - 4] / z[np0 - 2]; + if z[nn - 9] > z[nn - 11] { + s.tau = t; + return; + } + b2 = z[nn - 9] / z[nn - 11]; + np = nn as isize - 13; + } + // The contribution to the norm squared from the rows above. + a2 += b2; + let Some(a2) = norm_above(z, np, i0, pp, a2, b2) else { + s.tau = t; + return; + }; + let a2 = cnst3 * a2; + // The bound of the residual of the Rayleigh quotient. + if a2 < cnst1 { + t = gam * (one - a2.sqrt()) / (one + a2); + } + } + } else if dmin == dn2 { + // Case 5. + s.ttype = -5; + t = qurtr * dmin; + let np = nn - 2 * pp; + let b1 = z[np - 2]; + let mut b2 = z[np - 6]; + let gam = dn2; + if z[np - 8] > b2 || z[np - 4] > b1 { + s.tau = t; + return; + } + let mut a2 = (z[np - 8] / b2) * (one + z[np - 4] / b1); + if n0 - i0 > 2 { + b2 = z[nn - 13] / z[nn - 15]; + a2 += b2; + let Some(sum) = norm_above(z, nn as isize - 17, i0, pp, a2, b2) else { + s.tau = t; + return; + }; + a2 = cnst3 * sum; + } + if a2 < cnst1 { + t = gam * (one - a2.sqrt()) / (one + a2); + } + } else { + // Case 6: nothing to go by. + if s.ttype == -6 { + s.g += third * (one - s.g); + } else if s.ttype == -18 { + s.g = qurtr * third; + } else { + s.g = qurtr; + } + t = s.g * dmin; + s.ttype = -6; + } + } else if n0in == n0 + 1 { + // One value deflated: `dmin1` and `dn1` for `dmin` and `dn`. + if dmin1 == dn1 && dmin2 == dn2 { + // Cases 7 and 8. + s.ttype = -7; + t = third * dmin1; + if z[nn - 5] > z[nn - 7] { + s.tau = t; + return; + } + let mut b1 = z[nn - 5] / z[nn - 7]; + let mut b2 = b1; + if b2 != zero { + let mut i4 = 4 * n0 - 9 + pp; + while i4 >= 4 * i0 - 1 + pp { + let a2 = b1; + if z[i4] > z[i4 - 2] { + s.tau = t; + return; + } + b1 *= z[i4] / z[i4 - 2]; + b2 += b1; + if hundrd * b1.max(a2) < b2 { + break; + } + if i4 < 4 { + break; + } + i4 -= 4; + } + } + let b2 = (cnst3 * b2).sqrt(); + let a2 = dmin1 / (one + b2 * b2); + let gap2 = half * dmin2 - a2; + if gap2 > zero && gap2 > b2 * a2 { + t = t.max(a2 * (one - cnst2 * a2 * (b2 / gap2) * b2)); + } else { + t = t.max(a2 * (one - cnst2 * b2)); + s.ttype = -8; + } + } else { + // Case 9. + t = qurtr * dmin1; + if dmin1 == dn1 { + t = half * dmin1; + } + s.ttype = -9; + } + } else if n0in == n0 + 2 { + // Two values deflated: `dmin2` and `dn2` for `dmin` and `dn`. + if dmin2 == dn2 && two * z[nn - 5] < z[nn - 7] { + // Cases 10 and 11. + s.ttype = -10; + t = third * dmin2; + if z[nn - 5] > z[nn - 7] { + s.tau = t; + return; + } + let mut b1 = z[nn - 5] / z[nn - 7]; + let mut b2 = b1; + if b2 != zero { + let mut i4 = 4 * n0 - 9 + pp; + while i4 >= 4 * i0 - 1 + pp { + if z[i4] > z[i4 - 2] { + s.tau = t; + return; + } + b1 *= z[i4] / z[i4 - 2]; + b2 += b1; + if hundrd * b1 < b2 { + break; + } + if i4 < 4 { + break; + } + i4 -= 4; + } + } + let b2 = (cnst3 * b2).sqrt(); + let a2 = dmin2 / (one + b2 * b2); + let gap2 = z[nn - 7] + z[nn - 9] - z[nn - 11].sqrt() * z[nn - 9].sqrt() - a2; + if gap2 > zero && gap2 > b2 * a2 { + t = t.max(a2 * (one - cnst2 * a2 * (b2 / gap2) * b2)); + } else { + t = t.max(a2 * (one - cnst2 * b2)); + } + } else { + t = qurtr * dmin2; + s.ttype = -11; + } + } else { + // Case 12: more than two deflated, nothing to go by. + t = zero; + s.ttype = -12; + } + s.tau = t; +} + +/// The sum `a2` of the ratios `b2·Π z(i4)/z(i4 - 2)` from `i4 = np` up the block by fours, as +/// the loops of LAPACK's `lasq4` take it, until the terms are small; `None` when a ratio is +/// above one. +fn norm_above(z: &[R], np: isize, i0: usize, pp: usize, a2: R, b2: R) -> Option { + let (mut a2, mut b2) = (a2, b2); + let lo = (4 * i0 + pp) as isize - 1; + let mut i4 = np; + while i4 >= lo && b2 != R::ZERO { + let i = i4 as usize; + let b1 = b2; + if z[i] > z[i - 2] { + return None; + } + b2 *= z[i] / z[i - 2]; + a2 += b2; + if R::from_f64(100.0) * b2.max(b1) < a2 || R::from_f64(0.563) < a2 { + break; + } + i4 -= 4; + } + Some(a2) +} + +/// One dqds transform with the shift `tau`, from the ping half to the pong or back, as LAPACK's +/// `lasq5`. +fn lasq5(s: &mut State, z: &mut [R], eps: R) { + let (i0, n0, pp) = (s.i0, s.n0, s.pp); + if n0 < i0 + 2 { + return; + } + let dthresh = eps * (s.sigma + s.tau); + if s.tau < dthresh * R::HALF { + s.tau = R::ZERO; + } + let tau = s.tau; + // With no shift, tiny `d`s are set to zero. + let flush = tau == R::ZERO; + let j4 = 4 * i0 + pp - 3; + let mut emin = z[j4 + 4]; + let mut d = z[j4] - tau; + let mut dmin = d; + s.dmin1 = -z[j4]; + let mut j4 = 4 * i0; + if pp == 0 { + while j4 <= 4 * (n0 - 3) { + z[j4 - 2] = d + z[j4 - 1]; + let t = z[j4 + 1] / z[j4 - 2]; + d = d * t - tau; + if flush && d < dthresh { + d = R::ZERO; + } + dmin = dmin.min(d); + z[j4] = z[j4 - 1] * t; + emin = emin.min(z[j4]); + j4 += 4; + } + } else { + while j4 <= 4 * (n0 - 3) { + z[j4 - 3] = d + z[j4]; + let t = z[j4 + 2] / z[j4 - 3]; + d = d * t - tau; + if flush && d < dthresh { + d = R::ZERO; + } + dmin = dmin.min(d); + z[j4 - 1] = z[j4] * t; + emin = emin.min(z[j4 - 1]); + j4 += 4; + } + } + // The last two steps. + let dnm2 = d; + s.dmin2 = dmin; + let mut j4 = 4 * (n0 - 2) - pp; + let mut j4p2 = j4 + 2 * pp - 1; + z[j4 - 2] = dnm2 + z[j4p2]; + z[j4] = z[j4p2 + 2] * (z[j4p2] / z[j4 - 2]); + let dnm1 = z[j4p2 + 2] * (dnm2 / z[j4 - 2]) - tau; + dmin = dmin.min(dnm1); + s.dmin1 = dmin; + j4 += 4; + j4p2 = j4 + 2 * pp - 1; + z[j4 - 2] = dnm1 + z[j4p2]; + z[j4] = z[j4p2 + 2] * (z[j4p2] / z[j4 - 2]); + let dn = z[j4p2 + 2] * (dnm1 / z[j4 - 2]) - tau; + dmin = dmin.min(dn); + z[j4 + 2] = dn; + z[4 * n0 - pp] = emin; + s.dmin = dmin; + s.dn = dn; + s.dn1 = dnm1; + s.dn2 = dnm2; +} + +/// One dqd transform without a shift, guarded against underflow, as LAPACK's `lasq6`. +fn lasq6(s: &mut State, z: &mut [R]) { + let (i0, n0, pp) = (s.i0, s.n0, s.pp); + if n0 < i0 + 2 { + return; + } + let zero = R::ZERO; + let safmin = R::MIN_POSITIVE; + let j4 = 4 * i0 + pp - 3; + let mut emin = z[j4 + 4]; + let mut d = z[j4]; + let mut dmin = d; + // One step: `q̂ = d + e`, `ê = e·q_next/q̂`, `d = d·q_next/q̂`, at the offsets of the half. + let step = |z: &mut [R], d: R, qh: usize, e: usize, qn: usize, eh: usize| -> (R, bool) { + z[qh] = d + z[e]; + if z[qh] == zero { + z[eh] = zero; + (z[qn], true) + } else if safmin * z[qn] < z[qh] && safmin * z[qh] < z[qn] { + let t = z[qn] / z[qh]; + z[eh] = z[e] * t; + (d * t, false) + } else { + z[eh] = z[qn] * (z[e] / z[qh]); + (z[qn] * (d / z[qh]), false) + } + }; + let mut j4 = 4 * i0; + while j4 <= 4 * (n0 - 3) { + let (nd, reset) = if pp == 0 { + step(z, d, j4 - 2, j4 - 1, j4 + 1, j4) + } else { + step(z, d, j4 - 3, j4, j4 + 2, j4 - 1) + }; + d = nd; + if reset { + dmin = d; + emin = zero; + } + dmin = dmin.min(d); + emin = emin.min(if pp == 0 { z[j4] } else { z[j4 - 1] }); + j4 += 4; + } + let dnm2 = d; + s.dmin2 = dmin; + let mut j4 = 4 * (n0 - 2) - pp; + let mut j4p2 = j4 + 2 * pp - 1; + let (dnm1, reset) = step(z, dnm2, j4 - 2, j4p2, j4p2 + 2, j4); + if reset { + dmin = dnm1; + emin = zero; + } + dmin = dmin.min(dnm1); + s.dmin1 = dmin; + j4 += 4; + j4p2 = j4 + 2 * pp - 1; + let (dn, reset) = step(z, dnm1, j4 - 2, j4p2, j4p2 + 2, j4); + if reset { + dmin = dn; + emin = zero; + } + dmin = dmin.min(dn); + z[j4 + 2] = dn; + z[4 * n0 - pp] = emin; + s.dmin = dmin; + s.dn = dn; + s.dn1 = dnm1; + s.dn2 = dnm2; +} diff --git a/lib/src/math/matrix/svd/qr.rs b/lib/src/math/matrix/svd/qr.rs new file mode 100644 index 0000000..b5af8e4 --- /dev/null +++ b/lib/src/math/matrix/svd/qr.rs @@ -0,0 +1,316 @@ +//! Golub and Kahan's implicit QR algorithm on a real upper bidiagonal matrix, as LINPACK's +//! `svdc`: the small problems of divide and conquer, and the singular values alone. + +use crate::math::complex::Float; + +/// Columns of a column-major real block that the rotations act on. +pub(super) struct Columns<'a, R> { + /// The elements, column `j` from `j · ld`. + pub data: &'a mut [R], + /// The rows rotated. + pub rows: usize, + /// The leading dimension. + pub ld: usize, +} + +/// The singular vectors [`bidiagonal_qr`] turns: what it does to their columns. +pub(super) trait Vectors { + /// `(x, y) = (c·x + s·y, c·y - s·x)` on columns `i` and `j`. + fn rotate(&mut self, i: usize, j: usize, c: R, s: R); + /// Negates column `j`. + fn negate(&mut self, j: usize); + /// Swaps columns `i < j`. + fn swap(&mut self, i: usize, j: usize); +} + +impl Columns<'_, R> { + /// `(x, y) = (c·x + s·y, c·y - s·x)` on columns `i` and `j`. + #[inline] + pub fn rotate(&mut self, i: usize, j: usize, c: R, s: R) { + let (rows, ld) = (self.rows, self.ld); + let (lo, hi) = (i.min(j), i.max(j)); + let (left, right) = self.data.split_at_mut(hi * ld); + let (p, q) = (&mut left[lo * ld..lo * ld + rows], &mut right[..rows]); + let (x, y) = if i < j { (p, q) } else { (q, p) }; + rotate_real(x, y, c, s); + } +} + +impl Vectors for Columns<'_, R> { + fn rotate(&mut self, i: usize, j: usize, c: R, s: R) { + Columns::rotate(self, i, j, c, s); + } + + fn negate(&mut self, j: usize) { + for x in &mut self.data[j * self.ld..j * self.ld + self.rows] { + *x = -*x; + } + } + + fn swap(&mut self, i: usize, j: usize) { + let (rows, ld) = (self.rows, self.ld); + let (left, right) = self.data.split_at_mut(j * ld); + left[i * ld..i * ld + rows].swap_with_slice(&mut right[..rows]); + } +} + +/// `(x, y) = (c·x + s·y, c·y - s·x)`. Kept out of line: inlined into the QR sweep, the compiler +/// moves the sweep's next divisions below this loop, and each step of the sweep then waits for +/// the rotation of the vectors instead of running beside it. +#[inline(never)] +fn rotate_real(x: &mut [R], y: &mut [R], c: R, s: R) { + for (x, y) in x.iter_mut().zip(y.iter_mut()) { + let (a, b) = (*x, *y); + *x = fma(c, a, s * b); + *y = fma(c, b, -(s * a)); + } +} + +/// Whether the target fuses multiply-adds in hardware; elsewhere `mul_add` is a library call. +const FMA: bool = cfg!(any(target_arch = "aarch64", target_feature = "fma")); + +/// `a·b + c`, rounded once where the target fuses multiply-adds. +#[inline(always)] +pub(super) fn fma(a: R, b: R, c: R) -> R { + if FMA { a.mul_add(b, c) } else { a * b + c } +} + +/// The plane rotation `(c, s)` with `c·a + s·b = r` and `c·b - s·a = 0`, as BLAS's `rotg`, and +/// `r`: its length from the sum of the squares where neither can have overflowed or +/// underflowed to matter, as it nearly always is, and by `hypot` elsewhere. +#[inline] +pub(super) fn rotg(a: R, b: R) -> (R, R, R) { + let roe = if a.abs() > b.abs() { a } else { b }; + let ss = a * a + b * b; + let r = if ss >= R::MIN_POSITIVE / R::EPSILON && ss <= R::MAX { + ss.sqrt() + } else { + a.hypot(b) + }; + if r == R::ZERO { + return (R::ZERO, R::ONE, R::ZERO); + } + let r = r.copysign(roe); + (r, a / r, b / r) +} + +/// A rotation of the bulge chase and what it makes of the elements it combines. +struct Chase { + /// The length `r` of `(f, g)`, signed as `rotg`'s. + r: R, + /// The cosine. + cs: R, + /// The sine. + sn: R, + /// `cs·a + sn·b`, the next `f`. + f: R, + /// `cs·b - sn·a`. + b: R, + /// `sn·x`, the next `g`. + g: R, + /// `cs·x`. + x: R, +} + +/// The rotation `rotg(f, g)` applied to `(a, b)` and to `(0, x)`. Each step of a sweep waits on +/// the one before through this, so its products are formed as quotients by the length instead: +/// `(f·a + g·b)/r` needs only `r` from the square root, where `cs·a + sn·b` needs `cs` from a +/// division after it; `rotg` and the products as they are when the length is near an end of +/// the range. +#[inline(always)] +fn chase(f: R, g: R, a: R, b: R, x: R) -> Chase { + let ss = fma(f, f, g * g); + if ss >= R::MIN_POSITIVE / R::EPSILON && ss <= R::MAX { + let roe = if f.abs() > g.abs() { f } else { g }; + let r = ss.sqrt().copysign(roe); + Chase { + r, + f: fma(f, a, g * b) / r, + g: g * x / r, + cs: f / r, + sn: g / r, + b: fma(f, b, -(g * a)) / r, + x: f * x / r, + } + } else { + let (r, cs, sn) = rotg(f, g); + Chase { + r, + cs, + sn, + f: cs * a + sn * b, + b: cs * b - sn * a, + g: sn * x, + x: cs * x, + } + } +} + +/// The singular values of the real upper bidiagonal matrix with the diagonal `s` and the +/// superdiagonal `e` (with a zero appended), by Golub and Kahan's implicit QR algorithm as +/// LINPACK's `svdc`: `s` turned into the singular values, descending, and the rotations applied +/// to the columns of `u` and `v` when given. Values it doesn't converge to are NaNs, and then it +/// returns `false`. +pub(super) fn bidiagonal_qr, V: Vectors>( + s: &mut [R], + e: &mut [R], + mut u: Option<&mut U>, + mut v: Option<&mut V>, +) -> bool { + let p = s.len(); + let zero = R::ZERO; + // An element is negligible within a few units in the last place of its neighbours. LINPACK + // tests `test + |e| == test`, which a sweep's rounding can keep failing: for `f32` values + // near ±1 without fused multiply-adds, every sweep left `e = 2⁻²⁴`, half a unit of `test`, + // which rounded up, until the iterations ran out. + let tol = R::from_f64(4.0) * R::EPSILON; + // 1-based indices below, as in LINPACK: `s(k)` is `s[k - 1]`. + let mut m = p; + let mut iter = 0; + while m > 0 { + if iter >= 75 { + for x in &mut s[..m] { + *x = R::NAN; + } + return false; + } + // `l`: the last negligible superdiagonal element above `m`, 0 for none. + let mut l = m - 1; + while l > 0 { + let test = s[l - 1].abs() + s[l].abs(); + if e[l - 1].abs() <= tol * test { + e[l - 1] = zero; + break; + } + l -= 1; + } + let kase; + if l == m - 1 { + kase = 4; + } else { + let mut ls = m; + while ls > l { + let mut test = zero; + if ls != m { + test += e[ls - 1].abs(); + } + if ls != l + 1 { + test += e[ls - 2].abs(); + } + if s[ls - 1].abs() <= tol * test { + s[ls - 1] = zero; + break; + } + ls -= 1; + } + if ls == l { + kase = 3; + } else if ls == m { + kase = 1; + } else { + kase = 2; + l = ls; + } + } + l += 1; + match kase { + 1 => { + // `s(m)` is negligible: chase `e(m - 1)` up and out. + let mut f = e[m - 2]; + e[m - 2] = zero; + for k in (l..m).rev() { + let (r, cs, sn) = rotg(s[k - 1], f); + s[k - 1] = r; + if k != l { + f = -sn * e[k - 2]; + e[k - 2] = cs * e[k - 2]; + } + if let Some(v) = v.as_deref_mut() { + v.rotate(k - 1, m - 1, cs, sn); + } + } + } + 2 => { + // `s(l - 1)` is negligible: split there, chasing `e(l - 1)` down. + let mut f = e[l - 2]; + e[l - 2] = zero; + for k in l..=m { + let (r, cs, sn) = rotg(s[k - 1], f); + s[k - 1] = r; + f = -sn * e[k - 1]; + e[k - 1] = cs * e[k - 1]; + if let Some(u) = u.as_deref_mut() { + u.rotate(k - 1, l - 2, cs, sn); + } + } + } + 3 => { + // A QR step with the shift from the trailing 2 × 2 block of `Bᵀ·B`. + let scale = s[m - 1] + .abs() + .max(s[m - 2].abs()) + .max(e[m - 2].abs()) + .max(s[l - 1].abs()) + .max(e[l - 1].abs()); + let sm = s[m - 1] / scale; + let smm1 = s[m - 2] / scale; + let emm1 = e[m - 2] / scale; + let sl = s[l - 1] / scale; + let el = e[l - 1] / scale; + let b = ((smm1 + sm) * (smm1 - sm) + emm1 * emm1) * R::HALF; + let c = (sm * emm1) * (sm * emm1); + let mut shift = zero; + if b != zero || c != zero { + shift = (b * b + c).sqrt(); + if b < zero { + shift = -shift; + } + shift = c / (b + shift); + } + let mut f = (sl + sm) * (sl - sm) + shift; + let mut g = sl * el; + for k in l..m { + let c = chase(f, g, s[k - 1], e[k - 1], s[k]); + if k != l { + e[k - 2] = c.r; + } + (f, e[k - 1], g, s[k]) = (c.f, c.b, c.g, c.x); + if let Some(v) = v.as_deref_mut() { + v.rotate(k - 1, k, c.cs, c.sn); + } + let c = chase(f, g, e[k - 1], s[k], e[k]); + s[k - 1] = c.r; + (f, s[k], g, e[k]) = (c.f, c.b, c.g, c.x); + if let Some(u) = u.as_deref_mut() { + u.rotate(k - 1, k, c.cs, c.sn); + } + } + e[m - 2] = f; + iter += 1; + } + _ => { + // `s(l)` has converged: made not negative, and moved down into order. + if s[l - 1] < zero { + s[l - 1] = -s[l - 1]; + if let Some(v) = v.as_deref_mut() { + v.negate(l - 1); + } + } + let mut l = l; + while l < p && s[l - 1] < s[l] { + s.swap(l - 1, l); + if let Some(v) = v.as_deref_mut() { + v.swap(l - 1, l); + } + if let Some(u) = u.as_deref_mut() { + u.swap(l - 1, l); + } + l += 1; + } + iter = 0; + m -= 1; + } + } + } + true +} diff --git a/lib/src/math/matrix/svd/secular.rs b/lib/src/math/matrix/svd/secular.rs new file mode 100644 index 0000000..bdb8189 --- /dev/null +++ b/lib/src/math/matrix/svd/secular.rs @@ -0,0 +1,255 @@ +//! The singular value decomposition of a broken arrow matrix, the merge step of divide and +//! conquer: the roots of the secular equation, as LAPACK's `lasd4`, and the singular vectors +//! from Gu and Eisenstat's recomputed `z`, as `lasd3`. + +use crate::math::complex::Float; + +/// The most iterations of one root before it is taken as it is. +const MAXIT: usize = 64; + +/// The singular values and vectors of the `k × k` matrix `M` with `z` in its first row and +/// `d₁, …, d_{k-1}` on the rest of its diagonal, `0 = d₀ < d₁ < … < d_{k-1}`, the elements of `z` +/// not zero: `σ` ascending into `sigma`, and the left and right singular vectors as the columns +/// of the `k × k` blocks `u` and `v`, element `j` of each in row `pos[j]`. +pub(super) fn arrow( + d: &[R], + z: &[R], + pos: &[usize], + sigma: &mut [R], + u: &mut [R], + v: &mut [R], +) { + let k = d.len(); + if k == 1 { + sigma[0] = z[0].abs(); + u[pos[0]] = if z[0] < R::ZERO { -R::ONE } else { R::ONE }; + v[pos[0]] = R::ONE; + return; + } + // The roots for `z` of norm 1 and `ρ = ‖z‖²`. + let rho = z.iter().map(|&x| x * x).sum::().sqrt(); + let zn: Vec = z.iter().map(|&x| x / rho).collect(); + let rhoinv = R::ONE / (rho * rho); + // `prod[j + i·k] = d_j² - σ_i²`, as `(d_j - σ_i)·(d_j + σ_i)`, each part to high relative + // accuracy. + let mut prod = vec![R::ZERO; k * k]; + for (i, (sigma, prod)) in sigma.iter_mut().zip(prod.chunks_exact_mut(k)).enumerate() { + *sigma = root(i, d, &zn, rhoinv, prod); + } + // `ẑ`, which makes the computed values the exact ones of `D² + ẑ·ẑᵀ`, by Löwner's formula: + // `ẑ_j² = (d_j² - σ_{k-1}²)·Π_{i(); + let (vcc, vcr) = vc[1..].as_chunks::<4>(); + let (mut su, mut sv) = ([R::ZERO; 4], [R::ZERO; 4]); + for (v, d) in vcc.iter().zip(dc) { + for l in 0..4 { + let u = d[l] * v[l]; + su[l] += u * u; + sv[l] += v[l] * v[l]; + } + } + let mut nu = R::ONE + (su[0] + su[1]) + (su[2] + su[3]); + let mut nv = vc[0] * vc[0] + (sv[0] + sv[1]) + (sv[2] + sv[3]); + for (v, &d) in vcr.iter().zip(dr) { + nu += (d * *v) * (d * *v); + nv += *v * *v; + } + let (nu, nv) = (R::ONE / nu.sqrt(), R::ONE / nv.sqrt()); + let (ucol, vcol) = (&mut u[i * k..(i + 1) * k], &mut v[i * k..(i + 1) * k]); + ucol[pos[0]] = -nu; + vcol[pos[0]] = vc[0] * nv; + for j in 1..k { + ucol[pos[j]] = d[j] * vc[j] * nu; + vcol[pos[j]] = vc[j] * nv; + } + } +} + +/// Root `i` of `f(σ) = 1/ρ + Σ z_j²/((d_j - σ)(d_j + σ))`, `‖z‖ = 1`, in `(d_i, d_{i+1})`, or +/// above `d_{k-1}` for the last, by the middle way of Li's rational interpolation of the two +/// nearest poles, safeguarded by bisection; `prod[j]` set to `(d_j - σ)(d_j + σ)`. The root is +/// held as `σ = d_K + μ` from the nearer pole `d_K`, so that the differences from the poles keep +/// their relative accuracy. +fn root(i: usize, d: &[R], z: &[R], rhoinv: R, prod: &mut [R]) -> R { + let k = d.len(); + let (two, four) = (R::TWO, R::TWO + R::TWO); + let eps = R::EPSILON * R::HALF; + let last = i + 1 == k; + // The pole `K` the root is held from, and `τ = σ² - d_K²` with a bracket `(lo, hi]`. + let (origin, mut tau, mut lo, mut hi); + if !last { + let (di, dn) = (d[i], d[i + 1]); + let delsq = (dn - di) * (dn + di); + let half = delsq * R::HALF; + // `f` halfway between the poles in `σ²`, with the terms of the other poles in `c`. + let mut c = rhoinv; + for j in (0..k).filter(|&j| j != i && j != i + 1) { + c += z[j] * z[j] / ((d[j] - di) * (d[j] + di) - half); + } + let (zi, zn) = (z[i] * z[i], z[i + 1] * z[i + 1]); + let w = c - zi / half + zn / half; + if w >= R::ZERO { + // In the lower half: from `d_i`. + origin = i; + (lo, hi) = (R::ZERO, half); + let a = c * delsq + zi + zn; + let b = zi * delsq; + let disc = (a * a - four * b * c).abs().sqrt(); + tau = if a > R::ZERO { + two * b / (a + disc) + } else { + (a - disc) / (two * c) + }; + } else { + // In the upper half: from `d_{i+1}`, `τ` negative. + origin = i + 1; + (lo, hi) = (-half, R::ZERO); + let a = c * delsq - zi - zn; + let b = zn * delsq; + let disc = (a * a + four * b * c).abs().sqrt(); + tau = if a < R::ZERO { + two * b / (a - disc) + } else { + -(a + disc) / (two * c) + }; + } + } else { + // Above the last pole, at most `ρ` above it in `σ²`. + let (dm, dn) = (d[k - 2], d[k - 1]); + let rho = R::ONE / rhoinv; + let half = rho * R::HALF; + let mut c = rhoinv; + for j in 0..k - 2 { + c += z[j] * z[j] / ((d[j] - dn) * (d[j] + dn) - half); + } + let (zm, zn) = (z[k - 2] * z[k - 2], z[k - 1] * z[k - 1]); + let delsq = (dn - dm) * (dn + dm); + let w = c + zm / (-delsq - half) - zn / half; + let a = -c * delsq + zm + zn; + let b = zn * delsq; + let disc = (a * a + four * b * c).abs().sqrt(); + let guess = if a < R::ZERO { + two * b / (disc - a) + } else { + (a + disc) / (two * c) + }; + origin = k - 1; + if w <= R::ZERO { + (lo, hi) = (half, rho); + } else { + (lo, hi) = (R::ZERO, half); + } + tau = guess; + } + if !(tau > lo && tau <= hi) { + tau = (lo + hi) * R::HALF; + } + let dk = d[origin]; + let mut mu = tau / (dk + (dk * dk + tau).sqrt()); + // The poles below and above the root that model the two sums. + let (pa, pb) = if last { (k - 2, k - 1) } else { (i, i + 1) }; + let split = if last { k - 1 } else { i + 1 }; + let mut iter = 0; + loop { + // `f` and its derivative, split into the terms of the poles up to `pa` and the rest. + let (psi, dpsi, ea) = terms(&d[..split], &z[..split], dk, mu, &mut prod[..split]); + let (phi, dphi, eb) = terms(&d[split..], &z[split..], dk, mu, &mut prod[split..]); + let w = rhoinv + psi + phi; + let dw = dpsi + dphi; + let erretm = R::from_f64(8.0) * (ea + eb) + two * rhoinv + tau.abs() * dw; + if w.abs() <= eps * erretm || iter >= MAXIT { + break; + } + if w < R::ZERO { + lo = lo.max(tau); + } else { + hi = hi.min(tau); + } + // The root of `c + s/(a - η) + S/(b - η)`, which matches `ψ` and `φ` and their + // derivatives with the poles `a = d_pa² - σ²` and `b = d_pb² - σ²`. + let (a, b) = (prod[pa], prod[pb]); + let cc = w - a * dpsi - b * dphi; + let aa = (a + b) * w - a * b * dw; + let bb = a * b * w; + let disc = (aa * aa - four * bb * cc).abs().sqrt(); + let mut eta = if last { + if aa >= R::ZERO { + (aa + disc) / (two * cc) + } else { + two * bb / (aa - disc) + } + } else if cc == R::ZERO { + bb / aa + } else if aa <= R::ZERO { + (aa - disc) / (two * cc) + } else { + two * bb / (aa + disc) + }; + // A step against the slope: Newton's instead. + if w * eta >= R::ZERO || eta.is_nan() { + eta = -w / dw; + } + let mut next = tau + eta; + if !(next > lo && next < hi) { + next = (lo + hi) * R::HALF; + } + if next == tau { + break; + } + tau = next; + mu = tau / (dk + (dk * dk + tau).sqrt()); + iter += 1; + } + dk + mu +} + +/// `Σ z_j²/p_j`, `Σ (z_j/p_j)²` and `Σ |z_j²/p_j|` for `p_j = (d_j - σ)(d_j + σ)`, `σ = d_K + μ`, +/// the `p_j` into `prod`; in four sums each, which the compiler keeps in vector registers. +#[inline] +fn terms(d: &[R], z: &[R], dk: R, mu: R, prod: &mut [R]) -> (R, R, R) { + let (dc, dr) = d.as_chunks::<4>(); + let (zc, zr) = z.as_chunks::<4>(); + let (pc, pr) = prod.as_chunks_mut::<4>(); + let (mut f, mut df, mut ab) = ([R::ZERO; 4], [R::ZERO; 4], [R::ZERO; 4]); + for ((d, z), p) in dc.iter().zip(zc).zip(pc) { + for l in 0..4 { + p[l] = ((d[l] - dk) - mu) * ((d[l] + dk) + mu); + let t = z[l] / p[l]; + f[l] += z[l] * t; + df[l] += t * t; + ab[l] += (z[l] * t).abs(); + } + } + let sum = |x: [R; 4]| (x[0] + x[1]) + (x[2] + x[3]); + let (mut f, mut df, mut ab) = (sum(f), sum(df), sum(ab)); + for ((&d, &z), p) in dr.iter().zip(zr).zip(pr) { + *p = ((d - dk) - mu) * ((d + dk) + mu); + let t = z / *p; + f += z * t; + df += t * t; + ab += (z * t).abs(); + } + (f, df, ab) +} diff --git a/lib/src/math/matrix/svd/small.rs b/lib/src/math/matrix/svd/small.rs new file mode 100644 index 0000000..ee356b9 --- /dev/null +++ b/lib/src/math/matrix/svd/small.rs @@ -0,0 +1,466 @@ +//! The singular value decomposition of small matrices, where the blocked path's fixed costs +//! outweigh its arithmetic: the reduction to bidiagonal one column and row at a time, `U` and `V` +//! formed explicitly from its reflections, `U` in their place, and the implicit QR algorithm's +//! rotations applied straight to them; the singular values alone by dqds. No allocation but the +//! results': the scratch is on the stack. + +use super::super::hermitian::small::{make_column, reflect_columns, reflect_columns2}; +use super::super::hermitian::{dotc, fma, fma_conj, lanes, lanes_of}; +use super::super::{Matrix, Scalar}; +use super::bidiagonal::reflector; +use super::{dqds, qr}; +use crate::math::complex::Float; + +/// The most columns any type has solved here. +const LARGEST: usize = 192; + +/// The numbers of columns whose scratch is the smallest of the sizes of it, then the middle one, +/// so that the smaller problems don't clear room for the largest. +const TIERS: [usize; 2] = [16, 48]; + +/// The most columns of the elements `T` solved here, as measured on an M4: past them, divide and +/// conquer and the reflections applied in blocks take less time. The QR algorithm's rotations +/// of real single precision vectors go in the most lanes, which moves their crossover furthest; +/// without the vectors, the reduction here keeps ahead of the blocked one longest for complex +/// elements, and least long for real single precision ones. +pub(super) fn largest(want_uv: bool) -> usize { + match (want_uv, T::IS_COMPLEX, size_of::()) { + (true, false, 4) => 112, + (true, true, 4) => 88, + (true, _, _) => 96, + (false, false, 4) => 96, + (false, false, _) => 112, + (false, true, _) => LARGEST, + } +} + +/// [`decompose`](super::decompose) of the `m × n` column-major `a`, `m ≥ n`, `n` up to +/// [`largest`] and `m` below `2.5n` (QR goes first from there), its elements finite and of a +/// moderate size. +#[allow(clippy::type_complexity)] +pub(super) fn decompose( + a: Vec, + m: usize, + n: usize, + want_uv: bool, +) -> (Vec, Option<(Matrix, Matrix)>) { + // Room for up to `2.5` times as many rows as columns. + if n <= TIERS[0] { + solve::(a, m, n, want_uv) + } else if n <= TIERS[1] { + solve::(a, m, n, want_uv) + } else { + solve::(a, m, n, want_uv) + } +} + +/// [`decompose`] with room for `N` columns and `M` rows. +#[allow(clippy::type_complexity)] +fn solve( + mut a: Vec, + m: usize, + n: usize, + want_uv: bool, +) -> (Vec, Option<(Matrix, Matrix)>) { + debug_assert!(m >= n && n <= N && (n == 0 || m < M)); + let zero = T::Real::ZERO; + let mut d = vec![zero; n]; + let (mut e, mut tauq, mut taup) = ([zero; N], [T::zero(); N], [T::zero(); N]); + bidiagonalize::(&mut a, m, n, &mut d, &mut e, &mut tauq, &mut taup); + let k = n.saturating_sub(1); + if !want_uv { + if !dqds::singular_values(&mut d, &e[..k]) { + qr::bidiagonal_qr::<_, qr::Columns<_>, qr::Columns<_>>(&mut d, &mut e[..n], None, None); + } + return (d, None); + } + let mut v = form_v::(&a, m, n, &taup[..k]); + form_u(&mut a, m, n, &tauq[..n]); + let mut cu = Cols { + data: &mut a, + rows: m, + ld: m, + }; + let mut cv = Cols { + data: &mut v, + rows: n, + ld: n, + }; + qr::bidiagonal_qr(&mut d, &mut e[..n], Some(&mut cu), Some(&mut cv)); + ( + d, + Some(( + Matrix::from_column_major(m, n, a), + Matrix::from_column_major(n, n, v), + )), + ) +} + +/// Reduces the `m × n` column-major `a`, `m ≥ n`, to the real upper bidiagonal `B = Q*·A·P`, +/// one column and row at a time, the reflections of LAPACK's `gebd2`: `Q = H₀·H₁·…·H_{n-1}`, the +/// vector of `H_j` +/// below the diagonal of column `j` and its factor in `tauq`; `P = G₀·G₁·…·G_{n-2}`, `G_j` +/// acting on the elements from `j + 1`, the rest of its vector after the leading one in row `j` +/// from column `j + 2` and its factor in `taup`. `G_j*` turns the conjugate of the row into +/// `(β, 0, …)`, so that the row times `G_j` is `(β, 0, …)`. +/// +/// Each right reflection is applied with the next left one, in one pass over the rest of the +/// matrix that also forms the product for the right reflection after: the rest is read twice a +/// step and written once, not four times and twice. With `x_k` the columns as held, `t` and +/// `f_k = τ_p·conj(w_k)` from the right reflection not yet applied, and `v` the left one's +/// vector, the products `v*·(x_k - f_k·t) = v*·x_k - f_k·(v*·t)` come from the columns as held, +/// and the row the next right reflection is made of from them alone. +fn bidiagonalize( + a: &mut [T], + m: usize, + n: usize, + d: &mut [T::Real], + e: &mut [T::Real], + tauq: &mut [T], + taup: &mut [T], +) { + // The right reflection not yet applied: its factor and vector, `w[0]` for the column `i`, + // and `t`, the rows of `C·w` from `i`. + let (mut tp, mut w) = (T::zero(), [T::zero(); N]); + let mut room = ([T::zero(); M], [T::zero(); M]); + let (mut t, mut next) = (&mut room.0, &mut room.1); + let (mut f, mut g) = ([T::zero(); N], [T::zero(); N]); + for i in 0..n { + let (left, right) = a.split_at_mut((i + 1) * m); + let col = &mut left[i * m..]; + if tp != T::zero() { + for (x, &t) in col[i..].iter_mut().zip(&t[i..m]) { + *x = fma(t, -tp, *x); + } + } + let (tq, beta) = { + let (head, tail) = col[i..].split_at_mut(1); + reflector(head[0], tail) + }; + d[i] = beta.re(); + tauq[i] = tq; + if i + 1 == n { + break; + } + let k = n - i - 1; + let v = &col[i + 1..]; + let (f, g) = (&mut f[..k], &mut g[..k]); + for (f, &w) in f.iter_mut().zip(&w[1..=k]) { + *f = tp * w.conj(); + } + // `g_j = conj(τ_q)·v*·(x_j - f_j·t)`, the row `i` of the columns after both + // reflections, and from it the next right reflection. + let vt = if tp != T::zero() { + t[i] + dotc(v, &t[i + 1..m]) + } else { + T::zero() + }; + products(v, &right[i..], m, k, g); + let ct = tq.conj(); + let z = &mut w[..k]; + for (j, ((z, g), &f)) in z.iter_mut().zip(g.iter_mut()).zip(f.iter()).enumerate() { + let x = right[i + j * m]; + *g = ct * (*g - f * vt); + *z = (x - f * t[i] - *g).conj(); + } + let (tp1, beta) = { + let (head, tail) = z.split_at_mut(1); + reflector(head[0], tail) + }; + e[i] = beta.re(); + taup[i] = tp1; + z[0] = T::one(); + for (j, &w) in z.iter().enumerate().skip(1) { + right[i + j * m] = w; + } + // The rows after `i`: `x_j -= f_j·t + g_j·v`, and `C·w` of the result. + let rows = i + 1..m; + update( + v, + (&t[rows.clone()], &f[..]), + &g[..], + &mut right[i + 1..], + m, + (&*z, &mut next[rows.clone()]), + tp1 != T::zero(), + ); + tp = tp1; + core::mem::swap(&mut t, &mut next); + } +} + +/// `p_j = (1, v)*·x_j` for the `k` columns `x_j` of `c` (leading dimension `ld`), each of +/// `v.len() + 1` elements: four columns a pass, in lanes of two vector registers each, so that +/// the products' additions don't wait on one another. +fn products(v: &[T], c: &[T], ld: usize, k: usize, p: &mut [T]) { + match size_of::() { + 4 => products_by::(v, c, ld, k, p), + 8 => products_by::(v, c, ld, k, p), + _ => products_by::(v, c, ld, k, p), + } +} + +/// [`products`] with `L` lanes. +fn products_by(v: &[T], c: &[T], ld: usize, k: usize, p: &mut [T]) { + let len = v.len() + 1; + let col = |j: usize| &c[j * ld..j * ld + len]; + let (vb, vt) = v.as_chunks::(); + let body = len - 1 - vt.len(); + let mut j = 0; + while j + 4 <= k { + let x = [col(j), col(j + 1), col(j + 2), col(j + 3)]; + let mut s = [[T::zero(); L]; 4]; + let [x0, x1, x2, x3] = x.map(|x| lanes_of::(&x[1..])); + for ((v, (y0, y1)), (y2, y3)) in vb.iter().zip(x0.iter().zip(x1)).zip(x2.iter().zip(x3)) { + for l in 0..L { + s[0][l] = fma_conj(v[l], y0[l], s[0][l]); + s[1][l] = fma_conj(v[l], y1[l], s[1][l]); + s[2][l] = fma_conj(v[l], y2[l], s[2][l]); + s[3][l] = fma_conj(v[l], y3[l], s[3][l]); + } + } + for q in 0..4 { + let mut t = x[q][0] + lanes(s[q]); + for (&v, &y) in vt.iter().zip(&x[q][1 + body..]) { + t = fma_conj(v, y, t); + } + p[j + q] = t; + } + j += 4; + } + while j < k { + let x = col(j); + p[j] = x[0] + dotc(v, &x[1..]); + j += 1; + } +} + +/// `x_j -= f_j·t + g_j·v` for the `k = g.len()` columns `x_j` of `c` (leading dimension `ld`) +/// of `v.len()` elements, and with `accumulate`, `y = Σ w_j·x_j` of the results: four columns a +/// pass, each element of `t`, `v` and `y` loaded once for the four. +fn update( + v: &[T], + (t, f): (&[T], &[T]), + g: &[T], + c: &mut [T], + ld: usize, + (w, y): (&[T], &mut [T]), + accumulate: bool, +) { + // Separate loops with and without the sums, which the compiler vectorizes only apart. + if accumulate { + update_by::(v, (t, f), g, c, ld, (w, y)); + } else { + update_by::(v, (t, f), g, c, ld, (w, y)); + } +} + +/// [`update`], the sums with `ACCUMULATE`. +fn update_by( + v: &[T], + (t, f): (&[T], &[T]), + g: &[T], + c: &mut [T], + ld: usize, + (w, y): (&[T], &mut [T]), +) { + let len = v.len(); + let k = g.len(); + let (t, y) = (&t[..len], &mut y[..len]); + y.fill(T::zero()); + let mut j = 0; + while j + 4 <= k { + let block = &mut c[j * ld..]; + let (c0, block) = block.split_at_mut(ld); + let (c1, block) = block.split_at_mut(ld); + let (c2, c3) = block.split_at_mut(ld); + let m = |a: &[T]| [-a[j], -a[j + 1], -a[j + 2], -a[j + 3]]; + let wj = [w[j], w[j + 1], w[j + 2], w[j + 3]]; + let (x0, x1, x2, x3) = ( + &mut c0[..len], + &mut c1[..len], + &mut c2[..len], + &mut c3[..len], + ); + columns4::(x0, x1, x2, x3, (v, t, &mut *y), (m(f), m(g), wj)); + j += 4; + } + while j < k { + let x = &mut c[j * ld..j * ld + len]; + let (mf, mg, wj) = (-f[j], -g[j], w[j]); + for ((x, (&v, &t)), y) in x.iter_mut().zip(v.iter().zip(t)).zip(y.iter_mut()) { + let u = fma(v, mg, fma(t, mf, *x)); + *x = u; + if ACCUMULATE { + *y = fma(u, wj, *y); + } + } + j += 1; + } +} + +/// The four columns of a pass of [`update_by`], each its own argument, so that the compiler +/// knows that they don't overlap and vectorizes the loop without checking at run time. +#[inline(always)] +fn columns4( + x0: &mut [T], + x1: &mut [T], + x2: &mut [T], + x3: &mut [T], + (v, t, y): (&[T], &[T], &mut [T]), + (mf, mg, w): ([T; 4], [T; 4], [T; 4]), +) { + let it = x0 + .iter_mut() + .zip(x1.iter_mut()) + .zip(x2.iter_mut().zip(x3.iter_mut())) + .zip(v.iter().zip(t).zip(y.iter_mut())); + for (((x0, x1), (x2, x3)), ((&v, &t), y)) in it { + let u0 = fma(v, mg[0], fma(t, mf[0], *x0)); + let u1 = fma(v, mg[1], fma(t, mf[1], *x1)); + let u2 = fma(v, mg[2], fma(t, mf[2], *x2)); + let u3 = fma(v, mg[3], fma(t, mf[3], *x3)); + (*x0, *x1, *x2, *x3) = (u0, u1, u2, u3); + if ACCUMULATE { + *y = fma(u3, w[3], fma(u2, w[2], fma(u1, w[1], fma(u0, w[0], *y)))); + } + } +} + +/// `V = P = G₀·G₁·…·G_{n-2}`, `n × n`, from the right reflections [`bidiagonalize`] left in the +/// rows of `a`, as LAPACK's `ungbr`: from the last, two at a time, each pair applied to the +/// columns it changes, then turned into its own two columns. +fn form_v(a: &[T], m: usize, n: usize, taus: &[T]) -> Vec { + let mut v = vec![T::zero(); n * n]; + if n == 0 { + return v; + } + // `G_r` acts on the rows from `r + 1`, its vector after the leading one in row `r` of `a` + // from column `r + 2`, gathered into `w`, and makes column `r + 1` of `V`. + let gather = |w: &mut [T], r: usize| { + for (k, w) in w.iter_mut().enumerate() { + *w = a[r + (r + 2 + k) * m]; + } + }; + let (mut wa, mut wb) = ([T::zero(); N], [T::zero(); N]); + let mut r = n - 1; + while r >= 2 { + // `G_{r-2}·G_{r-1}`, on the rows from `r - 1`. + let (b, c) = (r - 1, r - 2); + let (va, vb) = (&mut wa[..n - r], &mut wb[..n - r - 1]); + gather(va, c); + gather(vb, b); + let (left, right) = v.split_at_mut(r * n); + if r + 1 < n { + let pair = ((taus[c], &*va), (taus[b], &*vb)); + reflect_columns2(pair.0, pair.1, &mut right[n + b..], n, n - r - 1); + } + let col = &mut right[..n]; + make_column(col, r, taus[b], vb); + if taus[c] != T::zero() { + reflect_columns(taus[c], va, &mut col[b..], n, 1); + } + make_column(&mut left[b * n..], b, taus[c], va); + r -= 2; + } + if r == 1 { + let w = &mut wa[..n - 2]; + gather(w, 0); + let right = &mut v[n..]; + if taus[0] != T::zero() && n > 2 { + reflect_columns(taus[0], w, &mut right[n + 1..], n, n - 2); + } + make_column(&mut right[..n], 1, taus[0], w); + } + v[0] = T::one(); + v +} + +/// `U = Q·[I; 0] = H₀·H₁·…·H_{n-1}·[I; 0]`, `m × n`, in place of the reflections whose vectors +/// [`bidiagonalize`] left below the diagonal of `a`, as LAPACK's `ung2r`: from the last, two at +/// a time, each pair applied to the columns of `U` it changes, then turned into its own two +/// columns. +fn form_u(a: &mut [T], m: usize, n: usize, taus: &[T]) { + // `H_j` acts on the rows from `j`, its vector after the leading one in column `j` below the + // diagonal, and makes column `j`. + let mut j = n; + while j >= 2 { + // `H_{j-2}·H_{j-1}`, on the rows from `j - 2`. + let (b, c) = (j - 1, j - 2); + let (left, right) = a.split_at_mut(j * m); + let (left, colb) = left.split_at_mut(b * m); + let colc = &mut left[c * m..]; + if j < n { + let pair = ((taus[c], &colc[c + 1..]), (taus[b], &colb[b + 1..])); + reflect_columns2(pair.0, pair.1, &mut right[c..], m, n - j); + } + in_place_column(colb, b, taus[b]); + if taus[c] != T::zero() { + reflect_columns(taus[c], &colc[c + 1..], &mut colb[c..], m, 1); + } + in_place_column(colc, c, taus[c]); + j -= 2; + } + if j == 1 { + let (left, right) = a.split_at_mut(m); + if taus[0] != T::zero() && n > 1 { + reflect_columns(taus[0], &left[1..], right, m, n - 1); + } + in_place_column(left, 0, taus[0]); + } +} + +/// [`make_column`] with the vector already in place below the diagonal of `col`. +fn in_place_column(col: &mut [T], c: usize, tau: T) { + col[..c].fill(T::zero()); + col[c] = T::one() - tau; + let minus = -tau; + for x in &mut col[c + 1..] { + *x *= minus; + } +} + +/// Columns of a column-major block of any element type that the QR algorithm's rotations, real, +/// act on: the same rotation of the real and the imaginary parts. +struct Cols<'a, T> { + /// The elements, column `j` from `j · ld`. + data: &'a mut [T], + /// The rows rotated. + rows: usize, + /// The leading dimension. + ld: usize, +} + +impl qr::Vectors for Cols<'_, T> { + fn rotate(&mut self, i: usize, j: usize, c: T::Real, s: T::Real) { + let (rows, ld) = (self.rows, self.ld); + let (lo, hi) = (i.min(j), i.max(j)); + let (left, right) = self.data.split_at_mut(hi * ld); + let (p, q) = (&mut left[lo * ld..lo * ld + rows], &mut right[..rows]); + let (x, y) = if i < j { (p, q) } else { (q, p) }; + rotate(x, y, c, s); + } + + fn negate(&mut self, j: usize) { + for x in &mut self.data[j * self.ld..j * self.ld + self.rows] { + *x = -*x; + } + } + + fn swap(&mut self, i: usize, j: usize) { + let (rows, ld) = (self.rows, self.ld); + let (left, right) = self.data.split_at_mut(j * ld); + left[i * ld..i * ld + rows].swap_with_slice(&mut right[..rows]); + } +} + +/// `(x, y) = (c·x + s·y, c·y - s·x)`, the parts alike. Kept out of line, as the real one is. +#[inline(never)] +fn rotate(x: &mut [T], y: &mut [T], c: T::Real, s: T::Real) { + let fma = qr::fma::; + for (x, y) in x.iter_mut().zip(y.iter_mut()) { + let (a, b) = (*x, *y); + *x = T::from_parts(fma(c, a.re(), s * b.re()), fma(c, a.im(), s * b.im())); + *y = T::from_parts(fma(c, b.re(), -(s * a.re())), fma(c, b.im(), -(s * a.im()))); + } +} diff --git a/lib/src/math/matrix/tridiagonal.rs b/lib/src/math/matrix/tridiagonal.rs new file mode 100644 index 0000000..17699b3 --- /dev/null +++ b/lib/src/math/matrix/tridiagonal.rs @@ -0,0 +1,1384 @@ +//! The real symmetric tridiagonal eigenproblem the Hermitian one reduces to: the eigenvalues +//! alone by the square-root-free QL and QR algorithm, and the eigenvectors too by Cuppen's divide +//! and conquer, the rank-one updates' secular equations solved as LAPACK's `dlaed4`. + +use super::Scalar; +use super::blas::{Op, gemm}; +use crate::math::complex::Float; + +/// Problems up to this order are left to the QL algorithm, as LAPACK's `SMLSIZ`. +const SMALL: usize = 25; + +/// The iterations a root of a secular equation may take, as LAPACK's. +const MAX_ITERATIONS: usize = 30; + +/// The unit roundoff, half the gap between 1 and the next float, LAPACK's `EPS`. +fn unit_roundoff() -> R { + R::EPSILON * R::HALF +} + +/// The eigenvalues of the symmetric tridiagonal matrix with the diagonal `d` and the +/// subdiagonal `e`, ascending, into `d`, by the QL and QR algorithm without square roots of +/// Pal, Walker and Kahan, as LAPACK's `dsterf`; `e` is destroyed. False if an eigenvalue takes +/// more than `30 · n` iterations. +pub(super) fn values(d: &mut [R], e: &mut [R]) -> bool { + let n = d.len(); + let zero = R::ZERO; + let eps = unit_roundoff::(); + let eps2 = eps * eps; + let safmin = R::MIN_POSITIVE; + let ssfmax = (R::ONE / safmin).sqrt() / R::from_f64(3.0); + let ssfmin = safmin.sqrt() / eps2; + let max_iterations = 30 * n; + let mut iterations = 0; + let mut l1 = 0; + while l1 < n { + if l1 > 0 { + e[l1 - 1] = zero; + } + // The block `start..=end`, split off where the subdiagonal is negligible. + let mut m = l1; + while m + 1 < n { + if e[m].abs() <= d[m].abs().sqrt() * d[m + 1].abs().sqrt() * eps { + e[m] = zero; + break; + } + m += 1; + } + let (start, end) = (l1, m); + l1 = m + 1; + if end == start { + continue; + } + // Scaled into range, with the squares of the subdiagonal. + let mut norm = zero; + for i in start..=end { + norm = norm.max(d[i].abs()); + if i < end { + norm = norm.max(e[i].abs()); + } + } + if norm == zero { + continue; + } + let scale = if norm > ssfmax { + Some(ssfmax) + } else if norm < ssfmin { + Some(ssfmin) + } else { + None + }; + if let Some(s) = scale { + let f = s / norm; + for x in &mut d[start..=end] { + *x *= f; + } + for x in &mut e[start..end] { + *x *= f; + } + } + for x in &mut e[start..end] { + *x = *x * *x; + } + // QL, converging at the top, when the top is the smaller end, QR otherwise. + let ok = if d[end].abs() >= d[start].abs() { + values_ql(d, e, start, end, &mut iterations, max_iterations, eps2) + } else { + values_qr(d, e, start, end, &mut iterations, max_iterations, eps2) + }; + if let Some(s) = scale { + let f = norm / s; + for x in &mut d[start..=end] { + *x *= f; + } + } + if !ok { + return false; + } + } + sort(d); + true +} + +/// The QL sweeps of [`values`] on the block `l..=end`, `e` holding squares, from the top. +fn values_ql( + d: &mut [R], + e: &mut [R], + mut l: usize, + end: usize, + iterations: &mut usize, + max_iterations: usize, + eps2: R, +) -> bool { + let zero = R::ZERO; + loop { + let mut m = l; + while m < end && e[m].abs() > eps2 * (d[m] * d[m + 1]).abs() { + m += 1; + } + if m < end { + e[m] = zero; + } + let mut p = d[l]; + if m == l { + // An eigenvalue. + d[l] = p; + l += 1; + if l > end { + return true; + } + continue; + } + if m == l + 1 { + let (rt1, rt2) = eigenvalues_2x2(d[l], e[l].sqrt(), d[l + 1]); + d[l] = rt1; + d[l + 1] = rt2; + e[l] = zero; + l += 2; + if l > end { + return true; + } + continue; + } + if *iterations == max_iterations { + return false; + } + *iterations += 1; + // The shift. + let rte = e[l].sqrt(); + let mut sigma = (d[l + 1] - p) / (R::TWO * rte); + let r = sigma.hypot(R::ONE); + sigma = p - rte / (sigma + r.copysign(sigma)); + let (mut c, mut s) = (R::ONE, zero); + let mut gamma = d[m] - sigma; + p = gamma * gamma; + for i in (l..m).rev() { + let bb = e[i]; + let r = p + bb; + if i != m - 1 { + e[i + 1] = s * r; + } + let oldc = c; + c = p / r; + s = bb / r; + let oldgam = gamma; + let alpha = d[i]; + gamma = c * (alpha - sigma) - s * oldgam; + d[i + 1] = oldgam + (alpha - gamma); + p = if c != zero { + gamma * gamma / c + } else { + oldc * bb + }; + } + e[l] = s * p; + d[l] = sigma + gamma; + } +} + +/// The QR sweeps of [`values`] on the block `start..=end`, `e` holding squares, from the bottom. +fn values_qr( + d: &mut [R], + e: &mut [R], + start: usize, + end: usize, + iterations: &mut usize, + max_iterations: usize, + eps2: R, +) -> bool { + let zero = R::ZERO; + // `l` runs down from `end` to `start`. + let lend = start; + let mut l = end; + loop { + let mut m = l; + while m > lend && e[m - 1].abs() > eps2 * (d[m] * d[m - 1]).abs() { + m -= 1; + } + if m > lend { + e[m - 1] = zero; + } + let mut p = d[l]; + if m == l { + d[l] = p; + if l == lend { + return true; + } + l -= 1; + continue; + } + if m + 1 == l { + let (rt1, rt2) = eigenvalues_2x2(d[l], e[l - 1].sqrt(), d[l - 1]); + d[l] = rt1; + d[l - 1] = rt2; + e[l - 1] = zero; + if l < lend + 2 { + return true; + } + l -= 2; + continue; + } + if *iterations == max_iterations { + return false; + } + *iterations += 1; + let rte = e[l - 1].sqrt(); + let mut sigma = (d[l - 1] - p) / (R::TWO * rte); + let r = sigma.hypot(R::ONE); + sigma = p - rte / (sigma + r.copysign(sigma)); + let (mut c, mut s) = (R::ONE, zero); + let mut gamma = d[m] - sigma; + p = gamma * gamma; + for i in m..l { + let bb = e[i]; + let r = p + bb; + if i != m { + e[i - 1] = s * r; + } + let oldc = c; + c = p / r; + s = bb / r; + let oldgam = gamma; + let alpha = d[i + 1]; + gamma = c * (alpha - sigma) - s * oldgam; + d[i] = oldgam + (alpha - gamma); + p = if c != zero { + gamma * gamma / c + } else { + oldc * bb + }; + } + e[l - 1] = s * p; + d[l] = sigma + gamma; + } +} + +/// The eigenvalues of `[[a, b], [b, c]]`, the larger in absolute value first, as LAPACK's +/// `dlae2`. +fn eigenvalues_2x2(a: R, b: R, c: R) -> (R, R) { + let sm = a + c; + let df = a - c; + let adf = df.abs(); + let tb = b + b; + let ab = tb.abs(); + let (acmx, acmn) = if a.abs() > c.abs() { (a, c) } else { (c, a) }; + let rt = if adf > ab { + adf * (R::ONE + (ab / adf) * (ab / adf)).sqrt() + } else if adf < ab { + ab * (R::ONE + (adf / ab) * (adf / ab)).sqrt() + } else { + ab * R::TWO.sqrt() + }; + if sm < R::ZERO { + let rt1 = R::HALF * (sm - rt); + (rt1, (acmx / rt1) * acmn - (b / rt1) * b) + } else if sm > R::ZERO { + let rt1 = R::HALF * (sm + rt); + (rt1, (acmx / rt1) * acmn - (b / rt1) * b) + } else { + (R::HALF * rt, -R::HALF * rt) + } +} + +/// Sorts ascending, NaNs last. +fn sort(d: &mut [R]) { + d.sort_by(|a, b| { + a.partial_cmp(b) + .unwrap_or_else(|| b.is_nan().cmp(&a.is_nan())) + }); +} + +/// The eigenvalues, ascending, into `d`, and the eigenvectors, the columns of the returned +/// `n × n` matrix, of the symmetric tridiagonal matrix with the diagonal `d` and the +/// subdiagonal `e`, as LAPACK's `dstedc`: split where the subdiagonal is negligible, each block +/// scaled to 1 and solved by divide and conquer, its pieces of up to [`SMALL`] by the QL +/// algorithm. `e` is destroyed. `None` if an iteration fails to converge. +pub(super) fn eigen(d: &mut [S::Real], e: &mut [S::Real]) -> Option> { + let n = d.len(); + let mut q = vec![S::zero(); n * n]; + for j in 0..n { + q[j + j * n] = S::one(); + } + if n <= SMALL { + return ql(d, e, Some((&mut q, n, n))).then_some(q); + } + let eps = unit_roundoff::(); + let mut start = 0; + while start < n { + let mut end = start; + while end + 1 < n && e[end].abs() > eps * d[end].abs().sqrt() * d[end + 1].abs().sqrt() { + end += 1; + } + let m = end - start + 1; + let block = &mut q[start + start * n..]; + if m > SMALL { + let mut norm = S::Real::ZERO; + for i in start..=end { + norm = norm.max(d[i].abs()); + if i < end { + norm = norm.max(e[i].abs()); + } + } + let r = S::Real::ONE / norm; + for x in &mut d[start..=end] { + *x *= r; + } + for x in &mut e[start..end] { + *x *= r; + } + if !divide(&mut d[start..=end], &mut e[start..end], block, n) { + return None; + } + for x in &mut d[start..=end] { + *x *= norm; + } + } else if m > 1 && !ql(&mut d[start..=end], &e[start..end], Some((block, n, m))) { + return None; + } + start = end + 1; + } + // The blocks' eigenvalues merged into one ascending order, the vectors along. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&i, &j| { + d[i].partial_cmp(&d[j]) + .unwrap_or(core::cmp::Ordering::Equal) + }); + let sorted: Vec = order.iter().map(|&i| d[i]).collect(); + d.copy_from_slice(&sorted); + permute_columns(&mut q, n, n, &order); + Some(q) +} + +/// Divide and conquer on the unreduced `m × m` problem `(d, e)`, its eigenvectors into the +/// identity block `q` (leading dimension `ldq`): the matrix is cut in two by a rank-one change, +/// the halves solved, then their eigensystems merged, as LAPACK's `dlaed0`. +fn divide(d: &mut [S::Real], e: &mut [S::Real], q: &mut [S], ldq: usize) -> bool { + let m = d.len(); + if m <= SMALL { + return ql(d, e, Some((q, ldq, m))); + } + let m1 = m / 2; + let rho = e[m1 - 1]; + d[m1 - 1] -= rho.abs(); + d[m1] -= rho.abs(); + let (d1, d2) = d.split_at_mut(m1); + let (e1, e2) = e.split_at_mut(m1 - 1); + if !divide(d1, e1, q, ldq) || !divide(d2, &mut e2[1..], &mut q[m1 + m1 * ldq..], ldq) { + return false; + } + merge(d, q, ldq, m1, rho) +} + +/// Kinds of the columns of the merged eigenvectors, by where their nonzeros are. +const UPPER: usize = 0; +/// Nonzeros in both halves. +const DENSE: usize = 1; +/// Nonzeros in the lower half only. +const LOWER: usize = 2; +/// Deflated: the column is an eigenvector already. +const DEFLATED: usize = 3; + +/// Merges the eigensystems of the two halves of an `n × n` problem cut after row `n1` by the +/// subdiagonal `rho`: `d` holds both halves' eigenvalues, each ascending, and `q` their +/// eigenvectors, block diagonal. The eigensystem of `diag(D₁, D₂) + ρ·z·zᵀ`, `z` from the last +/// row of `Q₁` and the first of `Q₂`, replaces them, ascending: deflated where a component of +/// `z` is negligible or two eigenvalues are close (LAPACK's `dlaed2`), the rest from the roots +/// of the secular equation, with `z` recomputed from them by Gu and Eisenstat's formula so that +/// the vectors are orthogonal, then multiplied into `q` (LAPACK's `dlaed3`). +fn merge(d: &mut [S::Real], q: &mut [S], ldq: usize, n1: usize, rho: S::Real) -> bool { + let n = d.len(); + let n2 = n - n1; + let zero = S::Real::ZERO; + let mut z: Vec = (0..n1) + .map(|j| q[n1 - 1 + j * ldq].re()) + .chain((n1..n).map(|j| q[n1 + j * ldq].re())) + .collect(); + // `z` of norm 1 (each half's is a row of an orthogonal matrix) and `ρ > 0`. + let r = S::Real::HALF.sqrt(); + for (j, x) in z.iter_mut().enumerate() { + *x = if j >= n1 && rho < zero { + -*x * r + } else { + *x * r + }; + } + let rho = (S::Real::TWO * rho).abs(); + // The order of both halves' eigenvalues together. + let mut indx = Vec::with_capacity(n); + let (mut i, mut j) = (0, n1); + while i < n1 || j < n { + if j == n || (i < n1 && d[i] <= d[j]) { + indx.push(i); + i += 1; + } else { + indx.push(j); + j += 1; + } + } + let zmax = z.iter().fold(zero, |m, x| m.max(x.abs())); + let dmax = d.iter().fold(zero, |m, x| m.max(x.abs())); + let tol = S::Real::from_f64(8.0) * unit_roundoff::() * dmax.max(zmax); + if rho * zmax <= tol { + // The change is negligible: only the order. + let sorted: Vec = indx.iter().map(|&i| d[i]).collect(); + d.copy_from_slice(&sorted); + permute_columns(q, ldq, n, &indx); + return true; + } + // Deflation: columns whose `z` is negligible, and pairs of close eigenvalues, rotated so that + // one of the pair's `z` vanishes. The rest, `k` of them, go on in `dlamda` and `w`, in the + // order of `indxp`; the deflated ones are at the end of `indxp`, descending. + let mut kind: Vec = (0..n).map(|j| if j < n1 { UPPER } else { LOWER }).collect(); + let mut indxp = vec![0; n]; + let mut dlamda = Vec::with_capacity(n); + let mut w = Vec::with_capacity(n); + let mut k2 = n; + let mut pending: Option = None; + for &nj in &indx { + if rho * z[nj].abs() <= tol { + k2 -= 1; + kind[nj] = DEFLATED; + indxp[k2] = nj; + continue; + } + let Some(pj) = pending else { + pending = Some(nj); + continue; + }; + let tau = z[nj].hypot(z[pj]); + let t = d[nj] - d[pj]; + let (c, s) = (z[nj] / tau, -z[pj] / tau); + if (t * c * s).abs() <= tol { + // Close enough: rotate `z[pj]` to zero. + z[nj] = tau; + z[pj] = zero; + if kind[nj] != kind[pj] { + kind[nj] = DENSE; + } + kind[pj] = DEFLATED; + rotate_columns(q, ldq, n, pj, nj, c, s); + let t = d[pj] * c * c + d[nj] * s * s; + d[nj] = d[pj] * s * s + d[nj] * c * c; + d[pj] = t; + // Into the deflated ones, keeping them descending. + k2 -= 1; + let mut i = k2 + 1; + while i < n && d[pj] < d[indxp[i]] { + indxp[i - 1] = indxp[i]; + i += 1; + } + indxp[i - 1] = pj; + } else { + indxp[dlamda.len()] = pj; + dlamda.push(d[pj]); + w.push(z[pj]); + } + pending = Some(nj); + } + if let Some(pj) = pending { + indxp[dlamda.len()] = pj; + dlamda.push(d[pj]); + w.push(z[pj]); + } + let k = dlamda.len(); + // The columns grouped by kind, upper, dense, lower, deflated, packed into `q2` without + // their zero halves: `grouped[g]` is the column, `indxc[g]` its place in `indxp`. + let mut count = [0; 4]; + for &c in &kind { + count[c] += 1; + } + debug_assert_eq!(k, n - count[DEFLATED]); + let mut next = [ + 0, + count[0], + count[0] + count[1], + count[0] + count[1] + count[2], + ]; + let mut grouped = vec![0; n]; + let mut indxc = vec![0; n]; + for (j, &js) in indxp.iter().enumerate() { + let c = kind[js]; + grouped[next[c]] = js; + indxc[next[c]] = j; + next[c] += 1; + } + let n12 = count[UPPER] + count[DENSE]; + let n23 = count[DENSE] + count[LOWER]; + let lower_at = n1 * n12; + let deflated_at = lower_at + n2 * n23; + let mut q2 = vec![S::zero(); deflated_at + n * count[DEFLATED]]; + let mut values = vec![zero; n]; + for (g, &js) in grouped.iter().enumerate() { + let col = &q[js * ldq..js * ldq + n]; + if g < n12 { + q2[g * n1..(g + 1) * n1].copy_from_slice(&col[..n1]); + } + if g >= count[UPPER] && g < k { + let l = g - count[UPPER]; + q2[lower_at + l * n2..lower_at + (l + 1) * n2].copy_from_slice(&col[n1..]); + } + if g >= k { + let l = g - k; + q2[deflated_at + l * n..deflated_at + (l + 1) * n].copy_from_slice(col); + } + values[g] = d[js]; + } + // The deflated eigensystem goes back at the end. + for l in 0..n - k { + q[(k + l) * ldq..(k + l) * ldq + n] + .copy_from_slice(&q2[deflated_at + l * n..deflated_at + (l + 1) * n]); + d[k + l] = values[k + l]; + } + if k > 0 { + // The roots of the secular equation, each with its `dlamda - λ` in a column of `u`. + let mut u = vec![zero; k * k]; + for j in 0..k { + match secular(j, &dlamda, &w, &mut u[j * k..(j + 1) * k], rho) { + Some(lambda) => d[j] = lambda, + None => return false, + } + } + let mut v = vec![S::zero(); k * k]; + if k <= 2 { + // The roots came with their normalized eigenvectors. + for j in 0..k { + for g in 0..k { + v[g + j * k] = S::from_real(u[indxc[g] + j * k]); + } + } + } else { + // `z` again from the roots, by Löwner's formula, then the eigenvectors + // `(dlamda - λ_j)⁻¹·z`, normalized. + let sign = w.clone(); + for i in 0..k { + w[i] = u[i + i * k]; + } + for j in 0..k { + let (col, dj) = (&u[j * k..(j + 1) * k], dlamda[j]); + let (left, right) = w.split_at_mut(j); + for ((w, &u), &dl) in left.iter_mut().zip(col).zip(&dlamda[..j]) { + *w *= u / (dl - dj); + } + let (col, dl) = (&col[j + 1..], &dlamda[j + 1..]); + for ((w, &u), &dl) in right[1..].iter_mut().zip(col).zip(dl) { + *w *= u / (dl - dj); + } + } + for (w, &s) in w.iter_mut().zip(&sign) { + *w = (-*w).sqrt().copysign(s); + } + let mut s = vec![zero; k]; + for j in 0..k { + for ((s, &w), &u) in s.iter_mut().zip(&w).zip(&u[j * k..(j + 1) * k]) { + *s = w / u; + } + let r = S::Real::ONE / norm(&s); + for (x, &g) in v[j * k..(j + 1) * k].iter_mut().zip(&indxc) { + *x = S::from_real(s[g] * r); + } + } + } + // The eigenvectors: the upper rows from the upper and dense columns, the lower rows + // from the dense and lower ones. + let (one, nil) = (S::one(), S::zero()); + if n23 > 0 { + let vl = &v[count[UPPER]..]; + gemm( + n2, + k, + n23, + one, + &q2[lower_at..], + n2, + Op::N, + vl, + k, + Op::N, + nil, + &mut q[n1..], + ldq, + ); + } else { + for j in 0..k { + q[j * ldq + n1..j * ldq + n].fill(nil); + } + } + if n12 > 0 { + gemm(n1, k, n12, one, &q2, n1, Op::N, &v, k, Op::N, nil, q, ldq); + } else { + for j in 0..k { + q[j * ldq..j * ldq + n1].fill(nil); + } + } + } + // The roots ascending, the deflated eigenvalues descending: merged. + let mut order = Vec::with_capacity(n); + let (mut i, mut j) = (0, n); + while i < k || j > k { + if j == k || (i < k && d[i] <= d[j - 1]) { + order.push(i); + i += 1; + } else { + j -= 1; + order.push(j); + } + } + let sorted: Vec = order.iter().map(|&i| d[i]).collect(); + d.copy_from_slice(&sorted); + permute_columns(q, ldq, n, &order); + true +} + +/// The 2-norm, in lanes, of a vector whose squares neither overflow nor underflow, as those of +/// the merged eigenvectors of a problem scaled to 1. +fn norm(x: &[R]) -> R { + let mut s = [R::ZERO; 4]; + let (quads, rest) = x.as_chunks::<4>(); + for c in quads { + for l in 0..4 { + s[l] += c[l] * c[l]; + } + } + for &v in rest { + s[0] += v * v; + } + ((s[0] + s[2]) + (s[1] + s[3])).sqrt() +} + +/// `(x, y) = (c·x + s·y, c·y - s·x)` for the columns `x` and `y` of the `rows`-row block `q`. +fn rotate_columns( + q: &mut [S], + ldq: usize, + rows: usize, + x: usize, + y: usize, + c: S::Real, + s: S::Real, +) { + for r in 0..rows { + let (a, b) = (q[r + x * ldq], q[r + y * ldq]); + q[r + x * ldq] = a.scale(c) + b.scale(s); + q[r + y * ldq] = b.scale(c) - a.scale(s); + } +} + +/// Reorders the columns of the `rows`-row block `q`: column `j` becomes the old column +/// `order[j]`, cycle by cycle. +fn permute_columns(q: &mut [S], ldq: usize, rows: usize, order: &[usize]) { + let mut done = vec![false; order.len()]; + let mut saved = vec![S::zero(); rows]; + for start in 0..order.len() { + if done[start] || order[start] == start { + continue; + } + saved.copy_from_slice(&q[start * ldq..start * ldq + rows]); + let mut j = start; + loop { + done[j] = true; + let from = order[j]; + if from == start { + q[j * ldq..j * ldq + rows].copy_from_slice(&saved); + break; + } + q.copy_within(from * ldq..from * ldq + rows, j * ldq); + j = from; + } + } +} + +/// `δ_j = (d_j - base) - offset`, the differences from a pole first so that they are exact. +fn shift(delta: &mut [R], d: &[R], base: R, offset: R) { + for (x, &dj) in delta.iter_mut().zip(d) { + *x = (dj - base) - offset; + } +} + +/// After `δ_j -= η`, the sums over the poles of `p_j = z_j²/δ_j`, of `(z_j/δ_j)²` and of +/// `p_j·w_j` with the weights `w_j = w₀ - j` (`w₀ > 0`, counting down) or `j + 1` (`w₀ < 0`): +/// the last, LAPACK's sum of the partial sums of the first in the order it adds them, is its +/// rounding error bound, which this computes in lanes instead of in order. +fn sums(delta: &mut [R], z: &[R], eta: R, w0: R) -> (R, R, R) { + let zero = R::ZERO; + let (mut s, mut ds, mut es) = ([zero; 4], [zero; 4], [zero; 4]); + let one = R::ONE; + let (start, step) = if w0 > zero { (w0, -one) } else { (one, one) }; + let mut w = [ + start, + start + step, + start + step + step, + start + step + step + step, + ]; + let step4 = step * R::from_f64(4.0); + let (head, tail) = delta.as_chunks_mut::<4>(); + let (zh, zt) = z.as_chunks::<4>(); + for (dv, zv) in head.iter_mut().zip(zh) { + for l in 0..4 { + dv[l] -= eta; + let t = zv[l] / dv[l]; + let p = zv[l] * t; + s[l] += p; + ds[l] += t * t; + es[l] += p * w[l]; + w[l] += step4; + } + } + for (dv, &zv) in tail.iter_mut().zip(zt) { + *dv -= eta; + let t = zv / *dv; + let p = zv * t; + s[0] += p; + ds[0] += t * t; + es[0] += p * w[0]; + w[0] += step; + } + let sum = |v: [R; 4]| (v[0] + v[2]) + (v[1] + v[3]); + (sum(s), sum(ds), sum(es)) +} + +/// The `i`-th smallest root `λ` of the secular equation `1/ρ + Σ z_j²/(d_j - λ) = 0`, for +/// ascending `d`, `ρ > 0` and `z` of norm 1, as LAPACK's `dlaed4`: `delta` gets `d_j - λ`, +/// computed from the nearest pole so that it is accurate, or for two poles the normalized +/// eigenvector. Each step interpolates the function by rational functions of the poles near the +/// root (two, or three by LAPACK's `dlaed6`), safeguarded by a bracket. `None` if it fails to +/// converge. +fn secular(i: usize, d: &[R], z: &[R], delta: &mut [R], rho: R) -> Option { + let n = d.len(); + if n == 1 { + delta[0] = R::ONE; + return Some(d[0] + rho * z[0] * z[0]); + } + if n == 2 { + return Some(secular2(i, d, z, delta, rho)); + } + if i == n - 1 { + secular_last(d, z, delta, rho) + } else { + secular_inner(i, d, z, delta, rho) + } +} + +/// [`secular`] of the largest root, beyond the last pole. +fn secular_last(d: &[R], z: &[R], delta: &mut [R], rho: R) -> Option { + let n = d.len(); + let (zero, two, four) = (R::ZERO, R::TWO, R::from_f64(4.0)); + let eps = unit_roundoff::(); + let rhoinv = R::ONE / rho; + let i = n - 1; + let ii = n - 2; + // The first guess, from the two last poles. + let midpt = rho / two; + shift(delta, d, d[i], midpt); + let (psi, _, _) = sums(&mut delta[..n - 2], &z[..n - 2], zero, zero); + let c = rhoinv + psi; + let w = c + z[ii] * z[ii] / delta[ii] + z[i] * z[i] / delta[i]; + let guess = |c: R| { + let del = d[i] - d[ii]; + let a = -c * del + z[ii] * z[ii] + z[i] * z[i]; + let b = z[i] * z[i] * del; + if a < zero { + two * b / ((a * a + four * b * c).sqrt() - a) + } else { + (a + (a * a + four * b * c).sqrt()) / (two * c) + } + }; + let (mut tau, mut lower, mut upper); + if w <= zero { + let temp = z[ii] * z[ii] / (d[i] - d[ii] + rho) + z[i] * z[i] / rho; + tau = if c <= temp { rho } else { guess(c) }; + // `d_n + ρ/2 ≤ λ_n < d_n + τ ≤ d_n + ρ`. + lower = midpt; + upper = rho; + } else { + tau = guess(c); + lower = zero; + upper = midpt; + } + shift(delta, d, d[i], tau); + // The function, its derivatives on both sides of the last pole and a bound on the rounding + // error, after `δ -= η`. + let evaluate = |delta: &mut [R], eta: R, tau: R| { + let (psi, dpsi, erretm) = sums( + &mut delta[..=ii], + &z[..=ii], + eta, + R::from_f64((ii + 1) as f64), + ); + delta[i] -= eta; + let t = z[i] / delta[i]; + let phi = z[i] * t; + let dphi = t * t; + let erretm = R::from_f64(8.0) * (-phi - psi) + erretm.abs() - phi + + rhoinv + + tau.abs() * (dpsi + dphi); + (rhoinv + phi + psi, dpsi, dphi, erretm) + }; + let (mut w, mut dpsi, mut dphi, mut erretm) = evaluate(delta, zero, tau); + for iteration in 1..=MAX_ITERATIONS { + if w.abs() <= eps * erretm { + return Some(d[i] + tau); + } + if w <= zero { + lower = lower.max(tau); + } else { + upper = upper.min(tau); + } + // The step by the rational interpolation of the two last poles. + let mut c = w - delta[ii] * dpsi - delta[i] * dphi; + let a = (delta[ii] + delta[i]) * w - delta[ii] * delta[i] * (dpsi + dphi); + let b = delta[ii] * delta[i] * w; + if iteration == 1 && c < zero { + c = c.abs(); + } + let newton = -w / (dpsi + dphi); + let mut eta = if c == zero { + newton + } else if a >= zero { + (a + (a * a - four * b * c).abs().sqrt()) / (two * c) + } else { + two * b / (a - (a * a - four * b * c).abs().sqrt()) + }; + // `η` must have the sign of `-w`; then it must stay in the bracket. + if w * eta > zero || (w * eta).is_nan() { + eta = newton; + } + let t = tau + eta; + if !(t <= upper && t >= lower) { + eta = if w < zero { + (upper - tau) / two + } else { + (lower - tau) / two + }; + } + tau += eta; + (w, dpsi, dphi, erretm) = evaluate(delta, eta, tau); + } + None +} + +/// [`secular`] of a root between the poles `d_i` and `d_{i+1}`. +fn secular_inner(i: usize, d: &[R], z: &[R], delta: &mut [R], rho: R) -> Option { + let n = d.len(); + let (zero, two, four) = (R::ZERO, R::TWO, R::from_f64(4.0)); + let eps = unit_roundoff::(); + let rhoinv = R::ONE / rho; + let ip1 = i + 1; + // The first guess, from the two nearest poles; the origin is the nearer one. + let del = d[ip1] - d[i]; + let midpt = del / two; + shift(delta, d, d[i], midpt); + let (psi, _, _) = sums(&mut delta[..i], &z[..i], zero, zero); + let (phi, _, _) = sums(&mut delta[i + 2..], &z[i + 2..], zero, zero); + let c = rhoinv + psi + phi; + let w = c + z[i] * z[i] / delta[i] + z[ip1] * z[ip1] / delta[ip1]; + let orgati = w > zero; + let (mut tau, mut lower, mut upper); + if orgati { + // `d_i < λ < (d_i + d_{i+1})/2`, from `d_i`. + let a = c * del + z[i] * z[i] + z[ip1] * z[ip1]; + let b = z[i] * z[i] * del; + tau = if a > zero { + two * b / (a + (a * a - four * b * c).abs().sqrt()) + } else { + (a - (a * a - four * b * c).abs().sqrt()) / (two * c) + }; + lower = zero; + upper = midpt; + } else { + // `(d_i + d_{i+1})/2 ≤ λ < d_{i+1}`, from `d_{i+1}`. + let a = c * del - z[i] * z[i] - z[ip1] * z[ip1]; + let b = z[ip1] * z[ip1] * del; + tau = if a < zero { + two * b / (a - (a * a + four * b * c).abs().sqrt()) + } else { + -(a + (a * a + four * b * c).abs().sqrt()) / (two * c) + }; + lower = -midpt; + upper = zero; + } + let origin = if orgati { d[i] } else { d[ip1] }; + shift(delta, d, origin, tau); + let ii = if orgati { i } else { ip1 }; + // After `δ -= η`, the function without its `ii` term, split at the pole `ii`, with the + // derivatives and a bound on the rounding error. + let evaluate = |delta: &mut [R], eta: R| { + let (left, right) = delta.split_at_mut(ii); + let (psi, dpsi, epsi) = sums(left, &z[..ii], eta, R::from_f64(ii as f64)); + let (phi, dphi, ephi) = sums(&mut right[1..], &z[ii + 1..], eta, -R::ONE); + right[0] -= eta; + let t = z[ii] / right[0]; + let dw = dpsi + dphi + t * t; + let t = z[ii] * t; + (psi, dpsi, phi, dphi, t, dw, epsi.abs() + ephi) + }; + let (mut psi, mut dpsi, mut phi, mut dphi, temp, mut dw, mut erretm) = evaluate(delta, zero); + let w0 = rhoinv + phi + psi; + // Three poles when the root is on the far side of `ii` from its neighbour pole. + let swtch3 = (if orgati { w0 < zero } else { w0 > zero }) && ii != 0 && ii != n - 1; + let mut w = w0 + temp; + let error = |erretm: R, phi: R, psi: R, temp: R, tau: R, dw: R| { + R::from_f64(8.0) * (phi - psi) + + erretm + + two * rhoinv + + R::from_f64(3.0) * temp.abs() + + tau.abs() * dw + }; + erretm = error(erretm, phi, psi, temp, tau, dw); + let zii2 = z[ii] * z[ii]; + let mut swtch = false; + for iteration in 1..=MAX_ITERATIONS { + if w.abs() <= eps * erretm { + return Some(origin + tau); + } + if w <= zero { + lower = lower.max(tau); + } else { + upper = upper.min(tau); + } + let mut eta; + if !swtch3 { + let c = if swtch { + let t = z[ii] / delta[ii]; + if orgati { + dpsi += t * t; + } else { + dphi += t * t; + } + w - delta[i] * dpsi - delta[ip1] * dphi + } else if orgati { + let t = z[i] / delta[i]; + w - delta[ip1] * dw - (d[i] - d[ip1]) * t * t + } else { + let t = z[ip1] / delta[ip1]; + w - delta[i] * dw - (d[ip1] - d[i]) * t * t + }; + let mut a = (delta[i] + delta[ip1]) * w - delta[i] * delta[ip1] * dw; + let b = delta[i] * delta[ip1] * w; + eta = if c == zero { + if a == zero { + a = if swtch { + delta[i] * delta[i] * dpsi + delta[ip1] * delta[ip1] * dphi + } else if orgati { + z[i] * z[i] + delta[ip1] * delta[ip1] * (dpsi + dphi) + } else { + z[ip1] * z[ip1] + delta[i] * delta[i] * (dpsi + dphi) + }; + } + b / a + } else if a <= zero { + (a - (a * a - four * b * c).abs().sqrt()) / (two * c) + } else { + two * b / (a + (a * a - four * b * c).abs().sqrt()) + }; + } else { + // Rational interpolation of the three poles around the root. + let (iim1, iip1) = (ii - 1, ii + 1); + let temp = rhoinv + psi + phi; + let (c, z1, z3); + if swtch { + c = temp - delta[iim1] * dpsi - delta[iip1] * dphi; + z1 = delta[iim1] * delta[iim1] * dpsi; + z3 = delta[iip1] * delta[iip1] * dphi; + } else if orgati { + let t = z[iim1] / delta[iim1]; + let t = t * t; + c = temp - delta[iip1] * (dpsi + dphi) - (d[iim1] - d[iip1]) * t; + z1 = z[iim1] * z[iim1]; + z3 = delta[iip1] * delta[iip1] * ((dpsi - t) + dphi); + } else { + let t = z[iip1] / delta[iip1]; + let t = t * t; + c = temp - delta[iim1] * (dpsi + dphi) - (d[iip1] - d[iim1]) * t; + z1 = delta[iim1] * delta[iim1] * (dpsi + (dphi - t)); + z3 = z[iip1] * z[iip1]; + } + let poles = [delta[iim1], delta[ii], delta[iip1]]; + eta = rational3(iteration + 1, orgati, c, &poles, &[z1, zii2, z3], w)?; + } + // `η` must have the sign of `-w`; then it must stay in the bracket. + if w * eta >= zero || (w * eta).is_nan() { + eta = -w / dw; + } + let t = tau + eta; + if !(t <= upper && t >= lower) { + eta = if w < zero { + (upper - tau) / two + } else { + (lower - tau) / two + }; + } + tau += eta; + let last = w; + let temp; + (psi, dpsi, phi, dphi, temp, dw, erretm) = evaluate(delta, eta); + w = rhoinv + phi + psi + temp; + erretm = error(erretm, phi, psi, temp, tau, dw); + let tenth = last.abs() / R::from_f64(10.0); + if iteration == 1 { + swtch = if orgati { -w > tenth } else { w > tenth }; + } else if w * last > zero && w.abs() > tenth { + swtch = !swtch; + } + } + None +} + +/// [`secular`] of two poles, as LAPACK's `dlaed5`: `delta` gets the normalized eigenvector. +fn secular2(i: usize, d: &[R], z: &[R], delta: &mut [R], rho: R) -> R { + let (two, four) = (R::TWO, R::from_f64(4.0)); + let del = d[1] - d[0]; + let (z0, z1) = (z[0] * z[0], z[1] * z[1]); + let lambda; + if i == 0 { + let w = R::ONE + two * rho * (z1 - z0) / del; + if w > R::ZERO { + let b = del + rho * (z0 + z1); + let c = rho * z0 * del; + // `b > 0` always. + let tau = two * c / (b + (b * b - four * c).abs().sqrt()); + lambda = d[0] + tau; + delta[0] = -z[0] / tau; + delta[1] = z[1] / (del - tau); + } else { + let b = -del + rho * (z0 + z1); + let c = rho * z1 * del; + let tau = if b > R::ZERO { + -two * c / (b + (b * b + four * c).sqrt()) + } else { + (b - (b * b + four * c).sqrt()) / two + }; + lambda = d[1] + tau; + delta[0] = -z[0] / (del + tau); + delta[1] = -z[1] / tau; + } + } else { + let b = -del + rho * (z0 + z1); + let c = rho * z1 * del; + let tau = if b > R::ZERO { + (b + (b * b + four * c).sqrt()) / two + } else { + two * c / (-b + (b * b + four * c).sqrt()) + }; + lambda = d[1] + tau; + delta[0] = -z[0] / (del + tau); + delta[1] = -z[1] / tau; + } + let t = delta[0].hypot(delta[1]); + delta[0] /= t; + delta[1] /= t; + lambda +} + +/// The step `τ` to the root of `ρ + Σ z_j/(d_j - x)` over three poles nearest to 0, from +/// `f(0) = finit`, by Gragg, Thornton and Warner's cubically convergent iteration, as LAPACK's +/// `dlaed6`; `orgati` tells whether the root is between `d₂` and `d₃` (or `d₁` and `d₂`). +fn rational3( + kniter: usize, + orgati: bool, + rho: R, + d: &[R; 3], + z: &[R; 3], + finit: R, +) -> Option { + let (zero, two, four) = (R::ZERO, R::TWO, R::from_f64(4.0)); + let eps = unit_roundoff::(); + let (mut lbd, mut ubd) = if orgati { (d[1], d[2]) } else { (d[0], d[1]) }; + if finit < zero { + lbd = zero; + } else { + ubd = zero; + } + let mut tau = zero; + let solve = |a: R, b: R, c: R| { + let m = a.abs().max(b.abs()).max(c.abs()); + let (a, b, c) = (a / m, b / m, c / m); + if c == zero { + b / a + } else if a <= zero { + (a - (a * a - four * b * c).abs().sqrt()) / (two * c) + } else { + two * b / (a + (a * a - four * b * c).abs().sqrt()) + } + }; + if kniter == 2 { + let (a, b, c); + if orgati { + let t = (d[2] - d[1]) / two; + c = rho + z[0] / ((d[0] - d[1]) - t); + a = c * (d[1] + d[2]) + z[1] + z[2]; + b = c * d[1] * d[2] + z[1] * d[2] + z[2] * d[1]; + } else { + let t = (d[0] - d[1]) / two; + c = rho + z[2] / ((d[2] - d[1]) - t); + a = c * (d[0] + d[1]) + z[0] + z[1]; + b = c * d[0] * d[1] + z[0] * d[1] + z[1] * d[0]; + } + tau = solve(a, b, c); + if !(tau >= lbd && tau <= ubd) { + tau = (lbd + ubd) / two; + } + if d[0] == tau || d[1] == tau || d[2] == tau { + tau = zero; + } else { + let f = finit + + tau * z[0] / (d[0] * (d[0] - tau)) + + tau * z[1] / (d[1] * (d[1] - tau)) + + tau * z[2] / (d[2] * (d[2] - tau)); + if f <= zero { + lbd = tau; + } else { + ubd = tau; + } + if finit.abs() <= f.abs() { + tau = zero; + } + } + } + // Scaled up when a pole is so near that `1/(d - τ)³` would overflow. + let exponent = (R::MIN_POSITIVE.to_f64().log2() / 3.0).trunc() as i32; + let small1 = two.powi(exponent); + let sminv1 = R::ONE / small1; + let small2 = small1 * small1; + let sminv2 = sminv1 * sminv1; + let near = if orgati { + (d[1] - tau).abs().min((d[2] - tau).abs()) + } else { + (d[0] - tau).abs().min((d[1] - tau).abs()) + }; + let (mut ds, mut zs) = (*d, *z); + let mut unscale = R::ONE; + if near <= small1 { + let (f, inv) = if near <= small2 { + (sminv2, small2) + } else { + (sminv1, small1) + }; + for j in 0..3 { + ds[j] *= f; + zs[j] *= f; + } + tau *= f; + lbd *= f; + ubd *= f; + unscale = inv; + } + let (mut fc, mut df, mut ddf) = (zero, zero, zero); + for j in 0..3 { + let t = R::ONE / (ds[j] - tau); + let t1 = zs[j] * t; + let t2 = t1 * t; + fc += t1 / ds[j]; + df += t2; + ddf += t2 * t; + } + let mut f = finit + tau * fc; + if f.abs() <= zero { + return Some(tau * unscale); + } + if f <= zero { + lbd = tau; + } else { + ubd = tau; + } + for _ in 2..=40 { + let (t1, t2) = if orgati { + (ds[1] - tau, ds[2] - tau) + } else { + (ds[0] - tau, ds[1] - tau) + }; + let a = (t1 + t2) * f - t1 * t2 * df; + let b = t1 * t2 * f; + let c = f - (t1 + t2) * df + t1 * t2 * ddf; + let mut eta = solve(a, b, c); + if f * eta >= zero || (f * eta).is_nan() { + eta = -f / df; + } + tau += eta; + if !(tau >= lbd && tau <= ubd) { + tau = (lbd + ubd) / two; + } + let (mut erretm, mut pole) = (zero, false); + fc = zero; + df = zero; + ddf = zero; + for j in 0..3 { + if ds[j] - tau == zero { + pole = true; + break; + } + let t = R::ONE / (ds[j] - tau); + let t1 = zs[j] * t; + let t2 = t1 * t; + let t4 = t1 / ds[j]; + fc += t4; + erretm += t4.abs(); + df += t2; + ddf += t2 * t; + } + if pole { + return Some(tau * unscale); + } + f = finit + tau * fc; + let erretm = R::from_f64(8.0) * (finit.abs() + tau.abs() * erretm) + tau.abs() * df; + if f.abs() <= four * eps * erretm || (ubd - lbd) <= four * eps * tau.abs() { + return Some(tau * unscale); + } + if f <= zero { + lbd = tau; + } else { + ubd = tau; + } + } + None +} + +/// The eigenvalues of the symmetric tridiagonal matrix with the diagonal `d` and the +/// subdiagonal `e`, ascending, into `d`, by the implicit QL algorithm with Wilkinson's shifts, +/// EISPACK's `tql2`; its rotations are applied to the columns of the block `q` of `rows` rows +/// and the leading dimension `ldq`, when given, which turns the identity into the eigenvectors. +/// False if an eigenvalue takes more than `30 · max(n, 10)` iterations. +/// +/// Each step of a sweep waits on the one before through a square root and a division, which on +/// small problems costs more than the rotations of the vectors, so the step is kept short: the +/// length of `(p, e_i)` from the sum of the squares where it can neither overflow nor underflow +/// enough to matter, `hypot` only elsewhere, and the next `p` divided by it directly, in +/// parallel with the rotation's cosine and sine. +pub(super) fn ql( + d: &mut [S::Real], + sub: &[S::Real], + mut q: Option<(&mut [S], usize, usize)>, +) -> bool { + let n = d.len(); + if n == 0 { + return true; + } + let zero = S::Real::ZERO; + let one = S::Real::ONE; + // The subdiagonal with a zero after it, on the stack for the orders that are common here: + // the leaves of divide and conquer and the small matrices of the Hermitian eigenproblem. + let mut stack = [zero; 160]; + let mut heap = Vec::new(); + let e: &mut [S::Real] = if n <= stack.len() { + &mut stack[..n] + } else { + heap.resize(n, zero); + &mut heap + }; + e[..n - 1].copy_from_slice(&sub[..n - 1]); + let eps = S::Real::EPSILON; + let mut f = zero; + let mut tst1 = zero; + for l in 0..n { + tst1 = tst1.max(d[l].abs() + e[l].abs()); + let mut m = l; + while m < n - 1 && e[m].abs() > eps * tst1 { + m += 1; + } + if m > l { + let mut its = 0; + loop { + its += 1; + if its > 30 * n.max(10) { + return false; + } + // The shift from the leading 2 × 2 block. + let g = d[l]; + let p = (d[l + 1] - g) / (S::Real::TWO * e[l]); + let r = length(p, one); + let r = if p < zero { -r } else { r }; + d[l] = e[l] / (p + r); + d[l + 1] = e[l] * (p + r); + let dl1 = d[l + 1]; + let h = g - d[l]; + for x in &mut d[l + 2..] { + *x -= h; + } + f += h; + // The QL sweep, from the bottom of the block up. + let mut p = d[m]; + let (mut c, mut c2, mut c3) = (one, one, one); + let el1 = e[l + 1]; + let (mut s, mut s2) = (zero, zero); + for i in (l..m).rev() { + c3 = c2; + c2 = c; + s2 = s; + let (ei, di) = (e[i], d[i]); + let g = c * ei; + let h = c * p; + let r = length(p, ei); + e[i + 1] = s * r; + // `c·d_i - s·g` with `c = p/r` and `s = e_i/r`, divided once. + let t = fma(p, di, -(ei * g)); + s = ei / r; + c = p / r; + p = t / r; + d[i + 1] = fma(s, fma(c, g, s * di), h); + if let Some((q, ldq, rows)) = q.as_mut() { + let (left, right) = q.split_at_mut((i + 1) * *ldq); + let x = &mut left[i * *ldq..i * *ldq + *rows]; + rotate(x, &mut right[..*rows], c, s); + } + } + let p = -s * s2 * c3 * el1 * e[l] / dl1; + e[l] = s * p; + d[l] = c * p; + if e[l].abs() <= eps * tst1 { + break; + } + } + } + d[l] += f; + e[l] = zero; + } + // Ascending, the vectors along. + for i in 0..n { + let mut k = i; + for j in i + 1..n { + if d[j] < d[k] { + k = j; + } + } + if k != i { + d.swap(i, k); + if let Some((q, ldq, rows)) = q.as_mut() { + let (left, right) = q.split_at_mut(k * *ldq); + left[i * *ldq..i * *ldq + *rows].swap_with_slice(&mut right[..*rows]); + } + } + } + true +} + +/// `√(a² + b²)`: from the sum of the squares when it is in the range where no square can have +/// overflowed or underflowed to matter, as it nearly always is, and by `hypot` otherwise. +#[inline(always)] +fn length(a: R, b: R) -> R { + let ss = fma(a, a, b * b); + if ss >= R::MIN_POSITIVE / R::EPSILON && ss <= R::MAX { + ss.sqrt() + } else { + a.hypot(b) + } +} + +/// Whether the target fuses multiply-adds in hardware; elsewhere `mul_add` is a library call. +const FMA: bool = cfg!(any(target_arch = "aarch64", target_feature = "fma")); + +/// `a·b + c`, rounded once where the target fuses multiply-adds. +#[inline(always)] +fn fma(a: R, b: R, c: R) -> R { + if FMA { a.mul_add(b, c) } else { a * b + c } +} + +/// `(x, y) = (c·x - s·y, s·x + c·y)` for the columns `x` and `y`, of any element type: the +/// same rotation of the real and the imaginary parts. Kept out of line: inlined into the sweep, +/// the compiler moves the sweep's next divisions below this loop, and each step of the sweep +/// then waits for the rotation of the vectors instead of running beside it. +#[inline(never)] +fn rotate(x: &mut [S], y: &mut [S], c: S::Real, s: S::Real) { + for (x, y) in x.iter_mut().zip(y.iter_mut()) { + let (a, b) = (*x, *y); + *x = S::from_parts(fma(-b.re(), s, a.re() * c), fma(-b.im(), s, a.im() * c)); + *y = S::from_parts(fma(a.re(), s, b.re() * c), fma(a.im(), s, b.im() * c)); + } +} diff --git a/lib/src/math/mod.rs b/lib/src/math/mod.rs index dc19335..9014270 100644 --- a/lib/src/math/mod.rs +++ b/lib/src/math/mod.rs @@ -1,12 +1,20 @@ //! Mathematics. //! +//! * [`complex`]: complex numbers with `f32` or `f64` parts, their arithmetic with `+`, `-`, `*` +//! and `/`, and their absolute values, arguments, exponentials, logarithms, square roots and +//! powers. //! * [`ec`]: elliptic curves over prime fields in Weierstrass, Montgomery and Edwards form, whose //! points add with `+`, subtract with `-` and multiply by a number with `*`, and the curves in //! common use; on numbers, and not constant-time. +//! * [`matrix`]: matrices of floats, complex numbers or numbers, their arithmetic with `+`, `-` +//! and `*`, transposes, row and column operations, determinants, decompositions, eigenvalues +//! and functions of matrices. //! * [`numbers`]: integers of any size, stored as a sign and the bytes of the absolute value //! in either byte order; they add with `+` and `+=`, subtract with `-` and `-=`, multiply //! with `*` and `*=`, divide with `/`, `/=`, `%` and `%=`, shift with `<<`, `>>`, `<<=` and //! `>>=`, and are written and read as text in any radix from 2 to 36. +pub mod complex; pub mod ec; +pub mod matrix; pub mod numbers; diff --git a/lib/tests/math/complex/mod.rs b/lib/tests/math/complex/mod.rs new file mode 100644 index 0000000..80e0435 --- /dev/null +++ b/lib/tests/math/complex/mod.rs @@ -0,0 +1,211 @@ +//! Complex numbers: exact cases, branch cuts, extreme magnitudes and identities. + +use a::math::complex::{Complex, Float}; +use core::f64::consts::{FRAC_PI_2, PI}; + +fn c(re: f64, im: f64) -> Complex { + Complex::new(re, im) +} + +/// Asserts `|a - b| ≤ tol·|b|`, or `≤ tol` for `b = 0`. +#[track_caller] +fn near(a: Complex, b: Complex, tol: f64) { + let scale = b.abs().max(1.0); + assert!((a - b).abs() <= tol * scale, "{a} is not {b}"); +} + +#[test] +fn arithmetic() { + let (a, b) = (c(1.0, 2.0), c(3.0, -4.0)); + assert_eq!(a + b, c(4.0, -2.0)); + assert_eq!(a - b, c(-2.0, 6.0)); + assert_eq!(a * b, c(11.0, 2.0)); + assert_eq!(a / b, c(-0.2, 0.4)); + assert_eq!(-a, c(-1.0, -2.0)); + assert_eq!(a.conj(), c(1.0, -2.0)); + assert_eq!(b.abs(), 5.0); + assert_eq!(b.norm_sqr(), 25.0); + assert_eq!(c(0.0, 2.0).recip(), c(0.0, -0.5)); + // Owned and borrowed, floats on either side, and the assignments. + #[allow(clippy::op_ref)] + let sum = &a + &b; + assert_eq!(sum, a + b); + assert_eq!(a * 2.0, c(2.0, 4.0)); + assert_eq!(2.0 * a, c(2.0, 4.0)); + assert_eq!(1.0 - a, c(0.0, -2.0)); + assert_eq!(10.0 / c(3.0, 4.0), c(1.2, -1.6)); + assert_eq!(a / 2.0, c(0.5, 1.0)); + assert_eq!(a + 1.0, c(2.0, 2.0)); + let mut d = a; + d += b; + d -= &b; + d *= 3.0; + d /= c(3.0, 0.0); + assert_eq!(d, a); + // `0·∞` stays out of a real factor's product. + let inf = c(f64::INFINITY, 0.0); + assert_eq!(2.0 * inf, c(f64::INFINITY, 0.0)); + assert_eq!( + [a, b, c(0.0, 1.0)].iter().sum::>(), + c(4.0, -1.0) + ); + assert_eq!([a, b].into_iter().product::>(), a * b); + assert_eq!(Complex::::I * Complex::I, -Complex::ONE); + assert_eq!(Complex::from(2.5), c(2.5, 0.0)); + assert_eq!(Complex::::default(), Complex::ZERO); +} + +#[test] +fn extreme_magnitudes() { + // The squares would overflow: `abs` and Smith's division don't. + let big = c(1e300, 1e300); + assert_eq!(big / big, c(1.0, 0.0)); + assert!((big.abs() - 1e300 * 2f64.sqrt()).abs() <= 1e285); + let tiny = c(1e-300, -1e-300); + near(tiny / c(1e-300, 0.0), c(1.0, -1.0), 1e-15); + near(big.sqrt() * big.sqrt(), big, 1e-15); + let sub = c(5e-324, 5e-324); + let r = sub.sqrt(); + assert!(r.re > 0.0 && r.im > 0.0); + // mpmath at 40 digits. + near( + c(f64::MAX, f64::MAX).sqrt(), + c(1.473_094_556_905_565_2e154, 6.101_757_441_282_702_4e153), + 1e-15, + ); + assert_eq!(c(1.0, 1.0) / c(0.0, 0.0), c(f64::INFINITY, f64::INFINITY)); +} + +#[test] +fn the_top_and_the_bottom_of_the_range() { + // Quotients whose sums would overflow, of either operand, and both. + let top = c(1e308, 1e308); + assert_eq!(top / top, Complex::ONE); + assert_eq!(top / c(1.0, 1.0), c(1e308, 0.0)); + assert_eq!(c(1.0, 1.0) / top, c(1e-308, 0.0)); + let max = c(f64::MAX, -f64::MAX); + assert_eq!(max / max, Complex::ONE); + // Its reciprocal is subnormal, a few bits short. + near(top.recip() * top, Complex::ONE, 1e-14); + // A zero divisor and non-finite operands as before. + assert!((top / Complex::ZERO).re.is_infinite()); + assert!((Complex::ZERO / Complex::::ZERO).re.is_nan()); + assert_eq!(c(1.0, 0.0) / c(f64::INFINITY, 0.0), Complex::ZERO); + // Logarithms whose magnitude overflows, or is subnormal: mpmath at 40 digits. + let ln = c(f64::MAX, f64::MAX).ln(); + assert!((ln.re - 710.129_286_483_663_9).abs() <= 1e-13, "{ln}"); + assert!((ln.im - core::f64::consts::FRAC_PI_4).abs() <= 1e-15); + let ln = c(3e-320, 4e-320).ln(); + assert!((ln.re + 735.217_802_978_539_8).abs() <= 1e-12, "{ln}"); + let ln32 = Complex::new(f32::MAX, f32::MAX).ln(); + assert!((ln32.re - 89.069_41).abs() <= 1e-5, "{ln32}"); + // Zero, infinities and NaNs as before. + assert_eq!(Complex::::ZERO.ln().re, f64::NEG_INFINITY); + assert_eq!(c(f64::INFINITY, f64::NAN).ln().re, f64::INFINITY); + assert_eq!(c(f64::NAN, f64::INFINITY).ln().re, f64::INFINITY); + assert!(c(f64::NAN, 1.0).ln().re.is_nan()); + // An exponential whose `e^re` overflows but whose parts don't: mpmath at 40 digits. + let e = c(710.0, core::f64::consts::FRAC_PI_4).exp(); + near( + e, + c(1.579_672_848_288_201_5e308, 1.579_672_848_288_201_5e308), + 1e-15, + ); + assert!(c(710.0, 0.0).exp().re.is_infinite()); + // Quotients whose intermediate products underflow: of subnormal operands, and where + // `q/p` underflows. + let u = f64::from_bits(1); + near(c(u, u) / c(u, 2.0 * u), c(0.6, -0.2), 1e-15); + let q = c(1e300, 0.0) / c(1e100, 1e-250); + assert!( + (q.re - 1e200).abs() <= 1e185 && (q.im + 1e-150).abs() <= 1e-164, + "{q:?}" + ); + near(c(3e-310, 4e-310) / c(1e-310, 0.0), c(3.0, 4.0), 1e-12); +} + +#[test] +fn square_roots() { + assert_eq!(c(3.0, 4.0).sqrt(), c(2.0, 1.0)); + assert_eq!(c(-4.0, 0.0).sqrt(), c(0.0, 2.0)); + // The sign of a zero imaginary part picks the side of the cut. + assert_eq!(c(-4.0, -0.0).sqrt(), c(0.0, -2.0)); + assert_eq!(c(0.0, 0.0).sqrt(), c(0.0, 0.0)); + assert!(c(0.0, -0.0).sqrt().im.is_sign_negative()); + assert_eq!(c(0.0, 2.0).sqrt(), c(1.0, 1.0)); + // C99's special values. + assert_eq!( + c(1.0, f64::INFINITY).sqrt(), + c(f64::INFINITY, f64::INFINITY) + ); + assert_eq!(c(f64::NEG_INFINITY, 1.0).sqrt(), c(0.0, f64::INFINITY)); + assert_eq!(c(f64::INFINITY, -1.0).sqrt(), c(f64::INFINITY, -0.0)); + assert!(c(f64::NAN, 1.0).sqrt().is_nan()); + for (re, im) in [(0.3, -7.0), (-2.5, 1e-8), (1e10, 3.0), (-1e-10, -1e-10)] { + let z = c(re, im); + let r = z.sqrt(); + assert!(r.re >= 0.0); + near(r * r, z, 4e-16); + } +} + +#[test] +fn exponentials_and_logarithms() { + near(c(0.0, PI).exp(), c(-1.0, 0.0), 1e-15); + assert_eq!(c(2.0, 0.0).exp(), c(2f64.exp(), 0.0)); + assert!(c(1.0, -0.0).exp().im.is_sign_negative()); + assert_eq!(c(-1.0, 0.0).ln(), c(0.0, PI)); + assert_eq!(c(-1.0, -0.0).ln(), c(0.0, -PI)); + assert_eq!(c(0.0, 1.0).ln(), c(0.0, FRAC_PI_2)); + assert_eq!(c(0.0, 0.0).ln().re, f64::NEG_INFINITY); + // Near the unit circle the real part keeps its digits: ln|1 + 1e-10·i| = 5e-21. + let l = c(1.0, 1e-10).ln(); + assert!((l.re - 5e-21).abs() <= 1e-35, "{}", l.re); + for z in [c(0.5, -3.0), c(-7.0, 0.25), c(1e-5, 1e5), c(2.0, 0.0)] { + near(z.ln().exp(), z, 4e-16); + near(z.exp().ln(), z.exp().ln().exp().ln(), 4e-16); + } + assert_eq!(c(1.0, 0.0).arg(), 0.0); + assert_eq!(c(-1.0, -0.0).arg(), -PI); + let (r, theta) = c(-3.0, 4.0).to_polar(); + near(Complex::from_polar(r, theta), c(-3.0, 4.0), 1e-15); +} + +#[test] +fn powers() { + assert_eq!(c(1.0, 1.0).powi(8), c(16.0, 0.0)); + assert_eq!(c(0.0, 2.0).powi(-2), c(-0.25, 0.0)); + assert_eq!(c(5.0, 3.0).powi(0), Complex::ONE); + assert_eq!(c(0.0, 0.0).powi(0), Complex::ONE); + near(c(-8.0, 0.0).powf(1.0 / 3.0), c(1.0, 3f64.sqrt()), 1e-15); + assert_eq!(c(0.0, 0.0).powf(2.0), Complex::ZERO); + assert_eq!(c(0.0, 0.0).powf(0.0), Complex::ONE); + // `i^i = e^(-π/2)`. + near( + Complex::I.powc(Complex::I), + c((-FRAC_PI_2).exp(), 0.0), + 1e-15, + ); + assert_eq!(c(0.0, 0.0).powc(c(1.0, 1.0)), Complex::ZERO); + assert_eq!(c(0.0, 0.0).powc(Complex::ZERO), Complex::ONE); +} + +#[test] +fn single_precision() { + let z = Complex::new(3.0f32, 4.0); + assert_eq!(z.abs(), 5.0); + assert_eq!(z.sqrt(), Complex::new(2.0, 1.0)); + assert_eq!(2.0f32 * z, Complex::new(6.0, 8.0)); + let w = Complex::new(0.5f32, -1.5).ln().exp(); + assert!((w - Complex::new(0.5, -1.5)).abs() < 1e-6); + assert_eq!(::EPSILON, f32::EPSILON); +} + +#[test] +fn formatting() { + assert_eq!(format!("{}", c(1.5, -2.0)), "1.5-2i"); + assert_eq!(format!("{}", c(0.0, 0.0)), "0+0i"); + assert_eq!(format!("{}", c(0.0, -0.0)), "0-0i"); + assert_eq!(format!("{:.2}", c(1.0, 1.0 / 3.0)), "1.00+0.33i"); + assert_eq!(format!("{:?}", c(1.0, 2.0)), "Complex { re: 1.0, im: 2.0 }"); +} diff --git a/lib/tests/math/main.rs b/lib/tests/math/main.rs index 20d1e2e..ccdd77b 100644 --- a/lib/tests/math/main.rs +++ b/lib/tests/math/main.rs @@ -1,3 +1,5 @@ mod allocations; +mod complex; mod ec; +mod matrix; mod numbers; diff --git a/lib/tests/math/matrix/basic.rs b/lib/tests/math/matrix/basic.rs new file mode 100644 index 0000000..2870bbd --- /dev/null +++ b/lib/tests/math/matrix/basic.rs @@ -0,0 +1,157 @@ +//! Constructors, access, arithmetic elementwise, transposes, row and column operations, traces, +//! powers and formatting. + +use a::math::complex::Complex; +use a::math::matrix::Matrix; + +fn m(rows: &[[f64; 3]]) -> Matrix { + Matrix::from_rows(rows) +} + +#[test] +fn constructors_and_access() { + let a = m(&[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + assert_eq!((a.rows(), a.cols(), a.shape()), (2, 3, (2, 3))); + assert!(!a.is_square() && !a.is_empty()); + // Held column after column. + assert_eq!(a.as_slice(), [1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); + assert_eq!(a.column(1), [2.0, 5.0]); + assert_eq!(a.row(1).copied().collect::>(), [4.0, 5.0, 6.0]); + assert_eq!( + (a[(0, 2)], a.get(1, 0), a.get(2, 0)), + (3.0, Some(&4.0), None) + ); + assert_eq!( + Matrix::from_columns(&[[1.0, 4.0], [2.0, 5.0], [3.0, 6.0]]), + a + ); + assert_eq!( + Matrix::from_row_major(2, 3, vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]), + a + ); + assert_eq!(Matrix::from_column_major(2, 3, a.as_slice().to_vec()), a); + assert_eq!(Matrix::from_fn(2, 3, |i, j| (3 * i + j + 1) as f64), a); + assert_eq!(a.diagonal(), [1.0, 5.0]); + assert_eq!(a.map(|x| x * 2.0)[(1, 2)], 12.0); + let mut b = a.clone(); + b[(1, 1)] = -1.0; + *b.get_mut(0, 0).unwrap() = 7.0; + b.column_mut(2)[0] = 9.0; + assert_eq!(b.into_vec(), [7.0, 4.0, 2.0, -1.0, 9.0, 6.0]); + assert_eq!( + Matrix::::identity(2), + Matrix::from_rows(&[[1.0, 0.0], [0.0, 1.0]]) + ); + assert_eq!(Matrix::::zeros(1, 2), Matrix::from_rows(&[[0.0, 0.0]])); + assert_eq!( + Matrix::from_diagonal(&[2.0, 3.0]), + Matrix::from_rows(&[[2.0, 0.0], [0.0, 3.0]]) + ); + let empty: Matrix = Matrix::from_rows::<[f64; 0]>(&[]); + assert_eq!((empty.shape(), empty.is_empty()), ((0, 0), true)); + assert_eq!(Matrix::::default(), empty); +} + +#[test] +fn arithmetic() { + let a = m(&[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + let b = m(&[[0.5, -1.0, 2.0], [1.0, 0.0, -3.0]]); + let sum = m(&[[1.5, 1.0, 5.0], [5.0, 5.0, 3.0]]); + let difference = m(&[[0.5, 3.0, 1.0], [3.0, 5.0, 9.0]]); + assert_eq!(&a + &b, sum); + assert_eq!(a.clone() + &b, sum); + assert_eq!(&a + b.clone(), sum); + assert_eq!(a.clone() + b.clone(), sum); + assert_eq!(&a - &b, difference); + assert_eq!(&a - b.clone(), difference); + let mut c = a.clone(); + c += &b; + c -= b.clone(); + assert_eq!(c, a); + assert_eq!(-&a, a.map(|x| -x)); + assert_eq!(&a * 2.0, a.map(|x| 2.0 * x)); + assert_eq!(2.0 * &a, a.map(|x| 2.0 * x)); + assert_eq!(&a / 2.0, a.map(|x| x / 2.0)); + let mut d = a.clone(); + d *= 3.0; + d /= 3.0; + assert_eq!(d, a); + assert_eq!(a.hadamard(&b), m(&[[0.5, -2.0, 6.0], [4.0, 0.0, -18.0]])); + let p = &a * &b.transpose(); + assert_eq!(p, Matrix::from_rows(&[[4.5, -8.0], [9.0, -14.0]])); + let mut q = a.clone(); + q *= &b.transpose(); + assert_eq!(q, p); +} + +#[test] +fn transposes_and_row_operations() { + let z = |re, im| Complex::new(re, im); + let a = Matrix::from_rows(&[[z(1.0, 2.0), z(3.0, -1.0)], [z(0.0, 1.0), z(5.0, 0.0)]]); + assert_eq!(a.transpose()[(0, 1)], z(0.0, 1.0)); + assert_eq!(a.conjugate_transpose()[(0, 1)], z(0.0, -1.0)); + assert_eq!(a.conjugate_transpose()[(0, 0)], z(1.0, -2.0)); + let r = m(&[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]); + assert_eq!(r.transpose().transpose(), r); + assert_eq!(r.conjugate_transpose(), r.transpose()); + let mut s = r.clone(); + s.swap_rows(0, 1); + assert_eq!(s, m(&[[4.0, 5.0, 6.0], [1.0, 2.0, 3.0]])); + s.scale_row(1, &2.0); + assert_eq!(s, m(&[[4.0, 5.0, 6.0], [2.0, 4.0, 6.0]])); + s.add_scaled_row(0, 1, &-2.0); + assert_eq!(s, m(&[[0.0, -3.0, -6.0], [2.0, 4.0, 6.0]])); + let mut c = r.clone(); + c.swap_columns(2, 0); + assert_eq!(c, m(&[[3.0, 2.0, 1.0], [6.0, 5.0, 4.0]])); + c.scale_column(1, &0.5); + c.add_scaled_column(2, 0, &-1.0); + assert_eq!(c, m(&[[3.0, 1.0, -2.0], [6.0, 2.5, -2.0]])); + let sq = Matrix::from_rows(&[[1.0, 1.0], [1.0, 0.0]]); + assert_eq!(sq.trace(), 1.0); + // Fibonacci's numbers. + assert_eq!(sq.pow(10), Matrix::from_rows(&[[89.0, 55.0], [55.0, 34.0]])); + assert_eq!(sq.pow(0), Matrix::identity(2)); +} + +#[test] +fn formatting() { + let a = Matrix::from_rows(&[[1.0, -2.5], [10.0, 0.25]]); + assert_eq!(format!("{a}"), " 1 -2.5\n10 0.25"); + assert_eq!(format!("{a:.1}"), " 1.0 -2.5\n10.0 0.2"); + assert_eq!(format!("{a:?}"), "[[1.0, -2.5], [10.0, 0.25]]"); + let c = Matrix::from_rows(&[[Complex::new(1.0, -1.0)]]); + assert_eq!(format!("{c}"), "1-1i"); +} + +#[test] +#[should_panic(expected = "the matrices are not of the same shape")] +fn adding_different_shapes_panics() { + let _ = &Matrix::::zeros(2, 2) + &Matrix::zeros(2, 3); +} + +#[test] +#[should_panic( + expected = "the columns of the left matrix are not as many as the rows of the right one" +)] +fn multiplying_mismatched_shapes_panics() { + let _ = &Matrix::::zeros(2, 3) * &Matrix::zeros(2, 3); +} + +#[test] +#[should_panic(expected = "the rows are not all of the same length")] +fn ragged_rows_panic() { + let _ = Matrix::from_rows(&[vec![1.0, 2.0], vec![3.0]]); +} + +#[test] +#[should_panic(expected = "the index is outside the matrix")] +fn indexing_outside_panics() { + let _ = Matrix::::zeros(2, 2)[(0, 2)]; +} + +#[test] +#[should_panic(expected = "the matrix is not square")] +fn trace_of_a_rectangle_panics() { + let _ = Matrix::::zeros(2, 3).trace(); +} diff --git a/lib/tests/math/matrix/common.rs b/lib/tests/math/matrix/common.rs new file mode 100644 index 0000000..2a684df --- /dev/null +++ b/lib/tests/math/matrix/common.rs @@ -0,0 +1,188 @@ +//! What the tests of matrices share: the element types under test, random matrices, and +//! comparisons up to rounding. + +use a::math::complex::Complex; +use a::math::matrix::{Matrix, Scalar}; + +/// Scales a test down when it runs under Miri. +pub(super) fn size(normal: usize, miri: usize) -> usize { + if cfg!(miri) { miri } else { normal } +} + +/// Small deterministic generator. +pub(super) struct Rng(pub(super) u64); + +impl Rng { + /// The next 64 bits. + pub(super) fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A uniform float in `[-1, 1)`. + pub(super) fn unit(&mut self) -> f64 { + (self.next() >> 11) as f64 / (1u64 << 52) as f64 - 1.0 + } +} + +/// An element type under test, seen through `Complex`. +pub(super) trait Num: Scalar + Copy + core::fmt::Debug { + /// The unit roundoff of the type. + const EPS: f64; + /// A random element, its parts uniform in `[-1, 1)`. + fn random(r: &mut Rng) -> Self; + /// The real number `x`. + fn real(x: f64) -> Self; + /// The element, exactly, as a `Complex`. + fn c64(self) -> Complex; +} + +impl Num for f32 { + const EPS: f64 = f32::EPSILON as f64; + fn random(r: &mut Rng) -> Self { + r.unit() as f32 + } + fn real(x: f64) -> Self { + x as f32 + } + fn c64(self) -> Complex { + Complex::new(self as f64, 0.0) + } +} + +impl Num for f64 { + const EPS: f64 = f64::EPSILON; + fn random(r: &mut Rng) -> Self { + r.unit() + } + fn real(x: f64) -> Self { + x + } + fn c64(self) -> Complex { + Complex::new(self, 0.0) + } +} + +impl Num for Complex { + const EPS: f64 = f32::EPSILON as f64; + fn random(r: &mut Rng) -> Self { + Complex::new(r.unit() as f32, r.unit() as f32) + } + fn real(x: f64) -> Self { + Complex::new(x as f32, 0.0) + } + fn c64(self) -> Complex { + Complex::new(self.re as f64, self.im as f64) + } +} + +impl Num for Complex { + const EPS: f64 = f64::EPSILON; + fn random(r: &mut Rng) -> Self { + Complex::new(r.unit(), r.unit()) + } + fn real(x: f64) -> Self { + Complex::new(x, 0.0) + } + fn c64(self) -> Complex { + self + } +} + +/// Runs `$f::()` for each element type. +macro_rules! each_type { + ($f:ident) => { + $f::(); + $f::(); + $f::>(); + $f::>(); + }; +} +pub(super) use each_type; + +/// A module `$name` of four tests, `$f::()` for each element type: for tests too large to +/// run all four types in one process under Miri on a 32-bit target, which never reuses stack +/// addresses and runs out of them. +macro_rules! test_each_type { + ($name:ident, $f:ident) => { + mod $name { + #[test] + fn f32() { + super::$f::(); + } + #[test] + fn f64() { + super::$f::(); + } + #[test] + fn complex_f32() { + super::$f::>(); + } + #[test] + fn complex_f64() { + super::$f::>(); + } + } + }; +} +pub(super) use test_each_type; + +/// A random `m × n` matrix. +pub(super) fn random(r: &mut Rng, m: usize, n: usize) -> Matrix { + Matrix::from_fn(m, n, |_, _| T::random(r)) +} + +/// A random matrix that is far from singular: random plus `n` on the diagonal. +pub(super) fn well_conditioned(r: &mut Rng, n: usize) -> Matrix { + let mut a = random::(r, n, n); + for i in 0..n { + a[(i, i)] += T::real(n as f64); + } + a +} + +/// A random Hermitian positive definite matrix: `B·B* + n·I`. +pub(super) fn positive_definite(r: &mut Rng, n: usize) -> Matrix { + let b = random::(r, n, n); + let mut a = &b * &b.conjugate_transpose(); + for i in 0..n { + a[(i, i)] += T::real(n as f64); + } + a +} + +/// The Frobenius norm, in `f64`. +pub(super) fn frobenius(a: &Matrix) -> f64 { + a.as_slice() + .iter() + .map(|x| x.c64().norm_sqr()) + .sum::() + .sqrt() +} + +/// Asserts `‖a - b‖ ≤ tol·ε·(‖b‖ + 1)` in the Frobenius norm, `ε` the unit roundoff of the type. +#[track_caller] +pub(super) fn assert_close(a: &Matrix, b: &Matrix, tol: f64) { + assert_eq!(a.shape(), b.shape()); + let d = frobenius(&(a - b)); + let bound = tol * T::EPS * (frobenius(b) + 1.0); + assert!( + d <= bound, + "‖a - b‖ = {d:e} > {bound:e} for {:?} of {}", + a.shape(), + core::any::type_name::() + ); +} + +/// Asserts that the columns of `q` are orthonormal: `Q*·Q = I`. +#[track_caller] +pub(super) fn assert_orthonormal(q: &Matrix, tol: f64) { + let n = q.cols(); + assert_close( + &(&q.conjugate_transpose() * q), + &Matrix::identity(n), + tol * n as f64, + ); +} diff --git a/lib/tests/math/matrix/eigen.rs b/lib/tests/math/matrix/eigen.rs new file mode 100644 index 0000000..0918479 --- /dev/null +++ b/lib/tests/math/matrix/eigen.rs @@ -0,0 +1,253 @@ +//! Schur decompositions and eigenvalues, checked by what defines them. + +use super::common::{Num, Rng, assert_close, assert_orthonormal, each_type, random}; +use a::math::complex::Complex; +use a::math::matrix::Matrix; + +/// Sizes around the exceptional shifts' period and a few larger ones. +fn sizes() -> Vec { + if cfg!(miri) { + vec![0, 1, 2, 3, 6] + } else { + vec![0, 1, 2, 3, 4, 5, 8, 11, 20, 33, 64, 100] + } +} + +/// Checks the Schur decomposition of `a`. +fn check_schur(a: &Matrix) { + let n = a.rows(); + let s = a.schur(); + let (q, t) = (s.q(), s.t()); + let tol = 32.0 * (n + 1) as f64; + assert_orthonormal(&q, tol); + assert_close(&(&(&q * &t) * &q.conjugate_transpose()), a, tol); + let zero = T::real(0.0); + let complex = core::any::type_name::().contains("Complex"); + for j in 0..n { + for i in j + 1..n { + if complex || i > j + 1 { + assert_eq!(t[(i, j)], zero, "{i} {j}"); + } + } + } + let values = s.eigenvalues(); + assert_eq!(values.len(), n); + let mut i = 0; + while i < n { + let d = t[(i, i)].c64(); + if !complex && i + 1 < n && t[(i + 1, i)] != zero { + // A standard 2 × 2 block: equal diagonal, `b·c < 0`, eigenvalues `a ± i√(-bc)`. + let (b, c) = (t[(i, i + 1)].c64().re, t[(i + 1, i)].c64().re); + assert_eq!(t[(i + 1, i + 1)].c64(), d); + assert!(b * c < 0.0); + let im = b.abs().sqrt() * c.abs().sqrt(); + let (w1, w2) = (to_c64::(values[i]), to_c64::(values[i + 1])); + assert!((w1 - Complex::new(d.re, im)).abs() <= 8.0 * T::EPS * (d.abs() + im)); + assert!((w2 - Complex::new(d.re, -im)).abs() <= 8.0 * T::EPS * (d.abs() + im)); + i += 2; + } else { + assert_eq!(to_c64::(values[i]), d); + i += 1; + } + } + // Their sum is the trace. + let sum: Complex = values.iter().map(|&w| to_c64::(w)).sum(); + let trace = a.trace().c64(); + let scale: f64 = + (0..n).map(|i| a[(i, i)].c64().abs()).sum::() + super::common::frobenius(a); + assert!((sum - trace).abs() <= tol * T::EPS * (scale + 1.0)); + // And the eigenvalues without `Q` are the same ones. + let mut alone: Vec> = a.eigenvalues().iter().map(|&w| to_c64::(w)).collect(); + let mut with: Vec> = values.iter().map(|&w| to_c64::(w)).collect(); + let key = + |w: &Complex, v: &Complex| w.re.total_cmp(&v.re).then(w.im.total_cmp(&v.im)); + alone.sort_by(key); + with.sort_by(key); + for (x, y) in alone.iter().zip(&with) { + assert!((x - y).abs() <= tol * T::EPS * (scale + 1.0), "{x} {y}"); + } +} + +/// An eigenvalue of the type's precision as a `Complex`. +fn to_c64(w: Complex) -> Complex { + use a::math::complex::Float; + Complex::new(w.re.to_f64(), w.im.to_f64()) +} + +fn schur_of() { + let mut r = Rng(0x5EED_0101); + for n in sizes() { + check_schur(&random::(&mut r, n, n)); + } + // Structured matrices: zero, diagonal, triangular, a Jordan block, a permutation. + for n in [1, 4, 9] { + check_schur(&Matrix::::zeros(n, n)); + check_schur(&Matrix::from_fn(n, n, |i, j| { + if i == j { + T::real(i as f64 - 2.0) + } else { + T::real(0.0) + } + })); + check_schur(&Matrix::from_fn(n, n, |i, j| { + if i <= j { + T::real((i + 2 * j) as f64) + } else { + T::real(0.0) + } + })); + check_schur(&Matrix::from_fn(n, n, |i, j| { + if i == j || i + 1 == j { + T::real(1.0) + } else { + T::real(0.0) + } + })); + check_schur(&Matrix::from_fn(n, n, |i, j| { + if (i + 1) % n == j { + T::real(1.0) + } else { + T::real(0.0) + } + })); + } +} + +#[test] +fn schur() { + each_type!(schur_of); +} + +#[test] +fn known_eigenvalues() { + // The companion matrix of (x - 1)(x - 2)…(x - 6): eigenvalues 1 to 6. + let coefficients = [720.0, -1764.0, 1624.0, -735.0, 175.0, -21.0]; + let c: Matrix = Matrix::from_fn(6, 6, |i, j| { + if j == 5 { + -coefficients[i] + } else if i == j + 1 { + 1.0 + } else { + 0.0 + } + }); + let mut w = c.eigenvalues(); + w.sort_by(|a, b| a.re.total_cmp(&b.re)); + for (k, w) in w.iter().enumerate() { + assert!( + (w.re - (k + 1) as f64).abs() < 1e-9 && w.im.abs() < 1e-9, + "{w}" + ); + } + // A rotation by θ: e^(±iθ). + let (s, c) = 0.3f64.sin_cos(); + let rot: Matrix = Matrix::from_rows(&[[c, -s], [s, c]]); + let mut w = rot.eigenvalues(); + w.sort_by(|a, b| a.im.total_cmp(&b.im)); + assert!((w[0] - Complex::new(c, -s)).abs() < 1e-15); + assert!((w[1] - Complex::new(c, s)).abs() < 1e-15); + // A complex triangular matrix's diagonal. + let t: Matrix> = Matrix::from_rows(&[ + [Complex::new(1.0, 2.0), Complex::new(3.0, 0.0)], + [Complex::new(0.0, 0.0), Complex::new(-1.0, 0.5)], + ]); + let mut w = t.eigenvalues(); + w.sort_by(|a, b| a.re.total_cmp(&b.re)); + assert_eq!(w, [Complex::new(-1.0, 0.5), Complex::new(1.0, 2.0)]); + // Not a number. + let nan: Matrix = Matrix::from_rows(&[[1.0, f64::NAN], [0.5, 2.0]]); + assert!(nan.eigenvalues().iter().all(|w| w.is_nan())); +} + +/// The matrix in `Complex`. +fn c64(a: &Matrix) -> Matrix> { + a.map(|x| x.c64()) +} + +/// A complex matrix of the type's precision in `Complex`. +fn widen(a: &Matrix>) -> Matrix> { + a.map(|&w| to_c64::(w)) +} + +fn eigen_of() { + let mut r = Rng(0x5EED_0102); + for n in sizes() { + let a = random::(&mut r, n, n); + let e = a.eigen(); + let (v, w) = (widen::(e.vectors()), e.values()); + let ac = c64(&a); + let scale = super::common::frobenius(&a) + 1.0; + for (j, &wj) in w.iter().enumerate() { + let x = Matrix::from_columns(&[v.column(j)]); + let norm: f64 = x + .as_slice() + .iter() + .map(|z| z.norm_sqr()) + .sum::() + .sqrt(); + assert!( + (norm - 1.0).abs() <= 8.0 * (n + 1) as f64 * T::EPS, + "{n} {j}" + ); + let d = &(&ac * &x) - &(&x * to_c64::(wj)); + let d: f64 = d + .as_slice() + .iter() + .map(|z| z.norm_sqr()) + .sum::() + .sqrt(); + assert!( + d <= 64.0 * (n + 1) as f64 * T::EPS * scale, + "{n} {j}: {d:e}" + ); + } + } +} + +#[test] +fn eigen() { + each_type!(eigen_of); +} + +fn hermitian_of() { + let mut r = Rng(0x5EED_0103); + for n in sizes() { + let b = random::(&mut r, n, n); + let a = &b + &b.conjugate_transpose(); + let e = a.hermitian_eigen(); + let (v, w) = (e.vectors(), e.values()); + let tol = 32.0 * (n + 1) as f64; + assert_orthonormal(v, tol); + let lambda = Matrix::from_diagonal(&w.iter().map(|&x| T::real(f(x))).collect::>()); + assert_close(&(&(v * &lambda) * &v.conjugate_transpose()), &a, tol); + assert!(w.windows(2).all(|p| p[0] <= p[1])); + let alone: Vec = a.hermitian_eigenvalues().into_iter().map(f).collect(); + let scale = super::common::frobenius(&a) + 1.0; + for (x, y) in alone.iter().zip(w) { + assert!((x - f(*y)).abs() <= tol * T::EPS * scale); + } + // The general algorithm's eigenvalues: real, and the same. + let mut general: Vec> = + a.eigenvalues().iter().map(|&z| to_c64::(z)).collect(); + general.sort_by(|x, y| x.re.total_cmp(&y.re)); + for (g, h) in general.iter().zip(w) { + assert!((g - Complex::new(f(*h), 0.0)).abs() <= 4.0 * tol * T::EPS * scale); + } + // Only the lower triangle is read. + if n > 1 { + let mut c = a.clone(); + c[(0, n - 1)] = T::real(1e3); + assert_eq!(c.hermitian_eigenvalues(), a.hermitian_eigenvalues()); + } + } +} + +/// A real eigenvalue as an `f64`. +fn f(x: R) -> f64 { + x.to_f64() +} + +#[test] +fn hermitian() { + each_type!(hermitian_of); +} diff --git a/lib/tests/math/matrix/exact.rs b/lib/tests/math/matrix/exact.rs new file mode 100644 index 0000000..b3e2e5a --- /dev/null +++ b/lib/tests/math/matrix/exact.rs @@ -0,0 +1,224 @@ +//! Exact algorithms on matrices of numbers, against independent ones: Laplace's expansion, +//! evaluation of the characteristic polynomial, and the identities of adjugates and null spaces. + +use super::common::{Rng, size}; +use a::math::matrix::Matrix; +use a::math::numbers::{ByteOrder, Number}; + +/// A random matrix of numbers of up to `bytes` bytes, of either sign. +fn random(r: &mut Rng, m: usize, n: usize, bytes: usize) -> Matrix { + Matrix::from_fn(m, n, |_, _| { + let len = (r.next() % (bytes as u64 + 1)) as usize; + let x = Number::new( + (0..len).map(|_| r.next() as u8).collect(), + ByteOrder::LittleEndian, + ); + if r.next() & 1 == 1 { -x } else { x } + }) +} + +/// A small random matrix of numbers in `-k..=k`. +fn small(r: &mut Rng, m: usize, n: usize, k: i64) -> Matrix { + Matrix::from_fn(m, n, |_, _| { + Number::from((r.next() % (2 * k as u64 + 1)) as i64 - k) + }) +} + +/// The determinant by Laplace's expansion along the first row. +fn laplace(a: &Matrix) -> Number { + let n = a.rows(); + if n == 0 { + return Number::from(1); + } + let mut d = Number::default(); + for j in 0..n { + let minor = Matrix::from_fn(n - 1, n - 1, |r, c| { + a[(r + 1, if c < j { c } else { c + 1 })].clone() + }); + let term = &a[(0, j)] * &laplace(&minor); + if j % 2 == 0 { + d += term; + } else { + d -= term; + } + } + d +} + +#[test] +fn determinants() { + let mut r = Rng(0x5EED_0401); + for n in 0..size(7, 4) { + for bytes in [1, 9, 40] { + let a = random(&mut r, n, n, bytes); + assert_eq!(a.det(), laplace(&a), "{n} {bytes}"); + } + } + // `det(A·B) = det(A)·det(B)`, larger. + let n = size(24, 6); + let (a, b) = (random(&mut r, n, n, 8), random(&mut r, n, n, 8)); + assert_eq!((&a * &b).det(), &a.det() * &b.det()); + // Singular: a repeated row, and a zero column. + let mut s = random(&mut r, 5, 5, 4); + for j in 0..5 { + s[(3, j)] = s[(1, j)].clone(); + } + assert!(s.det().is_zero()); + let mut s = random(&mut r, 5, 5, 4); + for i in 0..5 { + s[(i, 2)] = Number::default(); + } + assert!(s.det().is_zero()); + // A row swap at the first pivot. + let p = Matrix::from_rows(&[[0, 1], [1, 0]]).map(|&x| Number::from(x)); + assert_eq!(p.det(), -1); +} + +/// `A = B·C` with `B` of `m × r` and `C` of `r × n`, of rank `r` for random factors. +fn of_rank(r: &mut Rng, m: usize, n: usize, rank: usize) -> Matrix { + &small(r, m, rank, 9) * &small(r, rank, n, 9) +} + +#[test] +fn ranks_and_null_spaces() { + let mut r = Rng(0x5EED_0402); + for (m, n, rank) in [ + (4, 4, 4), + (5, 5, 2), + (6, 3, 3), + (3, 7, 2), + (8, 9, 5), + (4, 4, 0), + ] { + let a = of_rank(&mut r, m, n, rank); + assert_eq!(a.rank(), rank, "{m} {n} {rank}"); + let null = a.null_space(); + assert_eq!(null.shape(), (n, n - rank)); + assert!((&a * &null).as_slice().iter().all(Number::is_zero)); + // Independent, and each column primitive. + assert_eq!(null.rank(), n - rank); + for j in 0..null.cols() { + let g = null + .column(j) + .iter() + .fold(Number::default(), |g, x| g.gcd(x)); + assert_eq!(g, 1); + } + } + let z = Matrix::::zeros(3, 4); + assert_eq!(z.rank(), 0); + assert_eq!(z.null_space(), Matrix::identity(4)); + assert_eq!(Matrix::::identity(3).null_space().shape(), (3, 0)); +} + +/// `p(x)` for coefficients from the highest power down. +fn evaluate(p: &[Number], x: &Number) -> Number { + p.iter().fold(Number::default(), |s, c| &(&s * x) + c) +} + +#[test] +fn characteristic_polynomials() { + let mut r = Rng(0x5EED_0403); + for n in 0..size(9, 4) { + let a = random(&mut r, n, n, 6); + let p = a.characteristic_polynomial(); + assert_eq!(p.len(), n + 1); + assert_eq!(p[0], 1); + // `p(x) = det(x·I - A)` at `n + 1` points determines it. + for x in 0..=n as i64 { + let x = Number::from(x * 7 - 3); + let m = &Matrix::::identity(n) * &x - &a; + assert_eq!(evaluate(&p, &x), m.det(), "{n}"); + } + // Cayley and Hamilton. + let mut c = Matrix::::zeros(n, n); + for coefficient in &p { + c = &c * &a; + for i in 0..n { + c[(i, i)] += coefficient; + } + } + assert!(c.as_slice().iter().all(Number::is_zero), "{n}"); + } +} + +#[test] +fn adjugates_and_inverses() { + let mut r = Rng(0x5EED_0404); + for n in 0..size(8, 4) { + let a = random(&mut r, n, n, 5); + let adj = a.adjugate(); + let d = a.det(); + let di = &Matrix::::identity(n) * &d; + assert_eq!(&a * &adj, di); + assert_eq!(&adj * &a, di); + if n <= 4 { + // The cofactors themselves. + for i in 0..n { + for j in 0..n { + let minor = Matrix::from_fn(n - 1, n - 1, |r, c| { + a[(if r < j { r } else { r + 1 }, if c < i { c } else { c + 1 })].clone() + }); + let c = laplace(&minor); + let c = if (i + j) % 2 == 0 { c } else { -c }; + assert_eq!(adj[(i, j)], c); + } + } + } + match a.inverse() { + Some((adj2, det)) => { + assert_eq!((adj2, det), (adj, d)); + } + None => assert!(d.is_zero()), + } + } + // Singular of rank n - 1: a nonzero adjugate with `A·adj(A) = 0`. + let a = of_rank(&mut r, 4, 4, 3); + assert!(a.inverse().is_none()); + let adj = a.adjugate(); + assert!(!adj.as_slice().iter().all(Number::is_zero)); + assert!((&a * &adj).as_slice().iter().all(Number::is_zero)); +} + +#[test] +fn generic_operations_on_numbers() { + let mut r = Rng(0x5EED_0405); + let (a, b) = (small(&mut r, 3, 4, 50), small(&mut r, 3, 4, 50)); + let as_i = |m: &Matrix| m.map(|x| i64::try_from(x).unwrap()); + let (ai, bi) = (as_i(&a), as_i(&b)); + assert_eq!( + as_i(&(&a + &b)), + Matrix::from_fn(3, 4, |i, j| ai[(i, j)] + bi[(i, j)]) + ); + assert_eq!( + as_i(&(&a - &b)), + Matrix::from_fn(3, 4, |i, j| ai[(i, j)] - bi[(i, j)]) + ); + assert_eq!( + as_i(&a.hadamard(&b)), + Matrix::from_fn(3, 4, |i, j| ai[(i, j)] * bi[(i, j)]) + ); + let product = &a * &b.transpose(); + let want = Matrix::from_fn(3, 3, |i, j| { + (0..4).map(|k| ai[(i, k)] * bi[(j, k)]).sum::() + }); + assert_eq!(as_i(&product), want); + let three = Number::from(3); + assert_eq!( + as_i(&(&three * &a)), + Matrix::from_fn(3, 4, |i, j| 3 * ai[(i, j)]) + ); + let s = small(&mut r, 3, 3, 3); + let si = as_i(&s); + let mut p = Matrix::::from_fn(3, 3, |i, j| i64::from(i == j)); + for k in 0..=6 { + assert_eq!(as_i(&s.pow(k)), p); + p = Matrix::from_fn(3, 3, |i, j| (0..3).map(|q| p[(i, q)] * si[(q, j)]).sum()); + } + assert_eq!(s.trace(), si[(0, 0)] + si[(1, 1)] + si[(2, 2)]); + // Numbers of either byte order mix. + let big = Matrix::from_fn(2, 2, |i, j| { + Number::new_from_u64((i * 2 + j + 1) as u64, ByteOrder::BigEndian) + }); + assert_eq!(big.det(), -2); +} diff --git a/lib/tests/math/matrix/extremes.rs b/lib/tests/math/matrix/extremes.rs new file mode 100644 index 0000000..102f801 --- /dev/null +++ b/lib/tests/math/matrix/extremes.rs @@ -0,0 +1,244 @@ +//! Matrices at the ends of the floats' range: norms of subnormal elements, reflections near the +//! largest float, eigenvalues of matrices scaled up and down, complex elements with finite parts +//! whose absolute values overflow, and exponentials whose norms overflow. + +use super::common::{Num, Rng, assert_orthonormal, each_type, random}; +use a::math::complex::{Complex, Float}; +use a::math::matrix::{Matrix, Norm}; + +#[test] +fn norms_of_subnormal_elements() { + let tiny = 1e-310; + assert_eq!(Matrix::from_rows(&[[tiny]]).norm(Norm::Frobenius), tiny); + let a = Matrix::from_rows(&[[tiny, 0.0], [0.0, -tiny]]); + let want = tiny * 2f64.sqrt(); + let ulp = f64::from_bits(1); + assert!((a.norm(Norm::Frobenius) - want).abs() <= 2.0 * ulp); + let c = Matrix::from_rows(&[[Complex::new(tiny, tiny)], [Complex::ZERO]]); + assert!((c.norm(Norm::Frobenius) - want).abs() <= 2.0 * ulp); +} + +/// The largest power of two whose multiples of elements below 1, a few times over, stay finite. +fn top() -> f64 { + if T::EPS > 1e-10 { + 2f64.powi(126) + } else { + 2f64.powi(1022) + } +} + +fn near_the_top_of() { + // `alpha = s` and `x = [s/10]`, `1e308` and `1e307` in double precision: the norm is finite + // but `alpha - β`, about `2s`, is not. + let s = if T::EPS > 1e-10 { 3e38 } else { 1e308 }; + let a = Matrix::from_rows(&[[T::real(s)], [T::real(s / 10.0)]]); + let qr = a.qr(); + let (q, r) = (qr.q(), qr.r()); + assert!(q.as_slice().iter().all(|x| x.c64().is_finite())); + let norm = s.hypot(s / 10.0); + assert!((r[(0, 0)].c64().abs() - norm).abs() <= 8.0 * T::EPS * norm); + assert_orthonormal(&q, 16.0); + let back = &q * &r; + for (x, y) in back.as_slice().iter().zip(a.as_slice()) { + assert!((x.c64() - y.c64()).abs() <= 16.0 * T::EPS * norm); + } +} + +#[test] +fn reflections_near_the_largest_float() { + each_type!(near_the_top_of); +} + +/// The eigenvalues of `b`, each matched with the nearest of `got`, scaled by `1/s`. +#[track_caller] +fn assert_scaled(b: &Matrix, got: &[Complex], s: f64) { + let want = b.eigenvalues(); + let scale = want.iter().fold(1.0f64, |m, w| m.max(c64::(*w).abs())); + for w in &want { + let w = c64::(*w); + let d = got + .iter() + .map(|g| (c64::(*g) * (1.0 / s) - w).abs()) + .fold(f64::INFINITY, f64::min); + assert!(d <= 1e3 * T::EPS * scale, "{w}: {d:e}"); + } +} + +/// A complex number of the element's real type as a `Complex`. +fn c64(z: Complex) -> Complex { + Complex::new(z.re.to_f64(), z.im.to_f64()) +} + +fn eigenvalues_scaled_of() { + // `s·B` for `B` with elements in `[-1, 1)`: near the top, its reductions' differences would + // overflow, and near the bottom their products would underflow, unless the matrix is scaled + // first. The eigenvalues are `s` times `B`'s. + let mut r = Rng(0x5EED_0901); + let n = if cfg!(miri) { 4 } else { 12 }; + let bottom = if T::EPS > 1e-10 { + 2f64.powi(-120) + } else { + 2f64.powi(-1000) + }; + for s in [top::() / 4.0, bottom] { + let b = random::(&mut r, n, n); + let a = b.map(|&x| x * T::real(s)); + let schur = a.schur(); + assert_orthonormal(&schur.q(), 32.0 * n as f64); + assert!(schur.t().as_slice().iter().all(|x| x.c64().is_finite())); + assert_scaled(&b, schur.eigenvalues(), s); + assert_scaled(&b, &a.eigenvalues(), s); + let e = a.eigen(); + assert!( + e.vectors() + .as_slice() + .iter() + .all(|x| c64::(*x).is_finite()) + ); + } +} + +#[test] +fn eigenvalues_near_the_ends_of_the_range() { + each_type!(eigenvalues_scaled_of); +} + +#[test] +fn complex_elements_whose_absolute_values_overflow() { + // `|z| = √2·MAX` overflows although both parts are finite: the scaling of the singular + // values and the Hermitian eigenvalues takes the largest part for it. + let z = Complex::new(f64::MAX, f64::MAX); + let one = Complex::::ONE; + let mut a = Matrix::>::zeros(3, 3); + (a[(0, 0)], a[(1, 1)], a[(2, 2)]) = (z, one, Complex::new(0.5, 0.0)); + let s = a.singular_values(); + assert_eq!(s[0], f64::INFINITY); + assert!((s[1] - 1.0).abs() <= 4.0 * f64::EPSILON && (s[2] - 0.5).abs() <= 4.0 * f64::EPSILON); + let svd = a.svd(); + assert_eq!(svd.singular_values()[0], f64::INFINITY); + assert!(svd.singular_values()[1..].iter().all(|x| x.is_finite())); + assert!(svd.u().as_slice().iter().all(|x| x.is_finite())); + assert!(svd.v().as_slice().iter().all(|x| x.is_finite())); + // `[[0, z*], [z, 0]]`, of eigenvalues `±|z|`, beside ones: on the small path and the blocked + // one. + for n in [3, if cfg!(miri) { 3 } else { 200 }] { + let mut h = Matrix::>::identity(n); + (h[(0, 0)], h[(1, 1)], h[(1, 0)], h[(0, 1)]) = (Complex::ZERO, Complex::ZERO, z, z.conj()); + let w = h.hermitian_eigenvalues(); + assert_eq!((w[0], w[n - 1]), (f64::NEG_INFINITY, f64::INFINITY), "{n}"); + assert!(w[1..n - 1].iter().all(|x| x.is_finite()), "{n}"); + let e = h.hermitian_eigen(); + assert_eq!(e.values()[0], f64::NEG_INFINITY, "{n}"); + assert!(!e.values().iter().any(|x| x.is_nan()), "{n}"); + } +} + +fn exp_of_an_overflowing_norm() { + // `A = -c·(I + J)` with `J` all ones, its elements finite and its 1-norm `5c` not: its + // eigenvalues `-c` and `-5c` make `e^A` zero. The number of squarings stays near the type's + // range rather than becoming `i32::MAX`, which never finished. + let c = if T::EPS > 1e-10 { 8e37 } else { 4e307 }; + let a = Matrix::from_fn(4, 4, |i, j| T::real(if i == j { -2.0 * c } else { -c })); + assert!(!a.norm(Norm::One).to_f64().is_finite()); + let e = a.exp(); + assert!(e.as_slice().iter().all(|x| x.c64().abs() <= 1e-30), "{e:?}"); +} + +#[test] +fn exponentials_of_norms_that_overflow() { + each_type!(exp_of_an_overflowing_norm); +} + +#[test] +fn schur_blocks_whose_products_overflow() { + // The rotation that turns the real Schur form's block into a complex triangle squared its + // elements, `-1e400` here: the square root and the logarithm of a rotation by `1e200`. + let a = Matrix::from_rows(&[[0.0, -1e200], [1e200, 0.0]]); + let r = a.sqrt().unwrap(); + assert!(r.as_slice().iter().all(|x| x.is_finite()), "{r:?}"); + let h = 1e100 * core::f64::consts::FRAC_1_SQRT_2; + let want = Matrix::from_rows(&[[h, -h], [h, h]]); + assert!((&r - &want).norm(Norm::Max) <= 1e-14 * h, "{r:?}"); + let l = a.log().unwrap(); + assert!(l.as_slice().iter().all(|x| x.is_finite()), "{l:?}"); + let ln = 460.517_018_598_809_16; // mpmath, ln 1e200 + let want = Matrix::from_rows(&[ + [ln, -core::f64::consts::FRAC_PI_2], + [core::f64::consts::FRAC_PI_2, ln], + ]); + assert!((&l - &want).norm(Norm::Max) <= 1e-12, "{l:?}"); +} + +#[test] +#[should_panic(expected = "more elements than a vector can hold")] +fn zeros_too_large_to_hold() { + let _ = Matrix::::zeros(1 << (usize::BITS - 1), 2); +} + +#[test] +#[should_panic(expected = "more elements than a vector can hold")] +fn from_fn_too_large_to_hold() { + let _ = Matrix::from_fn(usize::MAX, 2, |_, _| 0.0); +} + +#[test] +fn ranks_and_condition_numbers_beyond_the_range() { + // One element whose absolute value overflows: its singular value does too, but its rank, + // null space and condition number are those of any nonzero 1 × 1 matrix. + let m = Matrix::from_rows(&[[Complex::new(f64::MAX, f64::MAX)]]); + assert_eq!(m.rank(), 1); + assert_eq!(m.null_space().shape(), (1, 0)); + assert_eq!(m.cond(Norm::Two), 1.0); + // The nuclear norm's sums overflow, or their reciprocals do, while the ratio is 4. + for x in [1e308f64, 1e-310] { + let d = Matrix::from_diagonal(&[x, x]); + assert!((d.cond(Norm::Nuclear) - 4.0).abs() <= 1e-15, "{x}"); + assert!((d.cond(Norm::Two) - 1.0).abs() <= 1e-15, "{x}"); + } +} + +#[test] +fn norms_carry_nans() { + let nan = f64::NAN; + for a in [ + Matrix::from_rows(&[[nan]]), + Matrix::from_rows(&[[nan, 0.0], [0.0, 0.0]]), + Matrix::from_rows(&[[nan, 1.0], [2.0, 3.0]]), + ] { + for norm in [ + Norm::One, + Norm::Infinity, + Norm::Max, + Norm::Frobenius, + Norm::Two, + Norm::Nuclear, + ] { + assert!(a.norm(norm).is_nan(), "{norm:?} of {a:?}"); + } + assert!(a.cond(Norm::Two).is_nan(), "{a:?}"); + assert!(a.cond(Norm::Nuclear).is_nan(), "{a:?}"); + } + let c = Matrix::from_rows(&[[Complex::new(0.0, nan)], [Complex::ZERO]]); + assert!(c.norm(Norm::Frobenius).is_nan()); +} + +fn subnormal_reflection_of() { + // `(u, u)` for the smallest subnormal `u` of the type: its norm rounds to `u`, which made + // `τ = 2` and a `Q` of norm √2. + let u = if T::EPS > 1e-10 { + f64::from(f32::from_bits(1)) + } else { + f64::from_bits(1) + }; + let a = Matrix::from_rows(&[[T::real(u)], [T::real(u)]]); + let qr = a.qr(); + assert_orthonormal(&qr.q(), 4.0); + assert_orthonormal(&qr.q_full(), 4.0); + let r = qr.r()[(0, 0)].c64().abs(); + assert!(r > 0.0 && r <= 2.0 * u, "{r:e}"); +} + +#[test] +fn reflections_of_subnormal_columns() { + each_type!(subnormal_reflection_of); +} diff --git a/lib/tests/math/matrix/factor.rs b/lib/tests/math/matrix/factor.rs new file mode 100644 index 0000000..6947639 --- /dev/null +++ b/lib/tests/math/matrix/factor.rs @@ -0,0 +1,258 @@ +//! LU, Cholesky and QR decompositions, checked by what defines them. + +use super::common::{ + Num, Rng, assert_close, assert_orthonormal, each_type, frobenius, positive_definite, random, + size, well_conditioned, +}; +use a::math::complex::{Complex, Float}; +use a::math::matrix::Matrix; + +/// Sizes around the recursions' and the panels' widths. +fn sizes() -> Vec { + if cfg!(miri) { + vec![0, 1, 2, 9, 17] + } else { + vec![0, 1, 2, 3, 7, 8, 9, 16, 17, 31, 32, 33, 64, 65, 100] + } +} + +fn lu_of() { + let mut r = Rng(0x5EED_0001); + for n in sizes() { + for m in [n, n + 3, n / 2] { + let a = random::(&mut r, m, n); + let lu = a.lu(); + let (l, u) = (lu.l(), lu.u()); + let k = m.min(n); + assert_eq!((l.shape(), u.shape()), ((m, k), (k, n))); + for i in 0..m { + for j in 0..k { + if j > i { + assert_eq!(l[(i, j)], T::real(0.0)); + } + if i == j { + assert_eq!(l[(i, j)], T::real(1.0)); + } + // Partial pivoting by `|re| + |im|`: no multiplier above 1, or √2 for + // complex elements. + let bound = if core::any::type_name::().contains("Complex") { + core::f64::consts::SQRT_2 + } else { + 1.0 + }; + assert!(l[(i, j)].c64().abs() <= bound * (1.0 + 4.0 * T::EPS)); + } + } + for i in 0..k { + for j in 0..i { + assert_eq!(u[(i, j)], T::real(0.0)); + } + } + assert_close(&(&lu.p() * &a), &(&l * &u), 8.0 * (n + 1) as f64); + } + } +} + +#[test] +fn lu() { + each_type!(lu_of); +} + +fn solve_of() { + let mut r = Rng(0x5EED_0002); + for n in sizes() { + let a = well_conditioned::(&mut r, n); + let b = random::(&mut r, n, 3); + let x = a.solve(&b).unwrap(); + assert_close(&(&a * &x), &b, 16.0 * (n + 1) as f64); + let inv = a.inverse().unwrap(); + assert_close(&(&a * &inv), &Matrix::identity(n), 16.0 * (n + 1) as f64); + // `A⁻²·A² = I`. + let p = a.powi(-2).unwrap(); + assert_close( + &(&p * &a.pow(2)), + &Matrix::identity(n), + 64.0 * (n + 1) as f64, + ); + } +} + +#[test] +fn solve_and_inverse() { + each_type!(solve_of); +} + +fn det_of() { + let mut r = Rng(0x5EED_0003); + for n in sizes() { + // Scaled to determinants near 1, which `f32` holds for every size here. + let s = T::real(1.0 / (n + 1) as f64); + let a = well_conditioned::(&mut r, n) * s; + let b = well_conditioned::(&mut r, n) * s; + // `det(A·B) = det(A)·det(B)`, for matrices whose determinants are well conditioned. + let (x, y) = ((&a * &b).det().c64(), (a.det() * b.det()).c64()); + assert!( + (x - y).abs() <= 1e3 * (n + 1) as f64 * T::EPS * (y.abs() + T::EPS), + "{n}" + ); + // A triangular matrix's determinant is its diagonal's product. + let t = Matrix::from_fn(n, n, |i, j| if i <= j { a[(i, j)] } else { T::real(0.0) }); + let want: Complex = t.diagonal().iter().map(|x| x.c64()).product(); + let got = t.det().c64(); + assert!( + (got - want).abs() <= 8.0 * (n + 1) as f64 * T::EPS * want.abs(), + "{n}" + ); + } + assert_eq!(Matrix::::zeros(0, 0).det(), T::real(1.0)); +} + +#[test] +fn determinants() { + each_type!(det_of); +} + +fn singular_of() { + let mut r = Rng(0x5EED_0004); + let mut a = random::(&mut r, 5, 5); + // A zero column, and a column that repeats another. + for i in 0..5 { + a[(i, 2)] = T::real(0.0); + } + assert_eq!(a.det(), T::real(0.0)); + assert!(a.inverse().is_none()); + assert!(a.solve(&random::(&mut r, 5, 1)).is_none()); + assert!(a.powi(-1).is_none()); + assert!(a.lu().is_singular()); + let lu = a.lu(); + assert_close(&(&lu.p() * &a), &(&lu.l() * &lu.u()), 50.0); +} + +#[test] +fn singular_matrices() { + each_type!(singular_of); +} + +fn cholesky_of() { + let mut r = Rng(0x5EED_0005); + for n in sizes() { + let a = positive_definite::(&mut r, n); + let c = a.cholesky().unwrap(); + let l = c.l(); + for i in 0..n { + assert_eq!(l[(i, i)].c64().im, 0.0); + assert!(l[(i, i)].c64().re > 0.0); + for j in i + 1..n { + assert_eq!(l[(i, j)], T::real(0.0)); + } + } + let tol = 8.0 * (n + 1) as f64; + assert_close(&(&l * &l.conjugate_transpose()), &a, tol); + let b = random::(&mut r, n, 2); + assert_close(&(&a * &c.solve(&b)), &b, 4.0 * tol); + assert_close(&(&a * &c.inverse()), &Matrix::identity(n), 4.0 * tol); + // As the LU decomposition's, unless both overflow, as `f32` does from about 30. + let (d, got) = (a.det().c64(), c.det().to_f64()); + if d.is_finite() { + assert!( + (got - d.re).abs() <= 1e3 * (n + 1) as f64 * T::EPS * d.abs(), + "{n}" + ); + } else { + assert!(got.is_infinite()); + } + } + // Indefinite, and only the lower triangle read. + let mut a = positive_definite::(&mut r, 6); + a[(4, 4)] = T::real(-1.0); + assert!(a.cholesky().is_none()); + let mut a = positive_definite::(&mut r, 6); + let l = a.cholesky().unwrap().l(); + a[(0, 5)] = T::real(1e6); + assert_eq!(a.cholesky().unwrap().l(), l); + let mut nan = positive_definite::(&mut r, 3); + nan[(1, 1)] = T::real(f64::NAN); + assert!(nan.cholesky().is_none()); +} + +#[test] +fn cholesky() { + each_type!(cholesky_of); +} + +fn qr_of() { + let mut r = Rng(0x5EED_0006); + for n in sizes() { + for m in [n, n + 5, n / 2, 2 * n + 1] { + let a = random::(&mut r, m, n); + let qr = a.qr(); + let (q, rr) = (qr.q(), qr.r()); + let k = m.min(n); + assert_eq!((q.shape(), rr.shape()), ((m, k), (k, n))); + let tol = 8.0 * (m + n + 1) as f64; + assert_orthonormal(&q, tol); + assert_close(&(&q * &rr), &a, tol); + for i in 0..k { + assert_eq!(rr[(i, i)].c64().im, 0.0); + for j in 0..i { + assert_eq!(rr[(i, j)], T::real(0.0)); + } + } + let full = qr.q_full(); + assert_eq!(full.shape(), (m, m)); + assert_orthonormal(&full, tol); + assert_close(&Matrix::from_fn(m, k, |i, j| full[(i, j)]), &q, tol); + } + } +} + +#[test] +fn qr() { + each_type!(qr_of); +} + +fn least_squares_of() { + let mut r = Rng(0x5EED_0007); + for n in sizes() { + let m = 2 * n + 3; + let a = random::(&mut r, m, n); + let b = random::(&mut r, m, 2); + let x = a.qr().solve(&b).unwrap(); + // The residual is orthogonal to the columns: `A*·(B - A·X) = 0`. + let residual = &b - &(&a * &x); + let g = &a.conjugate_transpose() * &residual; + let scale = frobenius(&a) * frobenius(&b) + 1.0; + assert!( + frobenius(&g) <= 64.0 * (m + 1) as f64 * T::EPS * scale, + "{n}" + ); + } + // Dependent columns. + let mut a = random::(&mut r, 6, 3); + for i in 0..6 { + a[(i, 1)] = T::real(0.0); + } + assert!(a.qr().solve(&random::(&mut r, 6, 1)).is_none()); +} + +#[test] +fn least_squares() { + each_type!(least_squares_of); +} + +#[test] +fn exact_small_cases() { + // Pivots that are powers of two keep everything exact. + let a = Matrix::from_rows(&[[0.0, 2.0, 1.0], [4.0, 1.0, 0.0], [2.0, 0.5, 3.0]]); + let lu = a.lu(); + assert_eq!(lu.permutation(), [1, 0, 2]); + assert_eq!(&lu.p() * &a, &lu.l() * &lu.u()); + assert_eq!(a.det(), -24.0); + let c = Matrix::from_rows(&[[4.0, 2.0, -2.0], [2.0, 10.0, 2.0], [-2.0, 2.0, 6.0]]); + let l = c.cholesky().unwrap().l(); + assert_eq!( + l, + Matrix::from_rows(&[[2.0, 0.0, 0.0], [1.0, 3.0, 0.0], [-1.0, 1.0, 2.0]]) + ); + let _ = size(1, 1); +} diff --git a/lib/tests/math/matrix/factor_paths.rs b/lib/tests/math/matrix/factor_paths.rs new file mode 100644 index 0000000..3eb8033 --- /dev/null +++ b/lib/tests/math/matrix/factor_paths.rs @@ -0,0 +1,280 @@ +//! The paths of the LU and Cholesky decompositions, the inverse and the triangular solves on +//! both sides of each of their thresholds: unrolled tiny orders, unblocked and recursive +//! factorizations, leaves of the triangular solves on the stack or allocated, and the fallbacks +//! for pivots whose reciprocals overflow, singular matrices and NaNs. + +use super::common::{ + Num, Rng, assert_close, each_type, positive_definite, random, well_conditioned, +}; +use a::math::complex::Complex; +use a::math::matrix::Matrix; + +/// Orders on both sides of the thresholds: the unrolled orders up to 16, the unblocked LU up to +/// 24, the leaves of 16 and 32, the recursion's splits, and the working memory on the stack up to +/// 1024 elements. +fn orders() -> Vec { + if cfg!(miri) { + vec![0, 1, 2, 3, 4, 5, 8, 9, 12, 13, 16, 17, 25, 33] + } else { + let mut v: Vec = (0..=34).collect(); + v.extend([40, 47, 48, 49, 63, 64, 65, 96, 129]); + v + } +} + +/// The LU decomposition by plain Gaussian elimination, element by element: the pivot of a +/// column is its first element of the largest `|re| + |im|`, as the scan from the diagonal with +/// `>` finds. Returns the permutation. +fn reference_permutation(a: &Matrix) -> Vec { + let (m, n) = a.shape(); + let mut lu: Vec> = a.as_slice().iter().map(|x| x.c64()).collect(); + let mut perm: Vec = (0..m).collect(); + let abs1 = |x: Complex| x.re.abs() + x.im.abs(); + for j in 0..m.min(n) { + let mut p = j; + let mut best = abs1(lu[j + j * m]); + for i in j + 1..m { + let v = abs1(lu[i + j * m]); + if v > best { + best = v; + p = i; + } + } + if p != j { + perm.swap(j, p); + for c in 0..n { + lu.swap(j + c * m, p + c * m); + } + } + let d = lu[j + j * m]; + if d == Complex::new(0.0, 0.0) { + continue; + } + for i in j + 1..m { + lu[i + j * m] /= d; + } + for c in j + 1..n { + let u = lu[j + c * m]; + for i in j + 1..m { + let l = lu[i + j * m]; + lu[i + c * m] -= l * u; + } + } + } + perm +} + +fn lu_paths_of() { + let mut r = Rng(0x5EED_0101); + for n in orders() { + for m in [n, n + 5, n / 2, 2 * n + 1] { + let a = random::(&mut r, m, n); + let lu = a.lu(); + let (l, u) = (lu.l(), lu.u()); + assert_close(&(&lu.p() * &a), &(&l * &u), 16.0 * (m + n + 1) as f64); + // The pivots of the elimination element by element: random elements leave no ties + // that rounding could break the other way. + if T::EPS < 1e-10 { + assert_eq!(lu.permutation(), reference_permutation(&a), "{m} × {n}"); + } + for i in 0..u.rows() { + for j in 0..i { + assert_eq!(u[(i, j)], T::real(0.0)); + } + } + } + } +} + +#[test] +fn lu_paths() { + each_type!(lu_paths_of); +} + +/// Ties choose the first element: columns of ones swap nothing. +fn lu_ties_of() { + for n in orders() { + let a = Matrix::from_fn(n, n, |i, j| { + if j == 0 || i == j { + T::real(1.0) + } else { + T::real(0.0) + } + }); + let lu = a.lu(); + assert_eq!(lu.permutation(), (0..n).collect::>(), "{n}"); + assert_eq!(a.det(), T::real(1.0), "{n}"); + } +} + +#[test] +fn lu_ties() { + each_type!(lu_ties_of); +} + +fn inverse_paths_of() { + let mut r = Rng(0x5EED_0102); + for n in orders() { + let a = well_conditioned::(&mut r, n); + let inv = a.inverse().unwrap(); + let tol = 32.0 * (n + 1) as f64; + assert_close(&(&a * &inv), &Matrix::identity(n), tol); + assert_close(&(&inv * &a), &Matrix::identity(n), tol); + // From the decomposition as it is kept, by the cycles of the permutation. + assert_close(&a.lu().inverse().unwrap(), &inv, tol); + for cols in [1, 2, 3, 17, 40] { + let b = random::(&mut r, n, cols); + let x = a.solve(&b).unwrap(); + assert_close(&(&a * &x), &b, tol); + } + } +} + +#[test] +fn inverse_paths() { + each_type!(inverse_paths_of); +} + +fn singular_paths_of() { + let mut r = Rng(0x5EED_0103); + for n in orders().into_iter().filter(|&n| n > 0) { + for k in [0, n / 2, n - 1] { + // A zero column, and a zero row: an exact zero pivot, at the step of the column or + // at the last. Scaled so that the products of the other pivots do not overflow. + let mut a = well_conditioned::(&mut r, n) * T::real(1.0 / (n + 1) as f64); + let mut b = a.clone(); + for i in 0..n { + a[(i, k)] = T::real(0.0); + b[(k, i)] = T::real(0.0); + } + for a in [a, b] { + assert!(a.inverse().is_none(), "{n} {k}"); + assert!(a.solve(&random::(&mut r, n, 2)).is_none(), "{n} {k}"); + assert_eq!(a.det(), T::real(0.0), "{n} {k}"); + let lu = a.lu(); + assert!(lu.is_singular()); + assert!(lu.inverse().is_none()); + assert_close(&(&lu.p() * &a), &(&lu.l() * &lu.u()), 16.0 * (n + 1) as f64); + } + } + } +} + +#[test] +fn singular_paths() { + each_type!(singular_paths_of); +} + +/// A pivot so small that its reciprocal overflows: the decomposition and the triangular solves +/// divide instead, and the solution stays finite and exact where the products by the reciprocal +/// would give infinities and NaNs. +fn tiny_pivots_of() { + // Subnormal, its reciprocal past the largest float. + let tiny = if T::EPS < 1e-10 { 1e-310 } else { 1e-40 }; + for n in [1, 2, 3, 9, 13, 16, 17, 33] { + if cfg!(miri) && n > 17 { + continue; + } + for k in [0, n / 2, n - 1] { + // Upper triangular, small integers above the diagonal but in row `k`, whose pivot is + // `tiny`, and powers of two on it: the decomposition is the matrix itself, and + // `U·1 = b` is solved exactly, the reciprocals of the other pivots being exact. + let u = Matrix::from_fn(n, n, |i, j| { + if i == j { + T::real(if i == k { tiny } else { [1.0, 2.0, 4.0][i % 3] }) + } else if i < j && i != k { + T::real(((i + 2 * j) % 5) as f64 - 2.0) + } else { + T::real(0.0) + } + }); + let b = Matrix::from_fn(n, 1, |i, _| (0..n).fold(T::real(0.0), |s, j| s + u[(i, j)])); + let ones = Matrix::from_fn(n, 1, |_, _| T::real(1.0)); + assert_eq!(u.solve(&b).unwrap(), ones, "{n} {k}"); + assert_eq!(u.lu().solve(&b).unwrap(), ones, "{n} {k}"); + // Its transpose, lower triangular: the multipliers below the tiny pivot are zeros + // divided by it, not zeros times an infinity. + let lu = u.transpose().lu(); + assert!( + lu.l().as_slice().iter().all(|x| x.c64().is_finite()), + "{n} {k}" + ); + } + } +} + +#[test] +fn tiny_pivots() { + each_type!(tiny_pivots_of); +} + +/// NaNs and infinities go through without a panic; a NaN first in its column is its pivot, as +/// the scan from it finds. +fn non_finite_of() { + let mut r = Rng(0x5EED_0104); + for n in orders().into_iter().filter(|&n| n > 0) { + for bad in [f64::NAN, f64::INFINITY] { + let mut a = well_conditioned::(&mut r, n); + a[(0, 0)] = T::real(bad); + let lu = a.lu(); + if bad.is_nan() { + assert_eq!(lu.permutation()[0], 0, "{n}"); + } + let _ = a.inverse(); + let _ = a.solve(&random::(&mut r, n, 3)); + let _ = a.det(); + let mut b = well_conditioned::(&mut r, n); + b[(n - 1, n / 2)] = T::real(bad); + let _ = b.lu(); + let _ = b.inverse(); + } + } +} + +#[test] +fn non_finite() { + each_type!(non_finite_of); +} + +fn cholesky_paths_of() { + let mut r = Rng(0x5EED_0105); + for n in orders() { + let a = positive_definite::(&mut r, n); + let c = a.cholesky().unwrap(); + let l = c.l(); + for i in 0..n { + assert!(l[(i, i)].c64().re > 0.0 && l[(i, i)].c64().im == 0.0); + for j in i + 1..n { + assert_eq!(l[(i, j)], T::real(0.0)); + } + } + let tol = 16.0 * (n + 1) as f64; + assert_close(&(&l * &l.conjugate_transpose()), &a, tol); + let b = random::(&mut r, n, 3); + assert_close(&(&a * &c.solve(&b)), &b, 8.0 * tol); + // Only the lower triangle is read: NaNs above it change nothing. + let mut nan = a.clone(); + for j in 0..n { + for i in 0..j { + nan[(i, j)] = T::real(f64::NAN); + } + } + assert_eq!(nan.cholesky().unwrap().l(), l, "{n}"); + // Not positive definite at a column of each part of the recursion. + for k in [0, n / 2, n.saturating_sub(1)] { + if n == 0 { + break; + } + let mut bad = a.clone(); + bad[(k, k)] = T::real(-1.0); + assert!(bad.cholesky().is_none(), "{n} {k}"); + bad[(k, k)] = T::real(f64::NAN); + assert!(bad.cholesky().is_none(), "{n} {k}"); + } + } +} + +#[test] +fn cholesky_paths() { + each_type!(cholesky_paths_of); +} diff --git a/lib/tests/math/matrix/functions.rs b/lib/tests/math/matrix/functions.rs new file mode 100644 index 0000000..3f916dc --- /dev/null +++ b/lib/tests/math/matrix/functions.rs @@ -0,0 +1,408 @@ +//! The exponential, the square root and the logarithm, against mpmath at 50 digits and by the +//! identities they satisfy. + +use super::common::{Num, Rng, assert_close, each_type, frobenius, random}; +use a::math::complex::{Complex, Float}; +use a::math::matrix::Matrix; + +/// The real test matrix of mpmath's script: `((7i + 3j + 1) mod 11 - 5) / 4`, plus `shift` on +/// the diagonal. +fn real(n: usize, shift: f64) -> Matrix { + Matrix::from_fn(n, n, |i, j| { + let x = ((i * 7 + j * 3 + 1) % 11) as f64 - 5.0; + T::real(x / 4.0 + if i == j { shift } else { 0.0 }) + }) +} + +/// The complex test matrix: the real one plus `i·((5i + 2j) mod 7 - 3) / 4`. +fn complex(n: usize, shift: f64) -> Matrix> { + let re = real::(n, shift); + Matrix::from_fn(n, n, |i, j| { + Complex::new(re[(i, j)], (((i * 5 + j * 2) % 7) as f64 - 3.0) / 4.0) + }) +} + +/// The reference values as a matrix of the type, column after column. +fn reference(values: &[f64]) -> Matrix { + let n = (values.len() as f64).sqrt() as usize; + Matrix::from_column_major(n, n, values.iter().map(|&x| T::real(x)).collect()) +} + +/// Complex reference values. +fn reference_c(values: &[(f64, f64)]) -> Matrix> { + let n = (values.len() as f64).sqrt() as usize; + Matrix::from_column_major( + n, + n, + values + .iter() + .map(|&(re, im)| Complex::new(re, im)) + .collect(), + ) +} + +/// Asserts that `got` is within `tol` of `want`, relatively in the Frobenius norm. +#[track_caller] +fn near(got: &Matrix, want: &Matrix, tol: f64) { + let d = frobenius(&(got - want)) / frobenius(want).max(f64::MIN_POSITIVE); + assert!(d <= tol, "relative error {d:e} > {tol:e}"); +} + +/// The same complex matrix in single precision. +fn narrow(a: &Matrix>) -> Matrix> { + a.map(|z| Complex::new(z.re as f32, z.im as f32)) +} + +#[test] +fn against_mpmath() { + near(&real::(5, 0.0).exp(), &reference(&EXP_REAL5), 1e-14); + near( + &(real::(5, 0.0) * 8.0).exp(), + &reference(&EXP_REAL5_X8), + 1e-12, + ); + near(&complex(4, 0.0).exp(), &reference_c(&EXP_CPLX4), 1e-14); + near( + &(complex(4, 0.0) * Complex::new(8.0, 0.0)).exp(), + &reference_c(&EXP_CPLX4_X8), + 1e-12, + ); + near( + &real::(5, 3.0).sqrt().unwrap(), + &reference(&SQRT_REAL5S), + 1e-14, + ); + near( + &complex(4, 0.0).sqrt().unwrap(), + &reference_c(&SQRT_CPLX4), + 1e-14, + ); + near( + &real::(5, 3.0).log().unwrap(), + &reference(&LOG_REAL5S), + 1e-14, + ); + near( + &complex(4, 0.0).log().unwrap(), + &reference_c(&LOG_CPLX4), + 1e-14, + ); + near( + &complex(4, 3.0).log().unwrap(), + &reference_c(&LOG_CPLX4S), + 1e-14, + ); + // In single precision. + near(&real::(5, 0.0).exp(), &reference(&EXP_REAL5), 1e-5); + near( + &(real::(5, 0.0) * 8.0).exp(), + &reference(&EXP_REAL5_X8), + 1e-4, + ); + near( + &narrow(&complex(4, 0.0)).exp(), + &narrow(&reference_c(&EXP_CPLX4)), + 1e-5, + ); + near( + &real::(5, 3.0).sqrt().unwrap(), + &reference(&SQRT_REAL5S), + 1e-5, + ); + near( + &narrow(&complex(4, 0.0)).log().unwrap(), + &narrow(&reference_c(&LOG_CPLX4)), + 1e-5, + ); + // A real matrix with a negative real eigenvalue has no real square root or logarithm. + assert!(real::(5, 0.0).sqrt().is_none()); + assert!(real::(5, 0.0).log().is_none()); +} + +fn identities_of() { + let mut r = Rng(0x5EED_0301); + for n in [0, 1, 2, 3, 6, 12, 25] { + let a = random::(&mut r, n, n); + let e = a.exp(); + let tol = 1e3 * (n + 1) as f64; + assert_close(&(&e * &(-&a).exp()), &Matrix::identity(n), tol); + // `det(e^A) = e^tr(A)`. + let (d, t) = (e.det().c64(), a.trace().c64().exp()); + assert!((d - t).abs() <= tol * T::EPS * t.abs(), "{n}"); + // A matrix whose eigenvalues are near 2 + n has a principal square root and logarithm. + let mut b = a.clone(); + for i in 0..n { + b[(i, i)] += T::real(2.0 + n as f64); + } + let s = b.sqrt().unwrap(); + assert_close(&(&s * &s), &b, tol); + let l = b.log().unwrap(); + assert_close(&l.exp(), &b, 4.0 * tol); + // Scaled up, so that the squaring is exercised: `e^B = (e^(B/2))²`. + let big = &a * T::real(4.0); + let half = (&big * T::real(0.5)).exp(); + near(&big.exp(), &(&half * &half), 1e3 * (n + 1) as f64 * T::EPS); + } +} + +#[test] +fn identities() { + each_type!(identities_of); +} + +fn special_of() { + let z = Matrix::::zeros(3, 3); + let i = Matrix::::identity(3); + assert_eq!(z.exp(), i); + assert_eq!(i.sqrt().unwrap(), i); + assert_eq!(z.sqrt().unwrap(), z); + assert!(z.log().is_none()); + assert_close(&i.log().unwrap(), &z, 4.0); + // A diagonal matrix's functions are its diagonal's. + let d = Matrix::from_diagonal(&[T::real(0.5), T::real(2.0), T::real(3.0)]); + let want = Matrix::from_diagonal(&[0.5f64, 2.0, 3.0].map(|x| T::real(x.exp()))); + assert_close(&d.exp(), &want, 32.0); + let want = Matrix::from_diagonal(&[0.5f64, 2.0, 3.0].map(|x| T::real(x.ln()))); + assert_close(&d.log().unwrap(), &want, 32.0); + // A nilpotent block has no square root. + let n: Matrix = Matrix::from_fn(2, 2, |i, j| { + T::real(if i == 0 && j == 1 { 1.0 } else { 0.0 }) + }); + assert!(n.sqrt().is_none()); + // Its exponential is `I + N`. + assert_close(&n.exp(), &(&Matrix::identity(2) + &n), 4.0); + // Not a number. + let mut nan = i.clone(); + nan[(0, 1)] = T::real(f64::NAN); + assert!(nan.exp().as_slice().iter().all(|x| x.c64().is_nan())); + assert!(nan.sqrt().is_none() && nan.log().is_none()); + let _ = ::ZERO; +} + +#[test] +fn special_matrices() { + each_type!(special_of); +} + +#[test] +fn complex_results_of_complex_matrices() { + // `-I` has the principal logarithm `iπ·I` and the square root `i·I`. + let m = Matrix::>::identity(2) * Complex::new(-1.0, 0.0); + let l = m.log().unwrap(); + near( + &l, + &(Matrix::identity(2) * Complex::new(0.0, core::f64::consts::PI)), + 1e-15, + ); + let s = m.sqrt().unwrap(); + near(&s, &(Matrix::identity(2) * Complex::new(0.0, 1.0)), 1e-15); +} + +// Generated by mpmath 1.3 at 50 digits (expm, sqrtm, logm), column after column. + +const EXP_REAL5: [f64; 25] = [ + -0.40864434110048936, + 0.07409153861270298, + 0.759611586396231, + -1.465438165588391, + 0.44300515672059304, + -0.4610621653682522, + 0.24820253399933903, + 1.1254496365586522, + -0.6863779982863284, + 0.6349008267013937, + 1.4773686802080166, + -0.7565080847952791, + 2.6373972924452285, + 1.084814467059713, + -2.0566547520192677, + 0.3540913447357498, + 0.9372094902871732, + -1.7019830326902705, + 0.9356763039814013, + 1.4802883726160143, + -1.2900313819662266, + 0.33613324561495866, + 1.2703897059875338, + -1.509272877545262, + 1.9077274898373084, +]; + +const EXP_REAL5_X8: [f64; 25] = [ + -843.6038595063792, + -526.038141015697, + 2076.455379057395, + -633.898100036784, + -54.41587440943577, + -860.1039306428373, + -320.01098471572135, + 1449.426754662667, + -679.934901874692, + 238.6959302639472, + 1609.9853002475372, + -799.8751477834916, + 1514.9199296889763, + 1494.0062990373685, + -2368.508600913391, + -1950.2937126329375, + 1237.3443466477381, + -2673.250490844341, + -1845.0706485995468, + 3233.743419548897, + -1944.2327244068822, + -331.18312684519736, + 2090.85375029058, + -1599.2562911860668, + 1078.222511954887, +]; + +const EXP_CPLX4: [(f64, f64); 16] = [ + (-0.46141242768543395, -0.13180022778915032), + (-0.12437596155949655, -0.26018447305443027), + (0.926685862836244, 0.08794890319650685), + (-1.095589195948113, 1.024967655656184), + (-0.07163644997253843, 0.30325315978584677), + (-0.09673645004761722, -0.1747872116773369), + (1.0446356250631441, -0.3627269814027823), + (0.1301173576710276, 0.6023872472117751), + (1.0824806915396525, -0.015595961370035728), + (-0.31187249893051217, 0.5186916680216554), + (1.663920249370939, -2.6467318436572884), + (0.9810115477432633, 0.017695731833510996), + (0.6900147195345175, -0.7846323385132603), + (0.9743734725799288, -0.19080537462073657), + (-1.5517810858226964, 1.7152957209822406), + (-0.026059396506887877, -2.0282874592772604), +]; + +const EXP_CPLX4_X8: [(f64, f64); 16] = [ + (-1382.16855959852, 2707.950343766726), + (-2161.424135663659, -422.1506447092399), + (4879.478260852578, -329.8530824965177), + (719.6889836017372, 4447.9622847929795), + (744.0876934652557, 1476.938378628567), + (-919.6584083097073, 690.6932510566278), + (1968.5840443348911, -1933.3883067602158), + (1929.921173336344, 1166.5316174486302), + (2443.819615409637, -53.82457038259746), + (102.0346899074381, 1511.5187816318376), + (33.6028020965853, -4063.8301523857667), + (2422.102100167491, -1396.982080839599), + (-2644.7997438238517, -3987.962134031208), + (2374.5429119546357, -2551.4770796676385), + (-4056.141481487488, 7044.093953742347), + (-6455.482330613634, -2682.8608932497846), +]; + +const SQRT_REAL5S: [f64; 25] = [ + 1.4726557017669157, + 0.25869963225054304, + -0.09695733963423324, + -0.3375522940041712, + 0.11003937885935161, + -0.05306358898879797, + 1.287238344333727, + 0.14405976010538635, + -0.12715784994954754, + 0.4005774480933583, + 0.1238945236586378, + -0.1416612549084581, + 2.070557416588283, + 0.05552394105539912, + -0.1775149282554959, + 0.3622556396230759, + 0.04179391263890585, + -0.17584448143131554, + 2.0326289640180164, + -0.022304840421976436, + -0.18806521468875462, + 0.330690324518398, + -0.028029941422934444, + -0.25903433472327514, + 1.9045928805296273, +]; + +const SQRT_CPLX4: [(f64, f64); 16] = [ + (-0.1305950873671088, 0.5002409831307766), + (0.5687364579965972, 2.1132549179666857), + (-0.10280809093801707, -0.29857194793352876), + (-1.2163614076316065, 0.2926712728290518), + (-0.09706385914292939, -0.47253446037707775), + (-0.059438751523433565, -1.4369140942553151), + (0.06454026572827669, 0.3471458286870586), + (-0.05229320784246422, -0.4398064014265005), + (0.4205289633395557, -0.19057214794812272), + (-0.08578810441581991, -0.8670958822472796), + (1.111851870534163, -0.2282710123556405), + (0.3675661678192271, 0.061086429402730937), + (1.243844813555689, 0.3317645285875142), + (0.9238390907465496, -0.8625483587505157), + (-0.3650042576031283, -0.07722885629177098), + (1.7816272707571552, -0.08380081337025719), +]; + +const LOG_REAL5S: [f64; 25] = [ + 0.8312833868435073, + 0.35952712970219924, + -0.1382131531820212, + -0.36316908993912417, + 0.07821608385096769, + -0.0382631762800978, + 0.4544911196313371, + 0.1751564103528259, + -0.12669487339916827, + 0.5248890878786384, + 0.12373506977331959, + -0.16534470558189626, + 1.4656313533606755, + 0.0486908462617314, + -0.16326468166391117, + 0.4145749036392581, + 0.011324702284040112, + -0.15946537483242978, + 1.4538041805785664, + -0.04159410892920429, + -0.18752401025243107, + 0.44412501480070243, + -0.06190249405993922, + -0.2670525161843763, + 1.2382259893775003, +]; + +const LOG_CPLX4: [(f64, f64); 16] = [ + (-1.2351916539007306, 2.3811530609544485), + (1.1013522710401091, 5.751437486062311), + (-0.16886490459480136, -0.6350009455521394), + (-2.7436646713405377, 1.8125147917338595), + (-0.16995412048423109, -1.12577866391652), + (-0.5792316538268505, -3.56389671457387), + (-0.21568718297142192, 0.4721285189603243), + (0.3331410408969381, -1.1714459533871073), + (0.7393146490474126, -0.9604020196584055), + (0.19948488199659853, -2.3136316018839027), + (0.24879178969959984, -0.3872612461615717), + (1.0134218889529298, -0.3335882845399491), + (2.946928526877611, -0.2381245317322044), + (2.763266668607822, -2.898482098359805), + (-0.5027866527105621, -0.10753849937985441), + (2.827599040318983, -0.15175923689983883), +]; + +const LOG_CPLX4S: [(f64, f64); 16] = [ + (0.7799512274899486, -0.18880625952760624), + (0.2927472636163324, 0.34760906934962343), + (-0.09058643529833572, -0.09156940431898979), + (-0.3673631771492822, -0.1927035138132526), + (-0.06261256733033056, -0.128982599321903), + (0.6210560902204794, -0.33636485787824044), + (0.115788554901526, 0.18604901220250206), + (-0.15728899455285544, -0.0826112883084996), + (0.1616718074299953, 0.060634945034871296), + (-0.15352763914843642, -0.1639427116680186), + (1.4609964275836624, -0.13699616505197557), + (0.05383354080708117, 0.13623565594166198), + (0.37321638171998595, 0.3320896072630335), + (0.06288512898227337, -0.01766861990230528), + (-0.1647967458212374, -0.06691084041452824), + (1.4343901979895786, -0.08839364489784675), +]; diff --git a/lib/tests/math/matrix/hermitian.rs b/lib/tests/math/matrix/hermitian.rs new file mode 100644 index 0000000..85a3f00 --- /dev/null +++ b/lib/tests/math/matrix/hermitian.rs @@ -0,0 +1,338 @@ +//! The Hermitian eigensolver on sizes across its thresholds and on the matrices that make divide +//! and conquer deflate: checked by orthonormality and `V·Λ·V* = A`, in `f64` arithmetic. + +use super::common::{Num, Rng, each_type, random, size}; +use a::math::complex::{Complex, Float}; +use a::math::matrix::Matrix; + +/// The matrix in `Complex`. +fn c64(a: &Matrix) -> Matrix> { + a.map(|x| x.c64()) +} + +/// The Frobenius norm of a `Complex` matrix. +fn norm(a: &Matrix>) -> f64 { + a.as_slice() + .iter() + .map(|x| x.norm_sqr()) + .sum::() + .sqrt() +} + +/// The eigendecomposition of `a` and its errors in units of `n·ε`: `‖V*·V - I‖` and +/// `‖A·V - V·Λ‖ / ‖A‖`, and the largest difference of the eigenvalues computed alone, in units +/// of `n·ε·‖A‖`, all with `A` and `Λ` divided by `f` first. Checks that the eigenvalues ascend. +fn errors(a: &Matrix, f: f64) -> (f64, f64, f64) { + let n = a.rows(); + let e = a.hermitian_eigen(); + let (v, w) = (c64(e.vectors()), e.values()); + assert_eq!((v.shape(), w.len()), ((n, n), n)); + assert!(w.windows(2).all(|p| p[0] <= p[1]), "not ascending"); + let unit = (n.max(1) as f64) * T::EPS; + let orth = norm(&(&(&v.conjugate_transpose() * &v) - &Matrix::identity(n))) / unit; + // The whole matrix, from its lower triangle. + let full = Matrix::from_fn(n, n, |i, j| { + let x = if i >= j { + a[(i, j)].c64() + } else { + a[(j, i)].c64().conj() + }; + x / f + }); + let vl = Matrix::from_fn(n, n, |i, j| v[(i, j)] * (w[j].to_f64() / f)); + let size = norm(&full); + let scale = unit * if size > 0.0 { size } else { 1.0 }; + let resid = norm(&(&(&full * &v) - &vl)) / scale; + let alone = a.hermitian_eigenvalues(); + let mut diff = 0f64; + for (x, y) in alone.iter().zip(w) { + diff = diff.max((x.to_f64() / f - y.to_f64() / f).abs()); + } + (orth, resid, diff / scale) +} + +/// Asserts the errors of [`errors`] are below `tol`. +#[track_caller] +fn check(a: &Matrix, tol: f64, what: &str) { + check_scaled(a, 1.0, tol, what); +} + +/// [`check`] of `a` divided by `f`. +#[track_caller] +fn check_scaled(a: &Matrix, f: f64, tol: f64, what: &str) { + let (orth, resid, diff) = errors(a, f); + let name = core::any::type_name::(); + assert!( + orth <= tol && resid <= tol && diff <= tol, + "{what} of {} for {name}: ‖V*V - I‖ {orth:.2}, ‖VΛV* - A‖ {resid:.2}, values {diff:.2} (n·ε)", + a.rows() + ); +} + +/// A random unitary matrix, from a QR decomposition. +fn unitary(r: &mut Rng, n: usize) -> Matrix { + random::(r, n, n).qr().q() +} + +/// `Q·diag(λ)·Q*` for a random unitary `Q`. +fn with_eigenvalues(r: &mut Rng, lambda: &[f64]) -> Matrix { + let n = lambda.len(); + let q = unitary::(r, n); + let l: Vec = lambda.iter().map(|&x| T::real(x)).collect(); + &(&q * &Matrix::from_diagonal(&l)) * &q.conjugate_transpose() +} + +/// The symmetric tridiagonal matrix with the diagonal `d` and the off-diagonal `e`. +fn tridiagonal(d: &[f64], e: &[f64]) -> Matrix { + Matrix::from_fn(d.len(), d.len(), |i, j| { + if i == j { + T::real(d[i]) + } else if i == j + 1 || j == i + 1 { + T::real(e[i.min(j)]) + } else { + T::real(0.0) + } + }) +} + +/// Sizes around the divide and conquer's smallest problems (25) and the reduction's panels and +/// unblocked tail (32 and 64), and a few levels and panels more; 300 for `f64` alone, which is +/// enough to check slowly. +fn sizes() -> Vec { + if cfg!(miri) { + return vec![0, 1, 2, 24, 26]; + } + let mut sizes = vec![ + 0, 1, 2, 3, 24, 25, 26, 27, 50, 51, 63, 64, 65, 66, 96, 97, 100, 130, 200, + ]; + if core::any::type_name::() == "f64" { + sizes.push(300); + } + sizes +} + +fn random_of() { + let mut r = Rng(0x5EED_0201); + for n in sizes::() { + let b = random::(&mut r, n, n); + let a = &b + &b.conjugate_transpose(); + check(&a, 4.0, "random"); + // Only the lower triangle is read: anything above it changes nothing. + let mut c = a.clone(); + for j in 1..n { + for i in 0..j { + c[(i, j)] = T::real(1e3 + i as f64); + } + } + if n > 0 && n <= 100 { + let (x, y) = (c.hermitian_eigen(), a.hermitian_eigen()); + assert_eq!((x.values(), x.vectors()), (y.values(), y.vectors())); + } + } +} + +#[test] +fn random_sizes() { + each_type!(random_of); +} + +fn deflating_of() { + let mut r = Rng(0x5EED_0202); + let n = size(130, 30); + deflating_at::(&mut r, n); + // Wilkinson's W21+ and a bigger one, whose eigenvalues come in pairs closer than any ulp. + for m in [10, size(60, 13)] { + wilkinson::(&mut r, m); + } +} + +/// The matrices that make divide and conquer deflate, of order `n`: low rank changes of the +/// identity, repeated and clustered eigenvalues, the second difference matrix, and a +/// tridiagonal one that splits. +fn deflating_at(r: &mut Rng, n: usize) { + // The identity plus rank one and rank two: all but one or two eigenvalues are 1. + for rank in [1, 2] { + let mut a = Matrix::::identity(n); + for _ in 0..rank { + let u = random::(r, n, 1); + a = &a + &(&u * &u.conjugate_transpose()); + } + check(&a, 4.0, "identity plus low rank"); + } + // Eigenvalues repeated in a few big clusters, exactly and to within a few ulps. + let repeated: Vec = (0..n).map(|i| (i % 3) as f64 - 1.0).collect(); + check(&with_eigenvalues::(r, &repeated), 8.0, "repeated"); + let clustered: Vec = (0..n) + .map(|i| 1.0 + (i % 7) as f64 * 4.0 * T::EPS) + .collect(); + check(&with_eigenvalues::(r, &clustered), 8.0, "clustered"); + let close: Vec = (0..n) + .map(|i| 1.0 + i as f64 * 1e-3 * T::EPS.sqrt()) + .collect(); + check(&with_eigenvalues::(r, &close), 8.0, "close"); + // The second difference matrix, with its known eigenvalues `2 - 2·cos(kπ/(n + 1))`. + let t = tridiagonal::(&vec![2.0; n], &vec![-1.0; n - 1]); + check(&t, 4.0, "second difference"); + let w = t.hermitian_eigenvalues(); + for (k, w) in w.iter().enumerate() { + let exact = 2.0 - 2.0 * ((k + 1) as f64 * core::f64::consts::PI / (n + 1) as f64).cos(); + assert!( + (w.to_f64() - exact).abs() <= 8.0 * T::EPS * 4.0, + "{k}: {w:?} {exact}" + ); + } + // Tridiagonal with negligible off-diagonals here and there: it splits. + let d: Vec = (0..n).map(|i| ((i * 7) % 11) as f64).collect(); + let e: Vec = (0..n - 1) + .map(|i| if i % 9 == 4 { 1e-30 } else { 0.5 }) + .collect(); + check(&tridiagonal::(&d, &e), 4.0, "splitting"); +} + +/// Wilkinson's matrix `W_{2m+1}+`, whose eigenvalues come in pairs closer than any ulp, as it +/// is and reduced from a dense matrix. +fn wilkinson(r: &mut Rng, m: usize) { + let d: Vec = (0..2 * m + 1) + .map(|i| (i as f64 - m as f64).abs()) + .collect(); + let e = vec![1.0; 2 * m]; + check(&tridiagonal::(&d, &e), 4.0, "Wilkinson"); + let q = unitary::(r, 2 * m + 1); + let t = tridiagonal::(&d, &e); + check( + &(&(&q * &t) * &q.conjugate_transpose()), + 4.0, + "dense Wilkinson", + ); +} + +#[test] +fn deflating() { + each_type!(deflating_of); +} + +fn structured_of() { + let mut r = Rng(0x5EED_0203); + structured_at::(&mut r, size(100, 30)); +} + +/// Matrices of order `n` with structure: diagonal, zero, graded, near the ends of the range, +/// not finite. +fn structured_at(r: &mut Rng, n: usize) { + // Diagonal, with repeats: the eigenvalues are its elements, the vectors the identity's. + let d: Vec = (0..n) + .map(|i| T::real(((i * 5) % 13) as f64 - 6.0)) + .collect(); + let a = Matrix::from_diagonal(&d); + check(&a, 1.0, "diagonal"); + let mut sorted: Vec = d.iter().map(|x| x.c64().re).collect(); + sorted.sort_by(f64::total_cmp); + let w: Vec = a + .hermitian_eigenvalues() + .iter() + .map(|x| x.to_f64()) + .collect(); + assert_eq!(w, sorted); + // Zero: zero eigenvalues and orthonormal vectors. + let z = Matrix::::zeros(n, n); + check(&z, 1.0, "zero"); + assert!( + z.hermitian_eigen() + .values() + .iter() + .all(|&x| x == Float::ZERO) + ); + // Graded: elements from 1 down to about 10⁻²⁰ (f64) or 10⁻¹⁰ (f32) across the matrix. + let decades = if T::EPS < 1e-10 { 20.0 } else { 10.0 }; + let g = 10f64.powf(-decades / (2 * n) as f64); + let b = random::(r, n, n); + let s = &b + &b.conjugate_transpose(); + let graded = Matrix::from_fn(n, n, |i, j| s[(i, j)] * T::real(g.powi((i + j) as i32))); + check(&graded, 4.0, "graded"); + // Far from 1: scaled near the ends of the range. + let (big, small) = if T::EPS < 1e-10 { + (1e300, 1e-300) + } else { + (1e35, 1e-35) + }; + for f in [big, small] { + let scaled = s.map(|&x| x * T::real(f)); + check_scaled(&scaled, f, 4.0, "scaled"); + } + // Not a number, or infinite: NaNs. + let mut bad = s.clone(); + bad[(n - 1, 0)] = T::real(f64::INFINITY); + let e = bad.hermitian_eigen(); + assert!(e.values().iter().all(|x| x.to_f64().is_nan())); + assert!(e.vectors().as_slice().iter().all(|x| x.c64().re.is_nan())); + assert!( + bad.hermitian_eigenvalues() + .iter() + .all(|x| x.to_f64().is_nan()) + ); +} + +#[test] +fn structured() { + each_type!(structured_of); +} + +/// The orders of the small path, on both sides of the switches of its scratch at 16 and 48 and +/// of the crossover of the complex vectors at 80. +fn small_sizes() -> Vec { + if cfg!(miri) { + vec![2, 5, 17] + } else { + vec![2, 3, 5, 8, 16, 17, 40, 48, 49, 80, 81] + } +} + +/// The orders about the other crossovers of the small path: those of the vectors, 112 for `f64` +/// and 160 for `f32`, and that of the eigenvalues alone, 128. +fn crossovers() -> Vec { + if cfg!(miri) { + return Vec::new(); + } + match core::any::type_name::() { + "f64" => vec![112, 113, 128, 129], + "f32" => vec![128, 129, 160, 161], + _ => vec![128, 129], + } +} + +fn small_of() { + let mut r = Rng(0x5EED_0204); + for n in small_sizes() { + deflating_at::(&mut r, n); + structured_at::(&mut r, n); + wilkinson::(&mut r, (n - 1) / 2); + // Random, with anything finite written above the diagonal, which is not read. + let b = random::(&mut r, n, n); + let a = &b + &b.conjugate_transpose(); + check(&a, 4.0, "random"); + let mut c = a.clone(); + for j in 1..n { + for i in 0..j { + c[(i, j)] = T::real(1e3 + i as f64); + } + } + // Miri perturbs the roundings of some operations at random, run to run. + if !cfg!(miri) { + let (x, y) = (c.hermitian_eigen(), a.hermitian_eigen()); + assert_eq!((x.values(), x.vectors()), (y.values(), y.vectors())); + } + } + for n in crossovers::() { + let b = random::(&mut r, n, n); + check(&(&b + &b.conjugate_transpose()), 4.0, "random"); + let clustered: Vec = (0..n) + .map(|i| 1.0 + (i % 7) as f64 * 4.0 * T::EPS) + .collect(); + check(&with_eigenvalues::(&mut r, &clustered), 8.0, "clustered"); + } +} + +#[test] +fn small_hard_cases() { + each_type!(small_of); +} diff --git a/lib/tests/math/matrix/mod.rs b/lib/tests/math/matrix/mod.rs new file mode 100644 index 0000000..062ac91 --- /dev/null +++ b/lib/tests/math/matrix/mod.rs @@ -0,0 +1,14 @@ +mod basic; +mod common; +mod eigen; +mod exact; +mod extremes; +mod factor; +mod factor_paths; +mod functions; +mod hermitian; +mod nonsymmetric; +mod product; +mod qr_paths; +mod svd; +mod svd_paths; diff --git a/lib/tests/math/matrix/nonsymmetric.rs b/lib/tests/math/matrix/nonsymmetric.rs new file mode 100644 index 0000000..0b26803 --- /dev/null +++ b/lib/tests/math/matrix/nonsymmetric.rs @@ -0,0 +1,339 @@ +//! The nonsymmetric eigenproblem past the small sizes: the blocked Hessenberg reduction, the +//! multishift QR algorithm with aggressive early deflation, and the eigenvectors, on random and +//! hard matrices of every element type, at the sizes where the algorithms change. + +use super::common::{Num, Rng, assert_close, assert_orthonormal, each_type, frobenius, random}; +use a::math::complex::{Complex, Float}; +use a::math::matrix::Matrix; + +/// Sizes across the switches of the algorithms: to the multishift QR algorithm (at 120, for +/// complex elements with `Q` at 90), to fewer shifts than LAPACK's (150), to sweeps that gather +/// their reflections without `Q` too (200); past the blocks of the Hessenberg reduction and of +/// the eigenvectors (64), and to windows reduced by the multishift algorithm themselves (with +/// the stalling matrices of [`hard_matrices`]). +fn sizes() -> Vec { + let mut sizes = if cfg!(miri) { + vec![] + } else if is_complex::() { + vec![89, 90, 119, 120, 150, 200] + } else { + vec![119, 120, 121, 150, 200] + }; + // The largest only in double precision, whose code is the single's. + if !cfg!(miri) && T::EPS < 1e-10 { + sizes.push(300); + } + sizes +} + +/// Whether the elements are complex. +fn is_complex() -> bool { + core::any::type_name::().contains("Complex") +} + +/// A complex number of the type's precision as a `Complex`. +fn c64(w: Complex) -> Complex { + Complex::new(w.re.to_f64(), w.im.to_f64()) +} + +/// Checks the Schur decomposition of `a`, its eigenvalues alone, and its eigenvectors, with +/// tolerances `tol·n·ε` relative to the norm of `a`. The eigenvalues of a `defective` matrix, +/// sensitive to the roundings, are checked by their sum only. +fn check(a: &Matrix, tol: f64, defective: bool) { + let n = a.rows(); + let complex = is_complex::(); + let scale = frobenius(a) + 1.0; + let s = a.schur(); + let (q, t) = (s.q(), s.t()); + let tol = tol * (n + 1) as f64; + assert_orthonormal(&q, tol); + assert_close(&(&(&q * &t) * &q.conjugate_transpose()), a, tol); + let zero = T::real(0.0); + let values = s.eigenvalues(); + assert_eq!(values.len(), n); + // The structure: triangular, or quasi-triangular with standard 2 × 2 blocks. + let mut i = 0; + while i < n { + for r in i + 2..n { + assert_eq!(t[(r, i)], zero, "below the subdiagonal at {r} {i}, n = {n}"); + } + let d = t[(i, i)].c64(); + if !complex && i + 1 < n && t[(i + 1, i)] != zero { + assert!( + i + 2 >= n || t[(i + 2, i + 1)] == zero, + "blocks overlap at {i}" + ); + let (b, c) = (t[(i, i + 1)].c64().re, t[(i + 1, i)].c64().re); + assert_eq!( + t[(i + 1, i + 1)].c64(), + d, + "unequal diagonal in the block at {i}" + ); + assert!(b * c < 0.0, "a block of real eigenvalues at {i}"); + let im = b.abs().sqrt() * c.abs().sqrt(); + let (w1, w2) = (c64(values[i]), c64(values[i + 1])); + assert!((w1 - Complex::new(d.re, im)).abs() <= 8.0 * T::EPS * (d.abs() + im)); + assert!((w2 - Complex::new(d.re, -im)).abs() <= 8.0 * T::EPS * (d.abs() + im)); + i += 2; + } else { + if i + 1 < n { + assert_eq!(t[(i + 1, i)], zero, "subdiagonal at {i}, n = {n}"); + } + assert_eq!(c64(values[i]), d); + i += 1; + } + } + // Their sum is the trace. + let sum: Complex = values.iter().map(|&w| c64(w)).sum(); + let trace = a.trace().c64(); + assert!( + (sum - trace).abs() <= tol * T::EPS * scale, + "trace {trace}, sum {sum}, n = {n}" + ); + if !defective { + same_values(a, values, tol, scale); + } + // The eigenvectors: of length 1, `A·v = λ·v` up to rounding, even for a defective matrix. + let e = a.eigen(); + let v: Matrix> = e.vectors().map(|&x| c64(x)); + let ac = a.map(|x| x.c64()); + for (j, &w) in e.values().iter().enumerate() { + let x = Matrix::from_columns(&[v.column(j)]); + let norm: f64 = x + .as_slice() + .iter() + .map(|z| z.norm_sqr()) + .sum::() + .sqrt(); + assert!( + (norm - 1.0).abs() <= tol * T::EPS, + "norm {norm} of vector {j}, n = {n}" + ); + let d = &(&ac * &x) - &(&x * c64(w)); + let d: f64 = d + .as_slice() + .iter() + .map(|z| z.norm_sqr()) + .sum::() + .sqrt(); + assert!( + d <= 4.0 * tol * T::EPS * scale, + "residual {d:e} of vector {j}, n = {n}" + ); + } +} + +/// Asserts that the eigenvalues alone are `values`, each matched to the nearest one left: the +/// order of nearly equal ones may differ. +fn same_values(a: &Matrix, values: &[Complex], tol: f64, scale: f64) { + let n = a.rows(); + let key = + |w: &Complex, v: &Complex| w.re.total_cmp(&v.re).then(w.im.total_cmp(&v.im)); + let mut alone: Vec> = a.eigenvalues().into_iter().map(c64).collect(); + let mut with: Vec> = values.iter().map(|&w| c64(w)).collect(); + alone.sort_by(key); + with.sort_by(key); + let mut used = vec![false; n]; + for x in &alone { + let (k, d) = with + .iter() + .enumerate() + .filter(|(k, _)| !used[*k]) + .map(|(k, y)| (k, (x - y).abs())) + .min_by(|p, q| p.1.total_cmp(&q.1)) + .unwrap(); + used[k] = true; + let bound = (tol * T::EPS * scale).sqrt().max(tol * T::EPS * scale); + assert!( + d <= bound, + "eigenvalue {x} alone, nearest {} with Q, n = {n}", + with[k] + ); + } +} + +fn random_of() { + let mut r = Rng(0x5EED_0201); + for n in sizes::() { + check(&random::(&mut r, n, n), 32.0, false); + } +} + +#[test] +fn random_matrices() { + each_type!(random_of); +} + +/// Matrices already upper Hessenberg, and ones with zeros on the subdiagonal. +fn hessenberg_of() { + let mut r = Rng(0x5EED_0202); + for n in sizes::().into_iter().filter(|&n| n < 300) { + let a = Matrix::from_fn(n, n, |i, j| { + if i <= j + 1 { + T::random(&mut r) + } else { + T::real(0.0) + } + }); + check(&a, 32.0, false); + let split = Matrix::from_fn(n, n, |i, j| { + if i == j + 1 && i % 7 == 0 { + T::real(0.0) + } else { + a[(i, j)] + } + }); + check(&split, 32.0, false); + } +} + +#[test] +fn hessenberg_matrices() { + each_type!(hessenberg_of); +} + +/// The sizes of the hard cases: one each side of the switches to the multishift algorithm, none +/// under Miri, which the small ones of [`super::eigen`] serve. +fn hard_sizes() -> Vec { + if cfg!(miri) { vec![] } else { vec![76, 150] } +} + +/// A cyclic permutation, whose shifts stall without exceptional ones. +fn cyclic(n: usize) -> Matrix { + Matrix::from_fn(n, n, |i, j| { + T::real(if (i + 1) % n == j { 1.0 } else { 0.0 }) + }) +} + +fn stalling_of() { + for n in hard_sizes() { + check(&cyclic::(n), 32.0, false); + } + // Its stalls grow the deflation window past the switch to the multishift algorithm, 120 + // for real elements, 90 for complex ones with `Q`; in double precision only, and not under + // Miri, where it takes hours. + if !cfg!(miri) && T::EPS < 1e-10 { + check( + &cyclic::(if is_complex::() { 300 } else { 400 }), + 32.0, + false, + ); + } +} + +#[test] +fn stalling_matrices() { + each_type!(stalling_of); +} + +/// Defective matrices: a Jordan block, and blocks of order 5, a few sharing an eigenvalue, +/// hidden by a similarity. +fn defective_of() { + let mut r = Rng(0x5EED_0203); + let zero = T::real(0.0); + for n in hard_sizes() { + let jordan = Matrix::from_fn(n, n, |i, j| { + if i == j { + T::real(2.0) + } else if i + 1 == j { + T::real(1.0) + } else { + zero + } + }); + check(&jordan, 32.0, true); + let blocks = Matrix::from_fn(n, n, |i, j| { + if i == j { + T::real((i / 5 % 3) as f64) + } else if i + 1 == j && i % 5 != 4 { + T::real(1.0) + } else { + zero + } + }); + let p = super::common::well_conditioned::(&mut r, n); + let pinv = p.inverse().unwrap(); + check(&(&(&p * &blocks) * &pinv), 1e3, true); + } +} + +#[test] +fn defective_matrices() { + each_type!(defective_of); +} + +/// Hard cases: repeated and clustered eigenvalues; a graded matrix; a companion matrix; real +/// matrices with complex eigenvalues only; zero and a multiple of the identity. +fn hard_of() { + let mut r = Rng(0x5EED_0204); + let zero = T::real(0.0); + for n in hard_sizes() { + // Repeated and clustered eigenvalues of a normal matrix: Q·D·Q*. + let q = random::(&mut r, n, n).qr().q(); + let d = Matrix::from_diagonal( + &(0..n) + .map(|i| T::real(1.0 + (i % 4) as f64 + 1e-9 * (i % 3) as f64)) + .collect::>(), + ); + check(&(&(&q * &d) * &q.conjugate_transpose()), 32.0, false); + // Graded: elements from 1 down to about 1e-12. + let g = Matrix::from_fn(n, n, |i, j| { + T::random(&mut r) * T::real(10f64.powf(-12.0 * (i + j) as f64 / (2 * n) as f64)) + }); + check(&g, 64.0, false); + // The companion matrix of a polynomial with random coefficients. + let coefficients: Vec = (0..n).map(|_| T::random(&mut r)).collect(); + let c = Matrix::from_fn(n, n, |i, j| { + if j == n - 1 { + -coefficients[i] + } else if i == j + 1 { + T::real(1.0) + } else { + zero + } + }); + check(&c, 64.0, false); + // Rotations by different angles on the diagonal, mixed by an orthogonal similarity: a + // real matrix of complex pairs only. + let mut rot = Matrix::::zeros(n, n); + for k in 0..n / 2 { + let (s, cth) = (0.1 + k as f64).sin_cos(); + rot[(2 * k, 2 * k)] = T::real(cth); + rot[(2 * k, 2 * k + 1)] = T::real(-s); + rot[(2 * k + 1, 2 * k)] = T::real(s); + rot[(2 * k + 1, 2 * k + 1)] = T::real(cth); + } + if n % 2 == 1 { + rot[(n - 1, n - 1)] = T::real(1.0); + } + check(&(&(&q * &rot) * &q.conjugate_transpose()), 32.0, false); + check(&Matrix::::zeros(n, n), 32.0, false); + check( + &Matrix::from_fn(n, n, |i, j| if i == j { T::real(3.0) } else { zero }), + 32.0, + false, + ); + } +} + +#[test] +fn hard_matrices() { + each_type!(hard_of); +} + +/// A matrix with a NaN or an infinity gives NaNs. +fn not_finite_of() { + let mut r = Rng(0x5EED_0204); + for n in hard_sizes() { + let mut a = random::(&mut r, n, n); + a[(n / 2, n / 3)] = T::real(f64::INFINITY); + assert!(a.eigenvalues().iter().all(|w| w.re.is_nan())); + let e = a.eigen(); + assert!(e.values().iter().all(|w| w.re.is_nan())); + } +} + +#[test] +fn not_finite() { + each_type!(not_finite_of); +} diff --git a/lib/tests/math/matrix/product.rs b/lib/tests/math/matrix/product.rs new file mode 100644 index 0000000..c0c6090 --- /dev/null +++ b/lib/tests/math/matrix/product.rs @@ -0,0 +1,135 @@ +//! Products of matrices of floats and complex numbers, `Matrix * Matrix`, against the textbook +//! sum computed here: every element within a bound on the rounding of a sum of `k` products, for +//! shapes on both sides of every edge of the blocks the product is computed by. + +use super::common::{Num, Rng}; +use a::math::complex::Complex; +use a::math::matrix::Matrix; + +/// Checks `a · b` of an `m × k` and a `k × n` random matrix against the textbook sum in `f64`: +/// each element within `4 (k + 2) ε` of the sum of the absolute values of its products. +fn check(m: usize, k: usize, n: usize, seed: u64) { + let mut r = Rng(seed | 1); + let a = Matrix::from_fn(m, k, |_, _| T::random(&mut r)); + let b = Matrix::from_fn(k, n, |_, _| T::random(&mut r)); + let c = &a * &b; + assert_eq!(c.shape(), (m, n)); + for j in 0..n { + for i in 0..m { + let (mut re, mut im, mut size) = (0.0, 0.0, 0.0); + for p in 0..k { + let (x, y) = (a[(i, p)].c64(), b[(p, j)].c64()); + let (pr, pi) = (x.re * y.re - x.im * y.im, x.re * y.im + x.im * y.re); + re += pr; + im += pi; + size += pr.abs() + pi.abs(); + } + let got = c[(i, j)].c64(); + let error = (got.re - re).abs() + (got.im - im).abs(); + let bound = 4.0 * (k as f64 + 2.0) * T::EPS * size; + assert!( + error <= bound, + "{m} × {k} · {k} × {n}: element ({i}, {j}) is {got:?}, not ({re}, {im})" + ); + } + } +} + +/// [`check`] on every type. +fn check_all(m: usize, k: usize, n: usize) { + let seed = (m as u64) << 40 ^ (k as u64) << 20 ^ n as u64; + check::(m, k, n, seed); + check::(m, k, n, seed); + check::>(m, k, n, seed); + check::>(m, k, n, seed); +} + +#[test] +fn small_shapes() { + // Every shape up to 9, and some past the tiles of the direct products. + let top = if cfg!(miri) { 5 } else { 9 }; + for m in 1..=top { + for k in 1..=top { + for n in 1..=top { + check_all(m, k, n); + } + } + } + for &(m, k, n) in &[ + (16, 16, 16), + (17, 3, 13), + (33, 9, 5), + (5, 31, 33), + (64, 64, 64), + ] { + if !cfg!(miri) || m * k * n <= 4096 { + check_all(m, k, n); + } + } +} + +#[test] +fn blocked_edges() { + // Large enough to be packed in blocks; rows on both sides of a tile's eight, columns of every + // remainder of the panels' widths (2 to 12), and depths across a block's. + let shapes: &[(usize, usize, usize)] = if cfg!(miri) { + &[ + (50, 257, 1), + (25, 600, 1), + (200, 70, 1), + (1, 9, 2100), + (9, 9, 700), + ] + } else { + &[ + (100, 100, 20), + (127, 129, 31), + (129, 255, 50), + (257, 257, 7), + (8, 600, 63), + (300, 70, 300), + (65, 513, 65), + (130, 30, 1030), + (4, 9, 2050), + ] + }; + for &(m, k, n) in shapes { + check_all(m, k, n); + } + if !cfg!(miri) { + // Every remainder of the columns, by the widest panels of each type. + for n in 40..=52 { + check_all(72, 90, n); + } + } +} + +#[test] +fn empty_and_zero_depth() { + for &(m, k, n) in &[(0, 3, 4), (3, 0, 4), (3, 4, 0), (0, 0, 0), (5, 0, 7)] { + let a = Matrix::::from_fn(m, k, |i, j| (i + 2 * j) as f64); + let b = Matrix::::from_fn(k, n, |i, j| (3 * i + j) as f64); + assert_eq!(&a * &b, Matrix::zeros(m, n)); + let a = a.map(|x| Complex::new(*x as f32, 1.0)); + let b = b.map(|x| Complex::new(1.0, *x as f32)); + assert_eq!(&a * &b, Matrix::zeros(m, n)); + } +} + +#[test] +fn identities_and_assignment() { + let mut r = Rng(7); + let a = Matrix::from_fn(37, 41, |_, _| r.unit()); + assert_eq!(&a * &Matrix::identity(41), a); + assert_eq!(&Matrix::identity(37) * &a, a); + let mut b = Matrix::from_fn(37, 37, |_, _| r.unit()); + let product = &b * &Matrix::identity(37); + b *= Matrix::identity(37); + assert_eq!(b, product); + // Infinities and NaNs propagate as in the textbook sums. + let x = Matrix::from_rows(&[[f64::INFINITY, 1.0], [2.0, f64::NAN]]); + let y = Matrix::from_rows(&[[1.0, 0.0], [0.0, 1.0]]); + let z = &x * &y; + assert_eq!(z[(0, 0)], f64::INFINITY); + assert!(z[(1, 1)].is_nan()); +} diff --git a/lib/tests/math/matrix/qr_paths.rs b/lib/tests/math/matrix/qr_paths.rs new file mode 100644 index 0000000..d81e23b --- /dev/null +++ b/lib/tests/math/matrix/qr_paths.rs @@ -0,0 +1,498 @@ +//! QR decompositions across the thresholds of the algorithm: unblocked for small matrices, and +//! blocked, the width of the blocks by the shape and the type, the panels halved down to their +//! leaves; `Q` formed one reflection at a time or by blocks; the least squares by one reflection +//! at a time or by blocks, by substitution or a triangular solve; and hard cases: zero and +//! dependent columns, scales whose squares overflow or underflow, NaNs and infinities. Each is +//! checked by what defines it and against the textbook decomposition. + +use super::common::{ + Num, Rng, assert_close, assert_orthonormal, each_type, frobenius, random, test_each_type, +}; +use a::math::complex::Complex; +use a::math::matrix::Matrix; + +/// The textbook Householder QR of `a`, one reflection at a time as LAPACK's `geqr2` and `org2r`, +/// in `Complex`: `(Q, R)`, `Q` of `m × k` and `R` of `k × n`, `k = min(m, n)`, the same +/// reflections as the decomposition's, `β = -sign(Re α)·‖(α, x)‖`. +fn textbook(a: &Matrix) -> (Matrix>, Matrix>) { + let (m, n) = a.shape(); + let k = m.min(n); + let mut x = Matrix::from_fn(m, n, |i, j| a[(i, j)].c64()); + let mut taus = vec![Complex::new(0.0, 0.0); k]; + for j in 0..k { + let alpha = x[(j, j)]; + let xnorm = (j + 1..m).map(|i| x[(i, j)].norm_sqr()).sum::().sqrt(); + if xnorm == 0.0 && alpha.im == 0.0 { + continue; + } + let norm = alpha.abs().hypot(xnorm); + let beta = if alpha.re >= 0.0 { -norm } else { norm }; + let tau = Complex::new((beta - alpha.re) / beta, -alpha.im / beta); + let scale = Complex::new(1.0, 0.0) / (alpha - Complex::new(beta, 0.0)); + for i in j + 1..m { + x[(i, j)] *= scale; + } + x[(j, j)] = Complex::new(beta, 0.0); + taus[j] = tau; + for c in j + 1..n { + let mut w = x[(j, c)]; + for i in j + 1..m { + w += x[(i, j)].conj() * x[(i, c)]; + } + let w = tau.conj() * w; + x[(j, c)] -= w; + for i in j + 1..m { + let d = x[(i, j)] * w; + x[(i, c)] -= d; + } + } + } + let r = Matrix::from_fn(k, n, |i, j| { + if i <= j { + x[(i, j)] + } else { + Complex::new(0.0, 0.0) + } + }); + let mut q = Matrix::from_fn(m, k, |i, j| { + Complex::new(if i == j { 1.0 } else { 0.0 }, 0.0) + }); + for j in (0..k).rev() { + for c in j..k { + let mut w = q[(j, c)]; + for i in j + 1..m { + w += x[(i, j)].conj() * q[(i, c)]; + } + let w = taus[j] * w; + q[(j, c)] -= w; + for i in j + 1..m { + q[(i, c)] -= x[(i, j)] * w; + } + } + } + (q, r) +} + +/// `‖a - b‖` in the Frobenius norm, of a matrix of the type against one of `Complex`. +fn distance(a: &Matrix, b: &Matrix>) -> f64 { + assert_eq!(a.shape(), b.shape()); + let (m, n) = a.shape(); + let mut s = 0.0; + for j in 0..n { + for i in 0..m { + s += (a[(i, j)].c64() - b[(i, j)]).norm_sqr(); + } + } + s.sqrt() +} + +/// Checks the decomposition of `a`: the shapes, `R` upper triangular with a real diagonal, `Q` +/// and the whole `Q` orthonormal, the first the leading columns of the second, `Q·R = A`. +#[track_caller] +fn check(a: &Matrix, tol: f64) { + let (m, n) = a.shape(); + let k = m.min(n); + let qr = a.qr(); + let (q, r) = (qr.q(), qr.r()); + assert_eq!((q.shape(), r.shape()), ((m, k), (k, n))); + for i in 0..k { + assert_eq!(r[(i, i)].c64().im, 0.0, "{m}×{n}"); + for j in 0..i { + assert_eq!(r[(i, j)], T::real(0.0), "{m}×{n}"); + } + } + assert_orthonormal(&q, tol); + // The whole `Q` of a very tall matrix would take long to check. + if m <= 256 { + let full = qr.q_full(); + assert_eq!(full.shape(), (m, m)); + assert_orthonormal(&full, tol); + assert_close(&Matrix::from_fn(m, k, |i, j| full[(i, j)]), &q, tol); + } + let scale = frobenius(a); + let d = frobenius(&(&(&q * &r) - a)); + assert!( + d <= tol * T::EPS * (scale + f64::MIN_POSITIVE), + "‖QR - A‖ = {d:e}, ‖A‖ = {scale:e}, {m}×{n}" + ); +} + +/// A random `m × n` matrix far from rank deficient: random plus `2·min(m, n)` on the diagonal, +/// whose decomposition the rounding moves by little. +fn far_from_deficient(r: &mut Rng, m: usize, n: usize) -> Matrix { + let mut a = random::(r, m, n); + for i in 0..m.min(n) { + a[(i, i)] += T::real(2.0 * m.min(n) as f64); + } + a +} + +/// Checks the decomposition of `a`, far from rank deficient, also against the textbook one. +#[track_caller] +fn check_textbook(a: &Matrix) { + let (m, n) = a.shape(); + let tol = 16.0 * (m + n) as f64; + check(a, tol); + let qr = a.qr(); + let (q, r) = textbook(a); + let scale = frobenius(a); + let dr = distance(&qr.r(), &r); + assert!( + dr <= tol * T::EPS * scale, + "‖R - R₀‖ = {dr:e}, ‖A‖ = {scale:e}, {m}×{n}" + ); + let dq = distance(&qr.q(), &q); + let k = m.min(n) as f64; + assert!(dq <= tol * T::EPS * k.sqrt(), "‖Q - Q₀‖ = {dq:e}, {m}×{n}"); +} + +/// Whether Miri takes the shapes past the blocked path's threshold: on 64-bit targets only. On a +/// 32-bit one, where Miri never reuses stack addresses, they and their reference decomposition +/// run out of the address space; its small shapes are checked there, and its blocked products +/// by the product tests. +const LARGE_UNDER_MIRI: bool = cfg!(target_pointer_width = "64"); + +/// Shapes on both sides of the thresholds of every type, as the decomposition picks its path: +/// unblocked up to 4096 elements (and complex ones of at most 64 columns up to 32768), then +/// blocks of 8 columns, the panels' leaves; partial last blocks, wide and tall matrices. +fn shapes() -> Vec<(usize, usize)> { + if cfg!(miri) { + let mut s = vec![ + (0, 0), + (0, 3), + (3, 0), + (1, 1), + (1, 5), + (5, 1), + (2, 2), + (7, 4), + (4, 7), + ]; + if LARGE_UNDER_MIRI { + s.extend([(65, 64), (66, 65)]); + } + return s; + } + let mut s = vec![ + (0, 0), + (0, 3), + (3, 0), + (1, 1), + (1, 9), + (9, 1), + (2, 2), + (3, 3), + (4, 4), + ]; + s.extend([ + (64, 64), + (64, 65), + (65, 64), + (4096, 1), + (1, 4097), + (2, 2049), + (128, 32), + (129, 32), + ]); + s.extend([(512, 64), (513, 64), (500, 65), (64, 512)]); + s.extend([ + (96, 96), + (97, 97), + (100, 75), + (75, 100), + (200, 130), + (130, 400), + (40, 300), + ]); + s +} + +/// The type's shapes on both sides of where its blocks widen to 16, 24 and 32 columns, at +/// 384, 640 and 896 columns of `f64`, half as many of `f32` and a quarter as many complex ones. +fn wider_blocks() -> Vec<(usize, usize)> { + if cfg!(miri) { + return vec![]; + } + let complex = core::any::type_name::().contains("Complex"); + match (complex, T::EPS > 1e-10) { + (true, _) => vec![ + (96, 96), + (97, 97), + (160, 160), + (161, 161), + (224, 224), + (225, 225), + ], + (false, true) => vec![(192, 192), (193, 193)], + (false, false) => vec![], + } +} + +fn shapes_of() { + let mut r = Rng(0x5EED_0401); + for (m, n) in shapes().into_iter().chain(wider_blocks::()) { + let a = far_from_deficient::(&mut r, m, n); + // The textbook decomposition takes long for the larger ones, unoptimized. + if m * n * m.min(n) <= 1 << 20 { + check_textbook(&a); + } else { + check(&a, 16.0 * (m + n) as f64); + } + if m * n <= 1 << 15 { + let a = random::(&mut r, m, n); + check(&a, 16.0 * (m + n) as f64); + } + } +} + +test_each_type!(shapes_across_thresholds, shapes_of); + +#[test] +fn tall_blocks() { + // The wider blocks of tall real matrices, four times as many rows as columns: 16 columns + // from 512 rows, 48 from 2048 and 64 from 4096; and the widest by the columns, in `f64`. + let mut r = Rng(0x5EED_0402); + let shapes = if cfg!(miri) { + vec![] + } else { + vec![ + (511, 80), + (512, 80), + (2047, 40), + (2048, 40), + (4095, 30), + (4096, 30), + (384, 384), + (385, 385), + ] + }; + for (m, n) in shapes { + check( + &far_from_deficient::(&mut r, m, n), + 16.0 * (m + n) as f64, + ); + } +} + +/// Checks `X` of the least squares `‖A·X - B‖`: the residual orthogonal to the columns of `A`. +#[track_caller] +fn check_least_squares(a: &Matrix, b: &Matrix) { + let (m, n) = a.shape(); + let x = a.qr().solve(b).unwrap(); + assert_eq!(x.shape(), (n, b.cols())); + let residual = b - &(a * &x); + let g = &a.conjugate_transpose() * &residual; + let scale = frobenius(a) * frobenius(b) + 1.0; + assert!( + frobenius(&g) <= 64.0 * (m + 1) as f64 * T::EPS * scale, + "{m}×{n}, {} columns", + b.cols() + ); +} + +fn least_squares_of() { + let mut r = Rng(0x5EED_0403); + // Unblocked and blocked decompositions, the right-hand side's columns from none to a few + // over where the reflections go four at a time for each column, 4, and `R` of up to 128 + // columns and more, solved by substitution and by the triangular solve. + let shapes: &[(usize, usize)] = if cfg!(miri) { + &[(5, 3), (70, 66)] + } else { + &[ + (5, 3), + (40, 40), + (100, 30), + (64, 64), + (65, 65), + (150, 100), + (300, 40), + (600, 100), + (200, 128), + (200, 129), + ] + }; + let cols: &[usize] = if cfg!(miri) { + &[0, 1, 5] + } else { + &[0, 1, 2, 3, 4, 5, 9] + }; + for &(m, n) in shapes { + let a = random::(&mut r, m, n); + for &c in cols { + let b = random::(&mut r, m, c); + check_least_squares(&a, &b); + } + } + // A square system, solved. + let a = far_from_deficient::(&mut r, 50, 50); + let x = random::(&mut r, 50, 3); + assert_close(&a.qr().solve(&(&a * &x)).unwrap(), &x, 64.0 * 50.0); +} + +#[test] +fn least_squares() { + each_type!(least_squares_of); +} + +fn deficient_of() { + let mut r = Rng(0x5EED_0404); + let sizes: &[(usize, usize)] = if cfg!(miri) && !LARGE_UNDER_MIRI { + &[(6, 4)] + } else if cfg!(miri) { + &[(6, 4), (70, 66)] + } else { + &[(6, 4), (40, 40), (70, 66), (150, 100), (100, 150)] + }; + for &(m, n) in sizes { + // Zero columns, first, inside and last: a zero on the diagonal of `R`, and no least + // squares. + for j in [0, n / 2, n - 1] { + let mut a = random::(&mut r, m, n); + for i in 0..m { + a[(i, j)] = T::real(0.0); + } + check(&a, 16.0 * (m + n) as f64); + if j < m && m >= n { + assert_eq!(a.qr().r()[(j, j)], T::real(0.0)); + assert!(a.qr().solve(&random::(&mut r, m, 1)).is_none()); + } + } + // A zero matrix: `R = 0`, `Q = I`. + let z = Matrix::::zeros(m, n); + let qr = z.qr(); + assert_eq!(qr.r(), Matrix::zeros(m.min(n), n)); + assert_eq!(qr.q_full(), Matrix::identity(m)); + // Rank one, and repeated columns. + let u = random::(&mut r, m, 1); + let v = random::(&mut r, 1, n); + check(&(&u * &v), 16.0 * (m + n) as f64); + let mut a = random::(&mut r, m, n); + for i in 0..m { + a[(i, n - 1)] = a[(i, 0)]; + } + check(&a, 16.0 * (m + n) as f64); + } +} + +test_each_type!(zero_and_dependent_columns, deficient_of); + +fn scaled_of() { + // Scaled by powers of two so far that the squares of the elements overflow or underflow: + // the same decomposition, scaled; and columns of both scales together. + let mut r = Rng(0x5EED_0405); + let e = if T::EPS > 1e-10 { 70 } else { 520 }; + let sizes: &[(usize, usize)] = if cfg!(miri) { + &[(7, 5), (70, 66)] + } else { + &[(7, 5), (40, 40), (70, 66), (150, 100), (60, 120)] + }; + for &(m, n) in sizes { + let a = far_from_deficient::(&mut r, m, n); + let (q0, r0) = (a.qr().q(), a.qr().r()); + for k in [e, -e] { + let s = T::real(2f64.powi(k)); + let b = a.map(|&x| x * s); + let qr = b.qr(); + assert_close(&qr.q(), &q0, 16.0 * (m + n) as f64); + assert_close( + &qr.r().map(|&x| x * T::real(2f64.powi(-k))), + &r0, + 16.0 * (m + n) as f64, + ); + } + let mut c = random::(&mut r, m, n); + for j in 0..n { + let s = T::real(2f64.powi(if j % 2 == 0 { e } else { -e })); + for i in 0..m { + c[(i, j)] *= s; + } + } + let qr = c.qr(); + assert_orthonormal(&qr.q(), 16.0 * (m + n) as f64); + assert!(qr.r().as_slice().iter().all(|x| x.c64().re.is_finite())); + } +} + +test_each_type!(scaled_to_the_ends_of_the_range, scaled_of); + +fn non_finite_of() { + let mut r = Rng(0x5EED_0406); + let sizes: &[(usize, usize)] = if cfg!(miri) { + &[(3, 3), (70, 66)] + } else { + &[ + (3, 3), + (50, 50), + (70, 66), + (150, 100), + (100, 150), + (600, 40), + ] + }; + for &(m, n) in sizes { + for bad in [f64::NAN, f64::INFINITY] { + let mut a = random::(&mut r, m, n); + a[(0, 0)] = T::real(bad); + let qr = a.qr(); + // The first column's reflection takes the NaN or the infinity into everything. + let nan = |x: &T| x.c64().re.is_nan() || x.c64().im.is_nan(); + assert!(qr.r().as_slice().iter().any(nan), "{m}×{n}"); + assert!(qr.q().as_slice().iter().any(nan), "{m}×{n}"); + if m >= n { + let x = qr.solve(&random::(&mut r, m, 2)).unwrap(); + assert!(x.as_slice().iter().any(nan), "{m}×{n}"); + } + } + } +} + +#[test] +fn nan_and_infinity() { + each_type!(non_finite_of); +} + +fn full_q_of() { + // The whole `Q` of tall matrices, formed in one block from 48 rows and four times as many + // rows as columns: its factor joined from blocks of 64 for more columns. `Q*·A = [R; 0]`. + let mut r = Rng(0x5EED_0407); + let shapes: &[(usize, usize)] = if cfg!(miri) { + &[(47, 4), (48, 4), (70, 17)] + } else { + &[ + (47, 4), + (48, 4), + (127, 32), + (128, 32), + (290, 70), + (300, 64), + (530, 130), + ] + }; + for &(m, n) in shapes { + let a = far_from_deficient::(&mut r, m, n); + let qr = a.qr(); + let (q, full) = (qr.q(), qr.q_full()); + let tol = 16.0 * (m + n) as f64; + assert_orthonormal(&full, tol); + assert_close(&Matrix::from_fn(m, n, |i, j| full[(i, j)]), &q, tol); + let mut rr = Matrix::::zeros(m, n); + let r = qr.r(); + for j in 0..n { + for i in 0..=j { + rr[(i, j)] = r[(i, j)]; + } + } + assert_close(&(&full.conjugate_transpose() * &a), &rr, tol); + let (q0, _) = textbook(&a); + let dq = distance(&q, &q0); + assert!( + dq <= tol * T::EPS * (n as f64).sqrt(), + "‖Q - Q₀‖ = {dq:e}, {m}×{n}" + ); + } +} + +#[test] +fn full_q_of_tall_matrices() { + each_type!(full_q_of); +} diff --git a/lib/tests/math/matrix/svd.rs b/lib/tests/math/matrix/svd.rs new file mode 100644 index 0000000..8e05c36 --- /dev/null +++ b/lib/tests/math/matrix/svd.rs @@ -0,0 +1,221 @@ +//! Singular value decompositions, norms, condition numbers, ranks, null spaces and +//! characteristic polynomials. + +use super::common::{ + Num, Rng, assert_close, assert_orthonormal, each_type, frobenius, random, well_conditioned, +}; +use a::math::complex::Float; +use a::math::matrix::{Matrix, Norm}; + +/// Shapes around the panels' width, tall, wide and square. +fn shapes() -> Vec<(usize, usize)> { + if cfg!(miri) { + vec![(0, 0), (1, 1), (3, 2), (2, 3), (5, 5)] + } else { + let mut v = vec![(0, 0), (0, 3), (3, 0), (1, 1), (1, 5), (5, 1)]; + for n in [2, 3, 7, 16, 33, 64] { + v.extend([(n, n), (n + 5, n), (n, n + 5), (2 * n + 1, n)]); + } + v + } +} + +/// A real value as an `f64`. +fn f(x: R) -> f64 { + x.to_f64() +} + +fn svd_of() { + let mut r = Rng(0x5EED_0201); + for (m, n) in shapes() { + let a = random::(&mut r, m, n); + let svd = a.svd(); + let (u, v) = (svd.u(), svd.v()); + let s: Vec = svd.singular_values().iter().map(|&x| f(x)).collect(); + let k = m.min(n); + assert_eq!((u.shape(), v.shape(), s.len()), ((m, k), (n, k), k)); + let tol = 16.0 * (m + n + 1) as f64; + assert_orthonormal(&u, tol); + assert_orthonormal(&v, tol); + assert!(s.windows(2).all(|p| p[0] >= p[1]) && s.iter().all(|&x| x >= 0.0)); + let sigma = Matrix::from_diagonal(&s.iter().map(|&x| T::real(x)).collect::>()); + assert_close(&(&(&u * &sigma) * &v.conjugate_transpose()), &a, tol); + // Without the vectors, the same values. + let alone: Vec = a.singular_values().iter().map(|&x| f(x)).collect(); + let scale = frobenius(&a) + 1.0; + for (x, y) in alone.iter().zip(&s) { + assert!((x - y).abs() <= tol * T::EPS * scale); + } + // The squares are the eigenvalues of `A*·A`, by the Hermitian algorithm. + let g = &a.conjugate_transpose() * &a; + let mut w: Vec = g.hermitian_eigenvalues().iter().map(|&x| f(x)).collect(); + w.reverse(); + for (x, w) in s.iter().zip(&w) { + assert!( + (x * x - w).abs() <= 4.0 * tol * T::EPS * scale * scale, + "{m} {n}" + ); + } + } +} + +#[test] +fn svd() { + each_type!(svd_of); +} + +/// A random `m × n` matrix of rank `r`. +fn of_rank(r: &mut Rng, m: usize, n: usize, rank: usize) -> Matrix { + &random::(r, m, rank) * &random::(r, rank, n) +} + +fn rank_of() { + let mut r = Rng(0x5EED_0202); + for (m, n, rank) in [ + (5, 5, 5), + (5, 5, 3), + (8, 4, 2), + (4, 8, 3), + (20, 30, 7), + (6, 6, 0), + ] { + let a = of_rank::(&mut r, m, n, rank); + assert_eq!(a.rank(), rank, "{m} {n} {rank}"); + let null = a.null_space(); + assert_eq!(null.shape(), (n, n - rank)); + assert_orthonormal(&null, 64.0); + let an = &a * &null; + assert!( + frobenius(&an) <= 1e3 * T::EPS * (frobenius(&a) + 1.0), + "{m} {n} {rank}" + ); + } + assert_eq!(Matrix::::identity(4).rank(), 4); + assert_eq!(Matrix::::zeros(3, 4).rank(), 0); + assert_eq!(Matrix::::zeros(3, 4).null_space().shape(), (4, 4)); +} + +#[test] +fn rank_and_null_space() { + each_type!(rank_of); +} + +fn norms_of() { + let mut r = Rng(0x5EED_0203); + for (m, n) in shapes() { + let a = random::(&mut r, m, n); + let s: Vec = a.singular_values().iter().map(|&x| f(x)).collect(); + let abs = |i: usize, j: usize| a[(i, j)].c64().abs(); + let one = (0..n) + .map(|j| (0..m).map(|i| abs(i, j)).sum::()) + .fold(0.0, f64::max); + let inf = (0..m) + .map(|i| (0..n).map(|j| abs(i, j)).sum::()) + .fold(0.0, f64::max); + let max = (0..m) + .flat_map(|i| (0..n).map(move |j| (i, j))) + .map(|(i, j)| abs(i, j)); + let max = max.fold(0.0, f64::max); + let tol = 16.0 * (m + n + 1) as f64 * T::EPS; + let close = |x: f64, y: f64| (x - y).abs() <= tol * (y.abs() + T::EPS); + assert!(close(f(a.norm(Norm::One)), one)); + assert!(close(f(a.norm(Norm::Infinity)), inf)); + assert!(close(f(a.norm(Norm::Max)), max)); + assert!(close(f(a.norm(Norm::Frobenius)), frobenius(&a))); + assert!(close( + f(a.norm(Norm::Two)), + s.first().copied().unwrap_or(0.0) + )); + assert!(close(f(a.norm(Norm::Nuclear)), s.iter().sum())); + let fro: f64 = s.iter().map(|x| x * x).sum::().sqrt(); + assert!(close(fro, frobenius(&a))); + } +} + +#[test] +fn norms() { + each_type!(norms_of); +} + +fn cond_of() { + let mut r = Rng(0x5EED_0204); + let i = Matrix::::identity(5); + for norm in [ + Norm::One, + Norm::Two, + Norm::Infinity, + Norm::Max, + Norm::Nuclear, + ] { + let c = f(i.cond(norm)); + let want = match norm { + Norm::Nuclear => 25.0, + _ => 1.0, + }; + assert!((c - want).abs() <= 64.0 * T::EPS * want, "{norm:?} {c}"); + } + assert!((f(i.cond(Norm::Frobenius)) - 5.0).abs() <= 64.0 * T::EPS * 5.0); + let a = well_conditioned::(&mut r, 12); + let s: Vec = a.singular_values().iter().map(|&x| f(x)).collect(); + let k = s[0] / s[11]; + assert!((f(a.cond(Norm::Two)) - k).abs() <= 1e3 * T::EPS * k); + let inv = a.inverse().unwrap(); + for norm in [Norm::One, Norm::Infinity, Norm::Frobenius] { + let want = f(a.norm(norm)) * f(inv.norm(norm)); + assert!((f(a.cond(norm)) - want).abs() <= 1e3 * T::EPS * want); + } + let singular = Matrix::::zeros(3, 3); + for norm in [ + Norm::One, + Norm::Two, + Norm::Infinity, + Norm::Frobenius, + Norm::Nuclear, + ] { + assert!(f(singular.cond(norm)).is_infinite()); + } + // In the 2-norm, of any shape. + let tall = random::(&mut r, 9, 4); + let s: Vec = tall.singular_values().iter().map(|&x| f(x)).collect(); + assert!((f(tall.cond(Norm::Two)) - s[0] / s[3]).abs() <= 1e3 * T::EPS * s[0] / s[3]); +} + +#[test] +fn condition_numbers() { + each_type!(cond_of); +} + +fn characteristic_of() { + let mut r = Rng(0x5EED_0205); + for n in [0, 1, 2, 3, 5, 8, 12] { + // Scaled so that the powers stay near 1. + let a = random::(&mut r, n, n) * T::real(1.0 / (n as f64).sqrt().max(1.0)); + let p = a.characteristic_polynomial(); + assert_eq!(p.len(), n + 1); + assert_eq!(p[0], T::real(1.0)); + // Cayley and Hamilton: `p(A) = 0`, by Horner's rule. + let mut x = Matrix::::zeros(n, n); + for &c in &p { + x = &x * &a; + for i in 0..n { + x[(i, i)] += c; + } + } + assert!( + frobenius(&x) <= 1e3 * (n + 1) as f64 * T::EPS, + "{n}: {:e}", + frobenius(&x) + ); + if n > 0 { + let tol = 1e3 * T::EPS * (frobenius(&a) + 1.0); + assert!((p[1].c64() + a.trace().c64()).abs() <= tol); + let det = a.det().c64() * if n % 2 == 1 { -1.0 } else { 1.0 }; + assert!((p[n].c64() - det).abs() <= tol); + } + } +} + +#[test] +fn characteristic_polynomials() { + each_type!(characteristic_of); +} diff --git a/lib/tests/math/matrix/svd_paths.rs b/lib/tests/math/matrix/svd_paths.rs new file mode 100644 index 0000000..c25250b --- /dev/null +++ b/lib/tests/math/matrix/svd_paths.rs @@ -0,0 +1,509 @@ +//! Singular value decompositions across the thresholds of the algorithms: divide and conquer and +//! its leaves, the panels of the reduction, QR first for tall matrices; and hard cases: low rank, +//! repeated, clustered, graded and tiny singular values, zeros. + +use super::common::{ + Num, Rng, assert_close, assert_orthonormal, each_type, frobenius, random, test_each_type, +}; +use a::math::complex::Float; +use a::math::matrix::Matrix; + +/// A real value as an `f64`. +fn f(x: R) -> f64 { + x.to_f64() +} + +/// Checks the decomposition of `a`: the shapes, `U` and `V` orthonormal, `U·Σ·V* = A`, the values +/// descending and not negative, the same values without the vectors; returns the values. +#[track_caller] +fn check(a: &Matrix, tol: f64) -> Vec { + let (m, n) = a.shape(); + let k = m.min(n); + let svd = a.svd(); + let (u, v) = (svd.u(), svd.v()); + let s: Vec = svd.singular_values().iter().map(|&x| f(x)).collect(); + assert_eq!((u.shape(), v.shape(), s.len()), ((m, k), (n, k), k)); + assert_orthonormal(&u, tol); + assert_orthonormal(&v, tol); + assert!( + s.windows(2).all(|p| p[0] >= p[1]) && s.iter().all(|&x| x >= 0.0), + "{s:?}" + ); + let sigma = Matrix::from_diagonal(&s.iter().map(|&x| T::real(x)).collect::>()); + let back = &(&u * &sigma) * &v.conjugate_transpose(); + let scale = frobenius(a); + let d = frobenius(&(&back - a)); + assert!( + d <= tol * T::EPS * (scale + f64::MIN_POSITIVE), + "‖UΣV* - A‖ = {d:e}, ‖A‖ = {scale:e}, {m}×{n}" + ); + let alone: Vec = a.singular_values().iter().map(|&x| f(x)).collect(); + for (x, y) in alone.iter().zip(&s) { + assert!((x - y).abs() <= tol * T::EPS * scale, "{x} {y} {m}×{n}"); + } + s +} + +/// A random matrix with orthonormal columns, `m × k`. +fn orthonormal(r: &mut Rng, m: usize, k: usize) -> Matrix { + random::(r, m, k).qr().q() +} + +/// `Q₁·diag(s)·Q₂*` for random `Q₁` of `m × k` and `Q₂` of `n × k`, `k = s.len()`. +fn with_values(r: &mut Rng, m: usize, n: usize, s: &[f64]) -> Matrix { + let k = s.len(); + let (q1, q2) = (orthonormal::(r, m, k), orthonormal::(r, n, k)); + let sigma = Matrix::from_diagonal(&s.iter().map(|&x| T::real(x)).collect::>()); + &(&q1 * &sigma) * &q2.conjugate_transpose() +} + +/// The shapes of [`shapes_of`] under Miri: below every crossover, as the large paths take hours +/// there and have no unsafe code of their own; the products they call are the product tests'. +const MIRI_SHAPES: [(usize, usize); 5] = [(1, 1), (6, 6), (9, 4), (4, 9), (13, 3)]; + +fn shapes_of() { + let mut r = Rng(0x5EED_0301); + if cfg!(miri) { + for (m, n) in MIRI_SHAPES { + let a = random::(&mut r, m, n); + check(&a, 32.0 * (m + n) as f64); + } + return; + } + let mut shapes = vec![(24, 24), (25, 25), (26, 26), (31, 31), (32, 32), (33, 33)]; + shapes.extend([ + (51, 51), + (52, 52), + (65, 65), + (100, 100), + (130, 97), + (97, 130), + ]); + // About the ratios where QR goes first, 1.75 for the values alone and 2.5 with the vectors, + // and very tall and wide ones. + shapes.extend([ + (52, 32), + (56, 32), + (57, 32), + (79, 32), + (80, 32), + (81, 32), + (32, 80), + ]); + shapes.extend([(300, 20), (20, 300), (400, 7), (80, 79), (1000, 40)]); + for (m, n) in shapes { + let a = random::(&mut r, m, n); + check(&a, 32.0 * (m + n) as f64); + } + // Deep trees, of both kinds of halves. + for n in [150, 201] { + let a = random::(&mut r, n, n); + check(&a, 32.0 * n as f64); + } +} + +test_each_type!(shapes_across_thresholds, shapes_of); + +fn low_rank_of() { + let mut r = Rng(0x5EED_0302); + let cases: &[(usize, usize, usize)] = if cfg!(miri) { + &[(12, 12, 3), (14, 9, 0), (9, 14, 1)] + } else { + &[ + (60, 60, 10), + (120, 100, 30), + (100, 120, 1), + (90, 90, 0), + (200, 60, 59), + ] + }; + for &(m, n, rank) in cases { + let a = &random::(&mut r, m, rank) * &random::(&mut r, rank, n); + let s = check(&a, 32.0 * (m + n) as f64); + let scale = frobenius(&a) + 1.0; + assert!( + s[rank..] + .iter() + .all(|&x| x <= 64.0 * (m + n) as f64 * T::EPS * scale), + "{m} {n} {rank}" + ); + assert_eq!(a.rank(), rank); + } +} + +test_each_type!(low_rank, low_rank_of); + +fn repeated_of() { + let mut r = Rng(0x5EED_0303); + if cfg!(miri) { + repeated_at::(&mut r, 9, 2); + } else { + repeated_at::(&mut r, 90, 3); + } +} + +/// Repeated, clustered, graded and tiny singular values of matrices with `n` columns, of the +/// first `shapes` of: square, tall, and tall past the ratio where QR goes first. +fn repeated_at(r: &mut Rng, n: usize, shapes: usize) { + let third = n.div_ceil(3); + let cases: Vec> = vec![ + // Repeated: three values, a third of the times each. + (0..n).map(|i| [3.0, 2.0, 1.0][i / third]).collect(), + // All equal. + vec![1.0; n], + // Clustered within 1e-9, and within ε. + (0..n).map(|i| 1.0 + 1e-9 * i as f64).collect(), + (0..n).map(|i| 1.0 + T::EPS * (i % 3) as f64).collect(), + // Graded over many orders, and some tiny. + (0..n).map(|i| 10f64.powf(-(i as f64) / 8.0)).collect(), + (0..n) + .map(|i| if i < 2 * n / 3 { 1.0 } else { 1e-30 }) + .collect(), + ]; + for (c, s) in cases.iter().enumerate() { + for &(m, k) in [(n, n), (n + 4 * n / 9, n), (2 * n + n / 9, n)] + .iter() + .take(shapes) + { + let a = with_values::(r, m, k, s); + let got = check(&a, 64.0 * (m + k) as f64); + let mut want = s.clone(); + want.sort_by(|x, y| y.total_cmp(x)); + let scale = want[0]; + for (g, w) in got.iter().zip(&want) { + // Absolute accuracy, to `ε·σ₁`. + assert!( + (g - w).abs() <= 64.0 * (m + k) as f64 * T::EPS * scale, + "case {c}: {g} {w}" + ); + } + } + } +} + +#[test] +fn repeated_clustered_graded() { + each_type!(repeated_of); +} + +fn exact_of() { + let mut r = Rng(0x5EED_0304); + // Zero matrices, of every path. + let zeros: &[(usize, usize)] = if cfg!(miri) { + &[(5, 5), (9, 4), (4, 9)] + } else { + &[(5, 5), (40, 40), (100, 30), (30, 100)] + }; + for &(m, n) in zeros { + let a = Matrix::::zeros(m, n); + let s = check(&a, 1.0); + assert!(s.iter().all(|&x| x == 0.0)); + } + // A permuted diagonal, its values over the whole range of the type: exact, each to its own + // relative accuracy. + for &n in if cfg!(miri) { &[10][..] } else { &[10, 60][..] } { + let lo = if T::EPS > 1e-10 { -37 } else { -300 }; + let vals: Vec = (0..n) + .map(|i| 10f64.powi(lo * i as i32 / n as i32) * (1.0 + 0.1 * r.unit())) + .collect(); + let mut perm: Vec = (0..n).collect(); + for i in (1..n).rev() { + perm.swap(i, (r.next() % (i as u64 + 1)) as usize); + } + let a = Matrix::from_fn(n, n, |i, j| { + if perm[i] == j { + T::real(vals[i]) + } else { + T::real(0.0) + } + }); + let got = check(&a, 16.0 * n as f64); + let mut want: Vec = vals.iter().map(|&x| T::real(x).c64().re).collect(); + want.sort_by(|x, y| y.total_cmp(x)); + for (g, w) in got.iter().zip(&want) { + assert!((g - w).abs() <= 4.0 * T::EPS * w, "{g:e} {w:e}"); + } + } + // An infinity or a NaN, on every path. + let non_finite: &[(usize, usize)] = if cfg!(miri) { + &[(3, 3), (9, 4), (4, 9)] + } else { + &[(3, 3), (50, 50), (100, 20), (20, 100)] + }; + for &(m, n) in non_finite { + let mut a = random::(&mut r, m, n); + a[(m / 2, n / 2)] = T::real(f64::NAN); + assert!(a.singular_values().iter().all(|x| x.is_nan())); + let svd = a.svd(); + assert!(svd.singular_values().iter().all(|x| x.is_nan())); + } +} + +#[test] +fn zeros_diagonals_and_non_finite() { + each_type!(exact_of); +} + +fn null_space_of() { + // `null_space` takes all `n` right singular vectors, padding a wide matrix with zero rows. + let mut r = Rng(0x5EED_0305); + let cases: &[(usize, usize, usize)] = if cfg!(miri) { + &[(5, 12, 5), (10, 10, 6), (14, 6, 4)] + } else { + &[(30, 80, 30), (70, 70, 40), (120, 40, 25)] + }; + for &(m, n, rank) in cases { + let a = &random::(&mut r, m, rank) * &random::(&mut r, rank, n); + let null = a.null_space(); + assert_eq!(null.shape(), (n, n - rank)); + assert_orthonormal(&null, 64.0); + assert_close( + &(&a * &null), + &Matrix::zeros(m, n - rank), + 1e3 * (frobenius(&a) + 1.0), + ); + } +} + +#[test] +fn null_spaces() { + each_type!(null_space_of); +} + +fn scaled_of() { + // Scaled by powers of two so far that the squares of the elements overflow or underflow: + // the same decomposition, scaled. + let mut r = Rng(0x5EED_0306); + let e = if T::EPS > 1e-10 { 80 } else { 560 }; + let shapes: &[(usize, usize)] = if cfg!(miri) { + &[(10, 10), (14, 6)] + } else { + &[(40, 40), (90, 60), (150, 30), (30, 90)] + }; + for &(m, n) in shapes { + let a = random::(&mut r, m, n); + let s = check(&a, 32.0 * (m + n) as f64); + for k in [e, -e] { + let scale = 2f64.powi(k); + let b = a.map(|&x| x * T::real(scale)); + let svd = b.svd(); + let (u, v) = (svd.u(), svd.v()); + assert_orthonormal(&u, 32.0 * (m + n) as f64); + assert_orthonormal(&v, 32.0 * (m + n) as f64); + let sb: Vec = svd + .singular_values() + .iter() + .map(|&x| f(x) / scale) + .collect(); + let alone: Vec = b.singular_values().iter().map(|&x| f(x) / scale).collect(); + for ((x, y), z) in sb.iter().zip(&s).zip(&alone) { + assert!( + (x - y).abs() <= 64.0 * (m + n) as f64 * T::EPS * s[0], + "{k}: {x} {y}" + ); + assert!( + (z - y).abs() <= 64.0 * (m + n) as f64 * T::EPS * s[0], + "{k}: {z} {y}" + ); + } + let sigma = Matrix::from_diagonal(&sb.iter().map(|&x| T::real(x)).collect::>()); + assert_close( + &(&(&u * &sigma) * &v.conjugate_transpose()), + &a, + 32.0 * (m + n) as f64, + ); + } + } +} + +test_each_type!(scaled_to_the_ends_of_the_range, scaled_of); + +fn blocks_of() { + // Block diagonal: the bidiagonal matrix splits where the blocks meet, into problems each + // solved by divide and conquer; the values are those of the blocks together. + let mut r = Rng(0x5EED_0307); + let cases: &[[usize; 3]] = if cfg!(miri) { + &[[4, 3, 6], [2, 7, 1]] + } else { + &[[40, 30, 70], [26, 90, 1]] + }; + for sizes in cases { + let blocks: Vec> = sizes.iter().map(|&k| random::(&mut r, k, k)).collect(); + let n: usize = sizes.iter().sum(); + let mut a = Matrix::::zeros(n, n); + let mut want = Vec::new(); + let mut at = 0; + for b in &blocks { + let k = b.rows(); + for i in 0..k { + for j in 0..k { + a[(at + i, at + j)] = b[(i, j)]; + } + } + want.extend(b.singular_values().iter().map(|&x| f(x))); + at += k; + } + want.sort_by(|x, y| y.total_cmp(x)); + let got = check(&a, 32.0 * n as f64); + for (g, w) in got.iter().zip(&want) { + assert!( + (g - w).abs() <= 64.0 * n as f64 * T::EPS * want[0], + "{g} {w}" + ); + } + } +} + +test_each_type!(block_diagonal, blocks_of); + +fn graded_bidiagonal_of() { + // A bidiagonal matrix passes through the reduction as it is, and dqds finds its singular + // values to high relative accuracy, the tiny ones too: those of the lower precisions agree + // with the higher's, each to a few units of its own size. + let mut r = Rng(0x5EED_0308); + for (n, step) in [(30, -1.0), (60, -0.5), (45, 0.7)] { + let a = Matrix::::from_fn(n, n, |i, j| { + let g = 10f64.powf(step * i as f64 * if T::EPS > 1e-10 { 0.5 } else { 1.0 }); + if j == i || j == i + 1 { + T::random(&mut r) * T::real(g) + } else { + T::real(0.0) + } + }); + let wide = Matrix::from_fn(n, n, |i, j| a[(i, j)].c64()); + let want: Vec = wide.singular_values(); + let got: Vec = a.singular_values().iter().map(|&x| f(x)).collect(); + for (g, w) in got.iter().zip(&want) { + assert!( + (g - w).abs() <= 16.0 * n as f64 * T::EPS * w, + "{n}: {g:e} {w:e}" + ); + } + } +} + +#[test] +fn graded_bidiagonal_relative_accuracy() { + each_type!(graded_bidiagonal_of); +} + +/// The numbers of columns of the small path, on both sides of the switches of its scratch at 16 +/// and 48 and of the crossover of single precision complex vectors at 88. +fn small_sizes() -> Vec { + if cfg!(miri) { + vec![2, 5, 17] + } else { + vec![1, 2, 3, 5, 16, 17, 40, 48, 49, 80, 88, 89] + } +} + +/// The numbers of columns about the other crossovers of the small path, with the vectors and +/// without: 96 and 112 for `f64`, 112 and 96 for `f32`, 96 and 192 for `Complex`, 192 for +/// `Complex`. +fn crossovers() -> Vec { + if cfg!(miri) { + return Vec::new(); + } + match core::any::type_name::() { + "f64" => vec![96, 97, 112, 113], + "f32" => vec![112, 113, 96, 97], + "a::math::complex::Complex" => vec![96, 97, 192, 193], + _ => vec![192, 193], + } +} + +fn small_shapes_of() { + let mut r = Rng(0x5EED_0309); + // About the crossovers: square, which `check` compares with the values alone. + for n in crossovers::() { + check(&random::(&mut r, n, n), 64.0 * n as f64); + } + for n in small_sizes() { + // Square, tall below and from the ratios where QR goes first (1.75 for the values alone, + // 2.5 with the vectors), and wide; the larger only about the first. + let a = (7 * n).div_ceil(4); + let b = (5 * n).div_ceil(2); + let rows = if n <= 40 { + vec![n, a - 1, a, b - 1, b] + } else { + vec![n] + }; + for m in rows { + let a = random::(&mut r, m, n); + check(&a, 32.0 * (m + n) as f64); + check(&a.conjugate_transpose(), 32.0 * (m + n) as f64); + } + // Low rank, square, wide and tall. + for (m, k, rank) in [(n, n, n / 3), (n, n + 7, n / 2), (n + 7, n, 1)] { + let a = &random::(&mut r, m, rank) * &random::(&mut r, rank, k); + let s = check(&a, 32.0 * (m + k) as f64); + let scale = frobenius(&a) + 1.0; + assert!( + s[rank..] + .iter() + .all(|&x| x <= 64.0 * (m + k) as f64 * T::EPS * scale), + "{m} {k} {rank}" + ); + } + // Zero, and not finite. + let s = check(&Matrix::::zeros(n, n), 1.0); + assert!(s.iter().all(|&x| x == 0.0)); + let mut a = random::(&mut r, n + 1, n); + a[(n / 2, n / 2)] = T::real(f64::INFINITY); + assert!(a.singular_values().iter().all(|x| x.is_nan())); + let svd = a.svd(); + assert!(svd.singular_values().iter().all(|x| x.is_nan())); + assert!(svd.u().as_slice().iter().all(|x| x.c64().re.is_nan())); + // Near the ends of the range: the same decomposition, scaled. + let a = random::(&mut r, n + 2, n); + let s = check(&a, 32.0 * (2 * n + 2) as f64); + let e = if T::EPS > 1e-10 { 80 } else { 560 }; + for k in [e, -e] { + let scale = 2f64.powi(k); + let b = a.map(|&x| x * T::real(scale)); + let sb: Vec = b.singular_values().iter().map(|&x| f(x) / scale).collect(); + for (x, y) in sb.iter().zip(&s) { + assert!( + (x - y).abs() <= 64.0 * (2 * n + 2) as f64 * T::EPS * s[0], + "{k}: {x} {y}" + ); + } + let svd = b.svd(); + assert_orthonormal(&svd.u(), 32.0 * (2 * n + 2) as f64); + assert_orthonormal(&svd.v(), 32.0 * (2 * n + 2) as f64); + } + } +} + +#[test] +fn small_shapes_and_hard_cases() { + each_type!(small_shapes_of); +} + +fn small_repeated_of() { + // Repeated, clustered and graded values on both sides of the switches of the scratch and of + // the first crossover of each type. + let mut r = Rng(0x5EED_0310); + let mut sizes = if cfg!(miri) { + vec![(5, 3)] + } else { + vec![(5, 3), (16, 3), (17, 3), (48, 2), (49, 2)] + }; + if !cfg!(miri) { + // About the crossover with the vectors. + let n = match core::any::type_name::() { + "f32" => 112, + "a::math::complex::Complex" => 88, + _ => 96, + }; + sizes.extend([(n, 1), (n + 1, 1)]); + } + for (n, shapes) in sizes { + repeated_at::(&mut r, n, shapes); + } +} + +#[test] +fn small_repeated_clustered_graded() { + each_type!(small_repeated_of); +}