1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
//! Architecture-specific support for aarch64 with neon.
use core::arch::asm;
pub fn fma(mut x: f64, y: f64, z: f64) -> f64 {
// SAFETY: `fmadd` is available with neon and has no side effects.
unsafe {
asm!(
"fmadd {x:d}, {x:d}, {y:d}, {z:d}",
x = inout(vreg) x,
y = in(vreg) y,
z = in(vreg) z,
options(nomem, nostack, pure)
);
}
x
}
pub fn fmaf(mut x: f32, y: f32, z: f32) -> f32 {
// SAFETY: `fmadd` is available with neon and has no side effects.
unsafe {
asm!(
"fmadd {x:s}, {x:s}, {y:s}, {z:s}",
x = inout(vreg) x,
y = in(vreg) y,
z = in(vreg) z,
options(nomem, nostack, pure)
);
}
x
}
// NB: `frintx` is technically the correct instruction for C's `rint`. However, in Rust (and LLVM
// by default), `rint` is identical to `roundeven` (no fpenv interaction) so we use the
// side-effect-free `frintn`.
//
// In general, C code that calls Rust's libm should assume that fpenv is ignored.
pub fn rint(mut x: f64) -> f64 {
// SAFETY: `frintn` is available with neon and has no side effects.
//
// `frintn` is always round-to-nearest which does not match the C specification, but Rust does
// not support rounding modes.
unsafe {
asm!(
"frintn {x:d}, {x:d}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
pub fn rintf(mut x: f32) -> f32 {
// SAFETY: `frintn` is available with neon and has no side effects.
//
// `frintn` is always round-to-nearest which does not match the C specification, but Rust does
// not support rounding modes.
unsafe {
asm!(
"frintn {x:s}, {x:s}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
#[cfg(all(f16_enabled, target_feature = "fp16"))]
pub fn rintf16(mut x: f16) -> f16 {
// SAFETY: `frintn` is available for `f16` with `fp16` (implies `neon`) and has no side effects.
//
// `frintn` is always round-to-nearest which does not match the C specification, but Rust does
// not support rounding modes.
unsafe {
asm!(
"frintn {x:h}, {x:h}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
pub fn sqrt(mut x: f64) -> f64 {
// SAFETY: `fsqrt` is available with neon and has no side effects.
unsafe {
asm!(
"fsqrt {x:d}, {x:d}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
pub fn sqrtf(mut x: f32) -> f32 {
// SAFETY: `fsqrt` is available with neon and has no side effects.
unsafe {
asm!(
"fsqrt {x:s}, {x:s}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
#[cfg(all(f16_enabled, target_feature = "fp16"))]
pub fn sqrtf16(mut x: f16) -> f16 {
// SAFETY: `fsqrt` is available for `f16` with `fp16` (implies `neon`) and has no
// side effects.
unsafe {
asm!(
"fsqrt {x:h}, {x:h}",
x = inout(vreg) x,
options(nomem, nostack, pure)
);
}
x
}
|
