diff options
Diffstat (limited to 'lib/rbcodec/codecs/libopus/silk/arm')
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_arm.h | 57 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_neon_intr.c | 280 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_arm.h | 100 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_neon_intr.c | 1124 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.c | 112 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.h | 114 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/arm_silk_map.c | 123 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_arm.h | 68 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_neon_intr.c | 156 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/macros_arm64.h | 39 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/macros_armv4.h | 13 | ||||
-rw-r--r-- | lib/rbcodec/codecs/libopus/silk/arm/macros_armv5e.h | 9 |
12 files changed, 2191 insertions, 4 deletions
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_arm.h b/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_arm.h new file mode 100644 index 0000000000..9895b555c8 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_arm.h | |||
@@ -0,0 +1,57 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifndef SILK_LPC_INV_PRED_GAIN_ARM_H | ||
29 | # define SILK_LPC_INV_PRED_GAIN_ARM_H | ||
30 | |||
31 | # include "celt/arm/armcpu.h" | ||
32 | |||
33 | # if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) | ||
34 | opus_int32 silk_LPC_inverse_pred_gain_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ | ||
35 | const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ | ||
36 | const opus_int order /* I Prediction order */ | ||
37 | ); | ||
38 | |||
39 | # if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) | ||
40 | # define OVERRIDE_silk_LPC_inverse_pred_gain (1) | ||
41 | # define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((void)(arch), PRESUME_NEON(silk_LPC_inverse_pred_gain)(A_Q12, order)) | ||
42 | # endif | ||
43 | # endif | ||
44 | |||
45 | # if !defined(OVERRIDE_silk_LPC_inverse_pred_gain) | ||
46 | /*Is run-time CPU detection enabled on this platform?*/ | ||
47 | # if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) | ||
48 | extern opus_int32 (*const SILK_LPC_INVERSE_PRED_GAIN_IMPL[OPUS_ARCHMASK+1])(const opus_int16 *A_Q12, const opus_int order); | ||
49 | # define OVERRIDE_silk_LPC_inverse_pred_gain (1) | ||
50 | # define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((*SILK_LPC_INVERSE_PRED_GAIN_IMPL[(arch)&OPUS_ARCHMASK])(A_Q12, order)) | ||
51 | # elif defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
52 | # define OVERRIDE_silk_LPC_inverse_pred_gain (1) | ||
53 | # define silk_LPC_inverse_pred_gain(A_Q12, order, arch) ((void)(arch), silk_LPC_inverse_pred_gain_neon(A_Q12, order)) | ||
54 | # endif | ||
55 | # endif | ||
56 | |||
57 | #endif /* end SILK_LPC_INV_PRED_GAIN_ARM_H */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_neon_intr.c b/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_neon_intr.c new file mode 100644 index 0000000000..ab426bcd66 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/LPC_inv_pred_gain_neon_intr.c | |||
@@ -0,0 +1,280 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifdef HAVE_CONFIG_H | ||
29 | #include "config.h" | ||
30 | #endif | ||
31 | |||
32 | #include <arm_neon.h> | ||
33 | #include "SigProc_FIX.h" | ||
34 | #include "define.h" | ||
35 | |||
36 | #define QA 24 | ||
37 | #define A_LIMIT SILK_FIX_CONST( 0.99975, QA ) | ||
38 | |||
39 | #define MUL32_FRAC_Q(a32, b32, Q) ((opus_int32)(silk_RSHIFT_ROUND64(silk_SMULL(a32, b32), Q))) | ||
40 | |||
41 | /* The difficulty is how to judge a 64-bit signed integer tmp64 is 32-bit overflowed, | ||
42 | * since NEON has no 64-bit min, max or comparison instructions. | ||
43 | * A failed idea is to compare the results of vmovn(tmp64) and vqmovn(tmp64) whether they are equal or not. | ||
44 | * However, this idea fails when the tmp64 is something like 0xFFFFFFF980000000. | ||
45 | * Here we know that mult2Q >= 1, so the highest bit (bit 63, sign bit) of tmp64 must equal to bit 62. | ||
46 | * tmp64 was shifted left by 1 and we got tmp64'. If high_half(tmp64') != 0 and high_half(tmp64') != -1, | ||
47 | * then we know that bit 31 to bit 63 of tmp64 can not all be the sign bit, and therefore tmp64 is 32-bit overflowed. | ||
48 | * That is, we judge if tmp64' > 0x00000000FFFFFFFF, or tmp64' <= 0xFFFFFFFF00000000. | ||
49 | * We use narrowing shift right 31 bits to tmp32' to save data bandwidth and instructions. | ||
50 | * That is, we judge if tmp32' > 0x00000000, or tmp32' <= 0xFFFFFFFF. | ||
51 | */ | ||
52 | |||
53 | /* Compute inverse of LPC prediction gain, and */ | ||
54 | /* test if LPC coefficients are stable (all poles within unit circle) */ | ||
55 | static OPUS_INLINE opus_int32 LPC_inverse_pred_gain_QA_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ | ||
56 | opus_int32 A_QA[ SILK_MAX_ORDER_LPC ], /* I Prediction coefficients */ | ||
57 | const opus_int order /* I Prediction order */ | ||
58 | ) | ||
59 | { | ||
60 | opus_int k, n, mult2Q; | ||
61 | opus_int32 invGain_Q30, rc_Q31, rc_mult1_Q30, rc_mult2, tmp1, tmp2; | ||
62 | opus_int32 max, min; | ||
63 | int32x4_t max_s32x4, min_s32x4; | ||
64 | int32x2_t max_s32x2, min_s32x2; | ||
65 | |||
66 | max_s32x4 = vdupq_n_s32( silk_int32_MIN ); | ||
67 | min_s32x4 = vdupq_n_s32( silk_int32_MAX ); | ||
68 | invGain_Q30 = SILK_FIX_CONST( 1, 30 ); | ||
69 | for( k = order - 1; k > 0; k-- ) { | ||
70 | int32x2_t rc_Q31_s32x2, rc_mult2_s32x2; | ||
71 | int64x2_t mult2Q_s64x2; | ||
72 | |||
73 | /* Check for stability */ | ||
74 | if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { | ||
75 | return 0; | ||
76 | } | ||
77 | |||
78 | /* Set RC equal to negated AR coef */ | ||
79 | rc_Q31 = -silk_LSHIFT( A_QA[ k ], 31 - QA ); | ||
80 | |||
81 | /* rc_mult1_Q30 range: [ 1 : 2^30 ] */ | ||
82 | rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); | ||
83 | silk_assert( rc_mult1_Q30 > ( 1 << 15 ) ); /* reduce A_LIMIT if fails */ | ||
84 | silk_assert( rc_mult1_Q30 <= ( 1 << 30 ) ); | ||
85 | |||
86 | /* Update inverse gain */ | ||
87 | /* invGain_Q30 range: [ 0 : 2^30 ] */ | ||
88 | invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); | ||
89 | silk_assert( invGain_Q30 >= 0 ); | ||
90 | silk_assert( invGain_Q30 <= ( 1 << 30 ) ); | ||
91 | if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { | ||
92 | return 0; | ||
93 | } | ||
94 | |||
95 | /* rc_mult2 range: [ 2^30 : silk_int32_MAX ] */ | ||
96 | mult2Q = 32 - silk_CLZ32( silk_abs( rc_mult1_Q30 ) ); | ||
97 | rc_mult2 = silk_INVERSE32_varQ( rc_mult1_Q30, mult2Q + 30 ); | ||
98 | |||
99 | /* Update AR coefficient */ | ||
100 | rc_Q31_s32x2 = vdup_n_s32( rc_Q31 ); | ||
101 | mult2Q_s64x2 = vdupq_n_s64( -mult2Q ); | ||
102 | rc_mult2_s32x2 = vdup_n_s32( rc_mult2 ); | ||
103 | |||
104 | for( n = 0; n < ( ( k + 1 ) >> 1 ) - 3; n += 4 ) { | ||
105 | /* We always calculate extra elements of A_QA buffer when ( k % 4 ) != 0, to take the advantage of SIMD parallelization. */ | ||
106 | int32x4_t tmp1_s32x4, tmp2_s32x4, t0_s32x4, t1_s32x4, s0_s32x4, s1_s32x4, t_QA0_s32x4, t_QA1_s32x4; | ||
107 | int64x2_t t0_s64x2, t1_s64x2, t2_s64x2, t3_s64x2; | ||
108 | tmp1_s32x4 = vld1q_s32( A_QA + n ); | ||
109 | tmp2_s32x4 = vld1q_s32( A_QA + k - n - 4 ); | ||
110 | tmp2_s32x4 = vrev64q_s32( tmp2_s32x4 ); | ||
111 | tmp2_s32x4 = vcombine_s32( vget_high_s32( tmp2_s32x4 ), vget_low_s32( tmp2_s32x4 ) ); | ||
112 | t0_s32x4 = vqrdmulhq_lane_s32( tmp2_s32x4, rc_Q31_s32x2, 0 ); | ||
113 | t1_s32x4 = vqrdmulhq_lane_s32( tmp1_s32x4, rc_Q31_s32x2, 0 ); | ||
114 | t_QA0_s32x4 = vqsubq_s32( tmp1_s32x4, t0_s32x4 ); | ||
115 | t_QA1_s32x4 = vqsubq_s32( tmp2_s32x4, t1_s32x4 ); | ||
116 | t0_s64x2 = vmull_s32( vget_low_s32 ( t_QA0_s32x4 ), rc_mult2_s32x2 ); | ||
117 | t1_s64x2 = vmull_s32( vget_high_s32( t_QA0_s32x4 ), rc_mult2_s32x2 ); | ||
118 | t2_s64x2 = vmull_s32( vget_low_s32 ( t_QA1_s32x4 ), rc_mult2_s32x2 ); | ||
119 | t3_s64x2 = vmull_s32( vget_high_s32( t_QA1_s32x4 ), rc_mult2_s32x2 ); | ||
120 | t0_s64x2 = vrshlq_s64( t0_s64x2, mult2Q_s64x2 ); | ||
121 | t1_s64x2 = vrshlq_s64( t1_s64x2, mult2Q_s64x2 ); | ||
122 | t2_s64x2 = vrshlq_s64( t2_s64x2, mult2Q_s64x2 ); | ||
123 | t3_s64x2 = vrshlq_s64( t3_s64x2, mult2Q_s64x2 ); | ||
124 | t0_s32x4 = vcombine_s32( vmovn_s64( t0_s64x2 ), vmovn_s64( t1_s64x2 ) ); | ||
125 | t1_s32x4 = vcombine_s32( vmovn_s64( t2_s64x2 ), vmovn_s64( t3_s64x2 ) ); | ||
126 | s0_s32x4 = vcombine_s32( vshrn_n_s64( t0_s64x2, 31 ), vshrn_n_s64( t1_s64x2, 31 ) ); | ||
127 | s1_s32x4 = vcombine_s32( vshrn_n_s64( t2_s64x2, 31 ), vshrn_n_s64( t3_s64x2, 31 ) ); | ||
128 | max_s32x4 = vmaxq_s32( max_s32x4, s0_s32x4 ); | ||
129 | min_s32x4 = vminq_s32( min_s32x4, s0_s32x4 ); | ||
130 | max_s32x4 = vmaxq_s32( max_s32x4, s1_s32x4 ); | ||
131 | min_s32x4 = vminq_s32( min_s32x4, s1_s32x4 ); | ||
132 | t1_s32x4 = vrev64q_s32( t1_s32x4 ); | ||
133 | t1_s32x4 = vcombine_s32( vget_high_s32( t1_s32x4 ), vget_low_s32( t1_s32x4 ) ); | ||
134 | vst1q_s32( A_QA + n, t0_s32x4 ); | ||
135 | vst1q_s32( A_QA + k - n - 4, t1_s32x4 ); | ||
136 | } | ||
137 | for( ; n < (k + 1) >> 1; n++ ) { | ||
138 | opus_int64 tmp64; | ||
139 | tmp1 = A_QA[ n ]; | ||
140 | tmp2 = A_QA[ k - n - 1 ]; | ||
141 | tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp1, | ||
142 | MUL32_FRAC_Q( tmp2, rc_Q31, 31 ) ), rc_mult2 ), mult2Q); | ||
143 | if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { | ||
144 | return 0; | ||
145 | } | ||
146 | A_QA[ n ] = ( opus_int32 )tmp64; | ||
147 | tmp64 = silk_RSHIFT_ROUND64( silk_SMULL( silk_SUB_SAT32(tmp2, | ||
148 | MUL32_FRAC_Q( tmp1, rc_Q31, 31 ) ), rc_mult2), mult2Q); | ||
149 | if( tmp64 > silk_int32_MAX || tmp64 < silk_int32_MIN ) { | ||
150 | return 0; | ||
151 | } | ||
152 | A_QA[ k - n - 1 ] = ( opus_int32 )tmp64; | ||
153 | } | ||
154 | } | ||
155 | |||
156 | /* Check for stability */ | ||
157 | if( ( A_QA[ k ] > A_LIMIT ) || ( A_QA[ k ] < -A_LIMIT ) ) { | ||
158 | return 0; | ||
159 | } | ||
160 | |||
161 | max_s32x2 = vmax_s32( vget_low_s32( max_s32x4 ), vget_high_s32( max_s32x4 ) ); | ||
162 | min_s32x2 = vmin_s32( vget_low_s32( min_s32x4 ), vget_high_s32( min_s32x4 ) ); | ||
163 | max_s32x2 = vmax_s32( max_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( max_s32x2 ), 32 ) ) ); | ||
164 | min_s32x2 = vmin_s32( min_s32x2, vreinterpret_s32_s64( vshr_n_s64( vreinterpret_s64_s32( min_s32x2 ), 32 ) ) ); | ||
165 | max = vget_lane_s32( max_s32x2, 0 ); | ||
166 | min = vget_lane_s32( min_s32x2, 0 ); | ||
167 | if( ( max > 0 ) || ( min < -1 ) ) { | ||
168 | return 0; | ||
169 | } | ||
170 | |||
171 | /* Set RC equal to negated AR coef */ | ||
172 | rc_Q31 = -silk_LSHIFT( A_QA[ 0 ], 31 - QA ); | ||
173 | |||
174 | /* Range: [ 1 : 2^30 ] */ | ||
175 | rc_mult1_Q30 = silk_SUB32( SILK_FIX_CONST( 1, 30 ), silk_SMMUL( rc_Q31, rc_Q31 ) ); | ||
176 | |||
177 | /* Update inverse gain */ | ||
178 | /* Range: [ 0 : 2^30 ] */ | ||
179 | invGain_Q30 = silk_LSHIFT( silk_SMMUL( invGain_Q30, rc_mult1_Q30 ), 2 ); | ||
180 | silk_assert( invGain_Q30 >= 0 ); | ||
181 | silk_assert( invGain_Q30 <= ( 1 << 30 ) ); | ||
182 | if( invGain_Q30 < SILK_FIX_CONST( 1.0f / MAX_PREDICTION_POWER_GAIN, 30 ) ) { | ||
183 | return 0; | ||
184 | } | ||
185 | |||
186 | return invGain_Q30; | ||
187 | } | ||
188 | |||
189 | /* For input in Q12 domain */ | ||
190 | opus_int32 silk_LPC_inverse_pred_gain_neon( /* O Returns inverse prediction gain in energy domain, Q30 */ | ||
191 | const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ | ||
192 | const opus_int order /* I Prediction order */ | ||
193 | ) | ||
194 | { | ||
195 | #ifdef OPUS_CHECK_ASM | ||
196 | const opus_int32 invGain_Q30_c = silk_LPC_inverse_pred_gain_c( A_Q12, order ); | ||
197 | #endif | ||
198 | |||
199 | opus_int32 invGain_Q30; | ||
200 | if( ( SILK_MAX_ORDER_LPC != 24 ) || ( order & 1 )) { | ||
201 | invGain_Q30 = silk_LPC_inverse_pred_gain_c( A_Q12, order ); | ||
202 | } | ||
203 | else { | ||
204 | opus_int32 Atmp_QA[ SILK_MAX_ORDER_LPC ]; | ||
205 | opus_int32 DC_resp; | ||
206 | int16x8_t t0_s16x8, t1_s16x8, t2_s16x8; | ||
207 | int32x4_t t0_s32x4; | ||
208 | const opus_int leftover = order & 7; | ||
209 | |||
210 | /* Increase Q domain of the AR coefficients */ | ||
211 | t0_s16x8 = vld1q_s16( A_Q12 + 0 ); | ||
212 | t1_s16x8 = vld1q_s16( A_Q12 + 8 ); | ||
213 | t2_s16x8 = vld1q_s16( A_Q12 + 16 ); | ||
214 | t0_s32x4 = vpaddlq_s16( t0_s16x8 ); | ||
215 | |||
216 | switch( order - leftover ) | ||
217 | { | ||
218 | case 24: | ||
219 | t0_s32x4 = vpadalq_s16( t0_s32x4, t2_s16x8 ); | ||
220 | /* FALLTHROUGH */ | ||
221 | |||
222 | case 16: | ||
223 | t0_s32x4 = vpadalq_s16( t0_s32x4, t1_s16x8 ); | ||
224 | vst1q_s32( Atmp_QA + 16, vshll_n_s16( vget_low_s16 ( t2_s16x8 ), QA - 12 ) ); | ||
225 | vst1q_s32( Atmp_QA + 20, vshll_n_s16( vget_high_s16( t2_s16x8 ), QA - 12 ) ); | ||
226 | /* FALLTHROUGH */ | ||
227 | |||
228 | case 8: | ||
229 | { | ||
230 | const int32x2_t t_s32x2 = vpadd_s32( vget_low_s32( t0_s32x4 ), vget_high_s32( t0_s32x4 ) ); | ||
231 | const int64x1_t t_s64x1 = vpaddl_s32( t_s32x2 ); | ||
232 | DC_resp = vget_lane_s32( vreinterpret_s32_s64( t_s64x1 ), 0 ); | ||
233 | vst1q_s32( Atmp_QA + 8, vshll_n_s16( vget_low_s16 ( t1_s16x8 ), QA - 12 ) ); | ||
234 | vst1q_s32( Atmp_QA + 12, vshll_n_s16( vget_high_s16( t1_s16x8 ), QA - 12 ) ); | ||
235 | } | ||
236 | break; | ||
237 | |||
238 | default: | ||
239 | DC_resp = 0; | ||
240 | break; | ||
241 | } | ||
242 | A_Q12 += order - leftover; | ||
243 | |||
244 | switch( leftover ) | ||
245 | { | ||
246 | case 6: | ||
247 | DC_resp += (opus_int32)A_Q12[ 5 ]; | ||
248 | DC_resp += (opus_int32)A_Q12[ 4 ]; | ||
249 | /* FALLTHROUGH */ | ||
250 | |||
251 | case 4: | ||
252 | DC_resp += (opus_int32)A_Q12[ 3 ]; | ||
253 | DC_resp += (opus_int32)A_Q12[ 2 ]; | ||
254 | /* FALLTHROUGH */ | ||
255 | |||
256 | case 2: | ||
257 | DC_resp += (opus_int32)A_Q12[ 1 ]; | ||
258 | DC_resp += (opus_int32)A_Q12[ 0 ]; | ||
259 | /* FALLTHROUGH */ | ||
260 | |||
261 | default: | ||
262 | break; | ||
263 | } | ||
264 | |||
265 | /* If the DC is unstable, we don't even need to do the full calculations */ | ||
266 | if( DC_resp >= 4096 ) { | ||
267 | invGain_Q30 = 0; | ||
268 | } else { | ||
269 | vst1q_s32( Atmp_QA + 0, vshll_n_s16( vget_low_s16 ( t0_s16x8 ), QA - 12 ) ); | ||
270 | vst1q_s32( Atmp_QA + 4, vshll_n_s16( vget_high_s16( t0_s16x8 ), QA - 12 ) ); | ||
271 | invGain_Q30 = LPC_inverse_pred_gain_QA_neon( Atmp_QA, order ); | ||
272 | } | ||
273 | } | ||
274 | |||
275 | #ifdef OPUS_CHECK_ASM | ||
276 | silk_assert( invGain_Q30_c == invGain_Q30 ); | ||
277 | #endif | ||
278 | |||
279 | return invGain_Q30; | ||
280 | } | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_arm.h b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_arm.h new file mode 100644 index 0000000000..9e76e16927 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_arm.h | |||
@@ -0,0 +1,100 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifndef SILK_NSQ_DEL_DEC_ARM_H | ||
29 | #define SILK_NSQ_DEL_DEC_ARM_H | ||
30 | |||
31 | #include "celt/arm/armcpu.h" | ||
32 | |||
33 | #if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) | ||
34 | void silk_NSQ_del_dec_neon( | ||
35 | const silk_encoder_state *psEncC, silk_nsq_state *NSQ, | ||
36 | SideInfoIndices *psIndices, const opus_int16 x16[], opus_int8 pulses[], | ||
37 | const opus_int16 PredCoef_Q12[2 * MAX_LPC_ORDER], | ||
38 | const opus_int16 LTPCoef_Q14[LTP_ORDER * MAX_NB_SUBFR], | ||
39 | const opus_int16 AR_Q13[MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER], | ||
40 | const opus_int HarmShapeGain_Q14[MAX_NB_SUBFR], | ||
41 | const opus_int Tilt_Q14[MAX_NB_SUBFR], | ||
42 | const opus_int32 LF_shp_Q14[MAX_NB_SUBFR], | ||
43 | const opus_int32 Gains_Q16[MAX_NB_SUBFR], | ||
44 | const opus_int pitchL[MAX_NB_SUBFR], const opus_int Lambda_Q10, | ||
45 | const opus_int LTP_scale_Q14); | ||
46 | |||
47 | #if !defined(OPUS_HAVE_RTCD) | ||
48 | #define OVERRIDE_silk_NSQ_del_dec (1) | ||
49 | #define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ | ||
50 | LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ | ||
51 | LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ | ||
52 | LTP_scale_Q14, arch) \ | ||
53 | ((void)(arch), \ | ||
54 | PRESUME_NEON(silk_NSQ_del_dec)( \ | ||
55 | psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, \ | ||
56 | AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, \ | ||
57 | Lambda_Q10, LTP_scale_Q14)) | ||
58 | #endif | ||
59 | #endif | ||
60 | |||
61 | #if !defined(OVERRIDE_silk_NSQ_del_dec) | ||
62 | /*Is run-time CPU detection enabled on this platform?*/ | ||
63 | #if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && \ | ||
64 | !defined(OPUS_ARM_PRESUME_NEON_INTR)) | ||
65 | extern void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( | ||
66 | const silk_encoder_state *psEncC, silk_nsq_state *NSQ, | ||
67 | SideInfoIndices *psIndices, const opus_int16 x16[], opus_int8 pulses[], | ||
68 | const opus_int16 PredCoef_Q12[2 * MAX_LPC_ORDER], | ||
69 | const opus_int16 LTPCoef_Q14[LTP_ORDER * MAX_NB_SUBFR], | ||
70 | const opus_int16 AR_Q13[MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER], | ||
71 | const opus_int HarmShapeGain_Q14[MAX_NB_SUBFR], | ||
72 | const opus_int Tilt_Q14[MAX_NB_SUBFR], | ||
73 | const opus_int32 LF_shp_Q14[MAX_NB_SUBFR], | ||
74 | const opus_int32 Gains_Q16[MAX_NB_SUBFR], | ||
75 | const opus_int pitchL[MAX_NB_SUBFR], const opus_int Lambda_Q10, | ||
76 | const opus_int LTP_scale_Q14); | ||
77 | #define OVERRIDE_silk_NSQ_del_dec (1) | ||
78 | #define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ | ||
79 | LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ | ||
80 | LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ | ||
81 | LTP_scale_Q14, arch) \ | ||
82 | ((*SILK_NSQ_DEL_DEC_IMPL[(arch)&OPUS_ARCHMASK])( \ | ||
83 | psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, \ | ||
84 | AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, \ | ||
85 | Lambda_Q10, LTP_scale_Q14)) | ||
86 | #elif defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
87 | #define OVERRIDE_silk_NSQ_del_dec (1) | ||
88 | #define silk_NSQ_del_dec(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ | ||
89 | LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ | ||
90 | LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ | ||
91 | LTP_scale_Q14, arch) \ | ||
92 | ((void)(arch), \ | ||
93 | silk_NSQ_del_dec_neon(psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, \ | ||
94 | LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, \ | ||
95 | LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, \ | ||
96 | LTP_scale_Q14)) | ||
97 | #endif | ||
98 | #endif | ||
99 | |||
100 | #endif /* end SILK_NSQ_DEL_DEC_ARM_H */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_neon_intr.c b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_neon_intr.c new file mode 100644 index 0000000000..212410f362 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_del_dec_neon_intr.c | |||
@@ -0,0 +1,1124 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifdef HAVE_CONFIG_H | ||
29 | #include "config.h" | ||
30 | #endif | ||
31 | |||
32 | #include <arm_neon.h> | ||
33 | #ifdef OPUS_CHECK_ASM | ||
34 | # include <string.h> | ||
35 | #endif | ||
36 | #include "main.h" | ||
37 | #include "stack_alloc.h" | ||
38 | |||
39 | /* NEON intrinsics optimization now can only parallelize up to 4 delay decision states. */ | ||
40 | /* If there are more states, C function is called, and this optimization must be expanded. */ | ||
41 | #define NEON_MAX_DEL_DEC_STATES 4 | ||
42 | |||
43 | typedef struct { | ||
44 | opus_int32 sLPC_Q14[ MAX_SUB_FRAME_LENGTH + NSQ_LPC_BUF_LENGTH ][ NEON_MAX_DEL_DEC_STATES ]; | ||
45 | opus_int32 RandState[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; | ||
46 | opus_int32 Q_Q10[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; | ||
47 | opus_int32 Xq_Q14[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; | ||
48 | opus_int32 Pred_Q15[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; | ||
49 | opus_int32 Shape_Q14[ DECISION_DELAY ][ NEON_MAX_DEL_DEC_STATES ]; | ||
50 | opus_int32 sAR2_Q14[ MAX_SHAPE_LPC_ORDER ][ NEON_MAX_DEL_DEC_STATES ]; | ||
51 | opus_int32 LF_AR_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
52 | opus_int32 Diff_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
53 | opus_int32 Seed[ NEON_MAX_DEL_DEC_STATES ]; | ||
54 | opus_int32 SeedInit[ NEON_MAX_DEL_DEC_STATES ]; | ||
55 | opus_int32 RD_Q10[ NEON_MAX_DEL_DEC_STATES ]; | ||
56 | } NSQ_del_decs_struct; | ||
57 | |||
58 | typedef struct { | ||
59 | opus_int32 Q_Q10[ NEON_MAX_DEL_DEC_STATES ]; | ||
60 | opus_int32 RD_Q10[ NEON_MAX_DEL_DEC_STATES ]; | ||
61 | opus_int32 xq_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
62 | opus_int32 LF_AR_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
63 | opus_int32 Diff_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
64 | opus_int32 sLTP_shp_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
65 | opus_int32 LPC_exc_Q14[ NEON_MAX_DEL_DEC_STATES ]; | ||
66 | } NSQ_samples_struct; | ||
67 | |||
68 | static OPUS_INLINE void silk_nsq_del_dec_scale_states_neon( | ||
69 | const silk_encoder_state *psEncC, /* I Encoder State */ | ||
70 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
71 | NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ | ||
72 | const opus_int16 x16[], /* I Input */ | ||
73 | opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ | ||
74 | const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ | ||
75 | opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ | ||
76 | opus_int subfr, /* I Subframe number */ | ||
77 | const opus_int LTP_scale_Q14, /* I LTP state scaling */ | ||
78 | const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ | ||
79 | const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ | ||
80 | const opus_int signal_type, /* I Signal type */ | ||
81 | const opus_int decisionDelay /* I Decision delay */ | ||
82 | ); | ||
83 | |||
84 | /******************************************/ | ||
85 | /* Noise shape quantizer for one subframe */ | ||
86 | /******************************************/ | ||
87 | static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_neon( | ||
88 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
89 | NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ | ||
90 | opus_int signalType, /* I Signal type */ | ||
91 | const opus_int32 x_Q10[], /* I */ | ||
92 | opus_int8 pulses[], /* O */ | ||
93 | opus_int16 xq[], /* O */ | ||
94 | opus_int32 sLTP_Q15[], /* I/O LTP filter state */ | ||
95 | opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ | ||
96 | const opus_int16 a_Q12[], /* I Short term prediction coefs */ | ||
97 | const opus_int16 b_Q14[], /* I Long term prediction coefs */ | ||
98 | const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ | ||
99 | opus_int lag, /* I Pitch lag */ | ||
100 | opus_int32 HarmShapeFIRPacked_Q14, /* I */ | ||
101 | opus_int Tilt_Q14, /* I Spectral tilt */ | ||
102 | opus_int32 LF_shp_Q14, /* I */ | ||
103 | opus_int32 Gain_Q16, /* I */ | ||
104 | opus_int Lambda_Q10, /* I */ | ||
105 | opus_int offset_Q10, /* I */ | ||
106 | opus_int length, /* I Input length */ | ||
107 | opus_int subfr, /* I Subframe number */ | ||
108 | opus_int shapingLPCOrder, /* I Shaping LPC filter order */ | ||
109 | opus_int predictLPCOrder, /* I Prediction filter order */ | ||
110 | opus_int warping_Q16, /* I */ | ||
111 | opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ | ||
112 | opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ | ||
113 | opus_int decisionDelay /* I */ | ||
114 | ); | ||
115 | |||
116 | static OPUS_INLINE void copy_winner_state_kernel( | ||
117 | const NSQ_del_decs_struct *psDelDec, | ||
118 | const opus_int offset, | ||
119 | const opus_int last_smple_idx, | ||
120 | const opus_int Winner_ind, | ||
121 | const int32x2_t gain_lo_s32x2, | ||
122 | const int32x2_t gain_hi_s32x2, | ||
123 | const int32x4_t shift_s32x4, | ||
124 | int32x4_t t0_s32x4, | ||
125 | int32x4_t t1_s32x4, | ||
126 | opus_int8 *const pulses, | ||
127 | opus_int16 *pxq, | ||
128 | silk_nsq_state *NSQ | ||
129 | ) | ||
130 | { | ||
131 | int16x8_t t_s16x8; | ||
132 | int32x4_t o0_s32x4, o1_s32x4; | ||
133 | |||
134 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); | ||
135 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); | ||
136 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); | ||
137 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); | ||
138 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); | ||
139 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); | ||
140 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); | ||
141 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Q_Q10[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); | ||
142 | t_s16x8 = vcombine_s16( vrshrn_n_s32( t0_s32x4, 10 ), vrshrn_n_s32( t1_s32x4, 10 ) ); | ||
143 | vst1_s8( &pulses[ offset ], vmovn_s16( t_s16x8 ) ); | ||
144 | |||
145 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); | ||
146 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); | ||
147 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); | ||
148 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); | ||
149 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); | ||
150 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); | ||
151 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); | ||
152 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Xq_Q14[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); | ||
153 | o0_s32x4 = vqdmulhq_lane_s32( t0_s32x4, gain_lo_s32x2, 0 ); | ||
154 | o1_s32x4 = vqdmulhq_lane_s32( t1_s32x4, gain_lo_s32x2, 0 ); | ||
155 | o0_s32x4 = vmlaq_lane_s32( o0_s32x4, t0_s32x4, gain_hi_s32x2, 0 ); | ||
156 | o1_s32x4 = vmlaq_lane_s32( o1_s32x4, t1_s32x4, gain_hi_s32x2, 0 ); | ||
157 | o0_s32x4 = vrshlq_s32( o0_s32x4, shift_s32x4 ); | ||
158 | o1_s32x4 = vrshlq_s32( o1_s32x4, shift_s32x4 ); | ||
159 | vst1_s16( &pxq[ offset + 0 ], vqmovn_s32( o0_s32x4 ) ); | ||
160 | vst1_s16( &pxq[ offset + 4 ], vqmovn_s32( o1_s32x4 ) ); | ||
161 | |||
162 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 0 ][ Winner_ind ], t0_s32x4, 0 ); | ||
163 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 1 ][ Winner_ind ], t0_s32x4, 1 ); | ||
164 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 2 ][ Winner_ind ], t0_s32x4, 2 ); | ||
165 | t0_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 3 ][ Winner_ind ], t0_s32x4, 3 ); | ||
166 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 4 ][ Winner_ind ], t1_s32x4, 0 ); | ||
167 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 5 ][ Winner_ind ], t1_s32x4, 1 ); | ||
168 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 6 ][ Winner_ind ], t1_s32x4, 2 ); | ||
169 | t1_s32x4 = vld1q_lane_s32( &psDelDec->Shape_Q14[ last_smple_idx - 7 ][ Winner_ind ], t1_s32x4, 3 ); | ||
170 | vst1q_s32( &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx + offset + 0 ], t0_s32x4 ); | ||
171 | vst1q_s32( &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx + offset + 4 ], t1_s32x4 ); | ||
172 | } | ||
173 | |||
174 | static OPUS_INLINE void copy_winner_state( | ||
175 | const NSQ_del_decs_struct *psDelDec, | ||
176 | const opus_int decisionDelay, | ||
177 | const opus_int smpl_buf_idx, | ||
178 | const opus_int Winner_ind, | ||
179 | const opus_int32 gain, | ||
180 | const opus_int32 shift, | ||
181 | opus_int8 *const pulses, | ||
182 | opus_int16 *pxq, | ||
183 | silk_nsq_state *NSQ | ||
184 | ) | ||
185 | { | ||
186 | opus_int i, last_smple_idx; | ||
187 | const int32x2_t gain_lo_s32x2 = vdup_n_s32( silk_LSHIFT32( gain & 0x0000FFFF, 15 ) ); | ||
188 | const int32x2_t gain_hi_s32x2 = vdup_n_s32( gain >> 16 ); | ||
189 | const int32x4_t shift_s32x4 = vdupq_n_s32( -shift ); | ||
190 | int32x4_t t0_s32x4, t1_s32x4; | ||
191 | |||
192 | t0_s32x4 = t1_s32x4 = vdupq_n_s32( 0 ); /* initialization */ | ||
193 | last_smple_idx = smpl_buf_idx + decisionDelay - 1 + DECISION_DELAY; | ||
194 | if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; | ||
195 | if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; | ||
196 | |||
197 | for( i = 0; ( i < ( decisionDelay - 7 ) ) && ( last_smple_idx >= 7 ); i += 8, last_smple_idx -= 8 ) { | ||
198 | copy_winner_state_kernel( psDelDec, i - decisionDelay, last_smple_idx, Winner_ind, gain_lo_s32x2, gain_hi_s32x2, shift_s32x4, t0_s32x4, t1_s32x4, pulses, pxq, NSQ ); | ||
199 | } | ||
200 | for( ; ( i < decisionDelay ) && ( last_smple_idx >= 0 ); i++, last_smple_idx-- ) { | ||
201 | pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); | ||
202 | pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], gain ), shift ) ); | ||
203 | NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; | ||
204 | } | ||
205 | |||
206 | last_smple_idx += DECISION_DELAY; | ||
207 | for( ; i < ( decisionDelay - 7 ); i++, last_smple_idx-- ) { | ||
208 | copy_winner_state_kernel( psDelDec, i - decisionDelay, last_smple_idx, Winner_ind, gain_lo_s32x2, gain_hi_s32x2, shift_s32x4, t0_s32x4, t1_s32x4, pulses, pxq, NSQ ); | ||
209 | } | ||
210 | for( ; i < decisionDelay; i++, last_smple_idx-- ) { | ||
211 | pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); | ||
212 | pxq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], gain ), shift ) ); | ||
213 | NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay + i ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; | ||
214 | } | ||
215 | } | ||
216 | |||
217 | void silk_NSQ_del_dec_neon( | ||
218 | const silk_encoder_state *psEncC, /* I Encoder State */ | ||
219 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
220 | SideInfoIndices *psIndices, /* I/O Quantization Indices */ | ||
221 | const opus_int16 x16[], /* I Input */ | ||
222 | opus_int8 pulses[], /* O Quantized pulse signal */ | ||
223 | const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ | ||
224 | const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ | ||
225 | const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ | ||
226 | const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ | ||
227 | const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ | ||
228 | const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ | ||
229 | const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ | ||
230 | const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ | ||
231 | const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ | ||
232 | const opus_int LTP_scale_Q14 /* I LTP state scaling */ | ||
233 | ) | ||
234 | { | ||
235 | #ifdef OPUS_CHECK_ASM | ||
236 | silk_nsq_state NSQ_c; | ||
237 | SideInfoIndices psIndices_c; | ||
238 | opus_int8 pulses_c[ MAX_FRAME_LENGTH ]; | ||
239 | const opus_int8 *const pulses_a = pulses; | ||
240 | |||
241 | ( void )pulses_a; | ||
242 | silk_memcpy( &NSQ_c, NSQ, sizeof( NSQ_c ) ); | ||
243 | silk_memcpy( &psIndices_c, psIndices, sizeof( psIndices_c ) ); | ||
244 | silk_memcpy( pulses_c, pulses, sizeof( pulses_c ) ); | ||
245 | silk_NSQ_del_dec_c( psEncC, &NSQ_c, &psIndices_c, x16, pulses_c, PredCoef_Q12, LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, Tilt_Q14, LF_shp_Q14, Gains_Q16, | ||
246 | pitchL, Lambda_Q10, LTP_scale_Q14 ); | ||
247 | #endif | ||
248 | |||
249 | /* The optimization parallelizes the different delay decision states. */ | ||
250 | if(( psEncC->nStatesDelayedDecision > NEON_MAX_DEL_DEC_STATES ) || ( psEncC->nStatesDelayedDecision <= 2 )) { | ||
251 | /* NEON intrinsics optimization now can only parallelize up to 4 delay decision states. */ | ||
252 | /* If there are more states, C function is called, and this optimization must be expanded. */ | ||
253 | /* When the number of delay decision states is less than 3, there are penalties using this */ | ||
254 | /* optimization, and C function is called. */ | ||
255 | /* When the number of delay decision states is 2, it's better to specialize another */ | ||
256 | /* structure NSQ_del_dec2_struct and optimize with shorter NEON registers. (Low priority) */ | ||
257 | silk_NSQ_del_dec_c( psEncC, NSQ, psIndices, x16, pulses, PredCoef_Q12, LTPCoef_Q14, AR_Q13, HarmShapeGain_Q14, | ||
258 | Tilt_Q14, LF_shp_Q14, Gains_Q16, pitchL, Lambda_Q10, LTP_scale_Q14 ); | ||
259 | } else { | ||
260 | opus_int i, k, lag, start_idx, LSF_interpolation_flag, Winner_ind, subfr; | ||
261 | opus_int smpl_buf_idx, decisionDelay; | ||
262 | const opus_int16 *A_Q12, *B_Q14, *AR_shp_Q13; | ||
263 | opus_int16 *pxq; | ||
264 | VARDECL( opus_int32, sLTP_Q15 ); | ||
265 | VARDECL( opus_int16, sLTP ); | ||
266 | opus_int32 HarmShapeFIRPacked_Q14; | ||
267 | opus_int offset_Q10; | ||
268 | opus_int32 RDmin_Q10, Gain_Q10; | ||
269 | VARDECL( opus_int32, x_sc_Q10 ); | ||
270 | VARDECL( opus_int32, delayedGain_Q10 ); | ||
271 | VARDECL( NSQ_del_decs_struct, psDelDec ); | ||
272 | int32x4_t t_s32x4; | ||
273 | SAVE_STACK; | ||
274 | |||
275 | /* Set unvoiced lag to the previous one, overwrite later for voiced */ | ||
276 | lag = NSQ->lagPrev; | ||
277 | |||
278 | silk_assert( NSQ->prev_gain_Q16 != 0 ); | ||
279 | |||
280 | /* Initialize delayed decision states */ | ||
281 | ALLOC( psDelDec, 1, NSQ_del_decs_struct ); | ||
282 | /* Only RandState and RD_Q10 need to be initialized to 0. */ | ||
283 | silk_memset( psDelDec->RandState, 0, sizeof( psDelDec->RandState ) ); | ||
284 | vst1q_s32( psDelDec->RD_Q10, vdupq_n_s32( 0 ) ); | ||
285 | |||
286 | for( k = 0; k < psEncC->nStatesDelayedDecision; k++ ) { | ||
287 | psDelDec->SeedInit[ k ] = psDelDec->Seed[ k ] = ( k + psIndices->Seed ) & 3; | ||
288 | } | ||
289 | vst1q_s32( psDelDec->LF_AR_Q14, vld1q_dup_s32( &NSQ->sLF_AR_shp_Q14 ) ); | ||
290 | vst1q_s32( psDelDec->Diff_Q14, vld1q_dup_s32( &NSQ->sDiff_shp_Q14 ) ); | ||
291 | vst1q_s32( psDelDec->Shape_Q14[ 0 ], vld1q_dup_s32( &NSQ->sLTP_shp_Q14[ psEncC->ltp_mem_length - 1 ] ) ); | ||
292 | for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { | ||
293 | vst1q_s32( psDelDec->sLPC_Q14[ i ], vld1q_dup_s32( &NSQ->sLPC_Q14[ i ] ) ); | ||
294 | } | ||
295 | for( i = 0; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) ); i++ ) { | ||
296 | vst1q_s32( psDelDec->sAR2_Q14[ i ], vld1q_dup_s32( &NSQ->sAR2_Q14[ i ] ) ); | ||
297 | } | ||
298 | |||
299 | offset_Q10 = silk_Quantization_Offsets_Q10[ psIndices->signalType >> 1 ][ psIndices->quantOffsetType ]; | ||
300 | smpl_buf_idx = 0; /* index of oldest samples */ | ||
301 | |||
302 | decisionDelay = silk_min_int( DECISION_DELAY, psEncC->subfr_length ); | ||
303 | |||
304 | /* For voiced frames limit the decision delay to lower than the pitch lag */ | ||
305 | if( psIndices->signalType == TYPE_VOICED ) { | ||
306 | opus_int pitch_min = pitchL[ 0 ]; | ||
307 | for( k = 1; k < psEncC->nb_subfr; k++ ) { | ||
308 | pitch_min = silk_min_int( pitch_min, pitchL[ k ] ); | ||
309 | } | ||
310 | decisionDelay = silk_min_int( decisionDelay, pitch_min - LTP_ORDER / 2 - 1 ); | ||
311 | } else { | ||
312 | if( lag > 0 ) { | ||
313 | decisionDelay = silk_min_int( decisionDelay, lag - LTP_ORDER / 2 - 1 ); | ||
314 | } | ||
315 | } | ||
316 | |||
317 | if( psIndices->NLSFInterpCoef_Q2 == 4 ) { | ||
318 | LSF_interpolation_flag = 0; | ||
319 | } else { | ||
320 | LSF_interpolation_flag = 1; | ||
321 | } | ||
322 | |||
323 | ALLOC( sLTP_Q15, psEncC->ltp_mem_length + psEncC->frame_length, opus_int32 ); | ||
324 | ALLOC( sLTP, psEncC->ltp_mem_length + psEncC->frame_length, opus_int16 ); | ||
325 | ALLOC( x_sc_Q10, psEncC->subfr_length, opus_int32 ); | ||
326 | ALLOC( delayedGain_Q10, DECISION_DELAY, opus_int32 ); | ||
327 | /* Set up pointers to start of sub frame */ | ||
328 | pxq = &NSQ->xq[ psEncC->ltp_mem_length ]; | ||
329 | NSQ->sLTP_shp_buf_idx = psEncC->ltp_mem_length; | ||
330 | NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; | ||
331 | subfr = 0; | ||
332 | for( k = 0; k < psEncC->nb_subfr; k++ ) { | ||
333 | A_Q12 = &PredCoef_Q12[ ( ( k >> 1 ) | ( 1 - LSF_interpolation_flag ) ) * MAX_LPC_ORDER ]; | ||
334 | B_Q14 = <PCoef_Q14[ k * LTP_ORDER ]; | ||
335 | AR_shp_Q13 = &AR_Q13[ k * MAX_SHAPE_LPC_ORDER ]; | ||
336 | |||
337 | /* Noise shape parameters */ | ||
338 | silk_assert( HarmShapeGain_Q14[ k ] >= 0 ); | ||
339 | HarmShapeFIRPacked_Q14 = silk_RSHIFT( HarmShapeGain_Q14[ k ], 2 ); | ||
340 | HarmShapeFIRPacked_Q14 |= silk_LSHIFT( (opus_int32)silk_RSHIFT( HarmShapeGain_Q14[ k ], 1 ), 16 ); | ||
341 | |||
342 | NSQ->rewhite_flag = 0; | ||
343 | if( psIndices->signalType == TYPE_VOICED ) { | ||
344 | /* Voiced */ | ||
345 | lag = pitchL[ k ]; | ||
346 | |||
347 | /* Re-whitening */ | ||
348 | if( ( k & ( 3 - silk_LSHIFT( LSF_interpolation_flag, 1 ) ) ) == 0 ) { | ||
349 | if( k == 2 ) { | ||
350 | /* RESET DELAYED DECISIONS */ | ||
351 | /* Find winner */ | ||
352 | int32x4_t RD_Q10_s32x4; | ||
353 | RDmin_Q10 = psDelDec->RD_Q10[ 0 ]; | ||
354 | Winner_ind = 0; | ||
355 | for( i = 1; i < psEncC->nStatesDelayedDecision; i++ ) { | ||
356 | if( psDelDec->RD_Q10[ i ] < RDmin_Q10 ) { | ||
357 | RDmin_Q10 = psDelDec->RD_Q10[ i ]; | ||
358 | Winner_ind = i; | ||
359 | } | ||
360 | } | ||
361 | psDelDec->RD_Q10[ Winner_ind ] -= ( silk_int32_MAX >> 4 ); | ||
362 | RD_Q10_s32x4 = vld1q_s32( psDelDec->RD_Q10 ); | ||
363 | RD_Q10_s32x4 = vaddq_s32( RD_Q10_s32x4, vdupq_n_s32( silk_int32_MAX >> 4 ) ); | ||
364 | vst1q_s32( psDelDec->RD_Q10, RD_Q10_s32x4 ); | ||
365 | |||
366 | /* Copy final part of signals from winner state to output and long-term filter states */ | ||
367 | copy_winner_state( psDelDec, decisionDelay, smpl_buf_idx, Winner_ind, Gains_Q16[ 1 ], 14, pulses, pxq, NSQ ); | ||
368 | |||
369 | subfr = 0; | ||
370 | } | ||
371 | |||
372 | /* Rewhiten with new A coefs */ | ||
373 | start_idx = psEncC->ltp_mem_length - lag - psEncC->predictLPCOrder - LTP_ORDER / 2; | ||
374 | silk_assert( start_idx > 0 ); | ||
375 | |||
376 | silk_LPC_analysis_filter( &sLTP[ start_idx ], &NSQ->xq[ start_idx + k * psEncC->subfr_length ], | ||
377 | A_Q12, psEncC->ltp_mem_length - start_idx, psEncC->predictLPCOrder, psEncC->arch ); | ||
378 | |||
379 | NSQ->sLTP_buf_idx = psEncC->ltp_mem_length; | ||
380 | NSQ->rewhite_flag = 1; | ||
381 | } | ||
382 | } | ||
383 | |||
384 | silk_nsq_del_dec_scale_states_neon( psEncC, NSQ, psDelDec, x16, x_sc_Q10, sLTP, sLTP_Q15, k, | ||
385 | LTP_scale_Q14, Gains_Q16, pitchL, psIndices->signalType, decisionDelay ); | ||
386 | |||
387 | silk_noise_shape_quantizer_del_dec_neon( NSQ, psDelDec, psIndices->signalType, x_sc_Q10, pulses, pxq, sLTP_Q15, | ||
388 | delayedGain_Q10, A_Q12, B_Q14, AR_shp_Q13, lag, HarmShapeFIRPacked_Q14, Tilt_Q14[ k ], LF_shp_Q14[ k ], | ||
389 | Gains_Q16[ k ], Lambda_Q10, offset_Q10, psEncC->subfr_length, subfr++, psEncC->shapingLPCOrder, | ||
390 | psEncC->predictLPCOrder, psEncC->warping_Q16, psEncC->nStatesDelayedDecision, &smpl_buf_idx, decisionDelay ); | ||
391 | |||
392 | x16 += psEncC->subfr_length; | ||
393 | pulses += psEncC->subfr_length; | ||
394 | pxq += psEncC->subfr_length; | ||
395 | } | ||
396 | |||
397 | /* Find winner */ | ||
398 | RDmin_Q10 = psDelDec->RD_Q10[ 0 ]; | ||
399 | Winner_ind = 0; | ||
400 | for( k = 1; k < psEncC->nStatesDelayedDecision; k++ ) { | ||
401 | if( psDelDec->RD_Q10[ k ] < RDmin_Q10 ) { | ||
402 | RDmin_Q10 = psDelDec->RD_Q10[ k ]; | ||
403 | Winner_ind = k; | ||
404 | } | ||
405 | } | ||
406 | |||
407 | /* Copy final part of signals from winner state to output and long-term filter states */ | ||
408 | psIndices->Seed = psDelDec->SeedInit[ Winner_ind ]; | ||
409 | Gain_Q10 = silk_RSHIFT32( Gains_Q16[ psEncC->nb_subfr - 1 ], 6 ); | ||
410 | copy_winner_state( psDelDec, decisionDelay, smpl_buf_idx, Winner_ind, Gain_Q10, 8, pulses, pxq, NSQ ); | ||
411 | |||
412 | t_s32x4 = vdupq_n_s32( 0 ); /* initialization */ | ||
413 | for( i = 0; i < ( NSQ_LPC_BUF_LENGTH - 3 ); i += 4 ) { | ||
414 | t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 0 ][ Winner_ind ], t_s32x4, 0 ); | ||
415 | t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 1 ][ Winner_ind ], t_s32x4, 1 ); | ||
416 | t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 2 ][ Winner_ind ], t_s32x4, 2 ); | ||
417 | t_s32x4 = vld1q_lane_s32( &psDelDec->sLPC_Q14[ i + 3 ][ Winner_ind ], t_s32x4, 3 ); | ||
418 | vst1q_s32( &NSQ->sLPC_Q14[ i ], t_s32x4 ); | ||
419 | } | ||
420 | |||
421 | for( ; i < NSQ_LPC_BUF_LENGTH; i++ ) { | ||
422 | NSQ->sLPC_Q14[ i ] = psDelDec->sLPC_Q14[ i ][ Winner_ind ]; | ||
423 | } | ||
424 | |||
425 | for( i = 0; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) - 3 ); i += 4 ) { | ||
426 | t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 0 ][ Winner_ind ], t_s32x4, 0 ); | ||
427 | t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 1 ][ Winner_ind ], t_s32x4, 1 ); | ||
428 | t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 2 ][ Winner_ind ], t_s32x4, 2 ); | ||
429 | t_s32x4 = vld1q_lane_s32( &psDelDec->sAR2_Q14[ i + 3 ][ Winner_ind ], t_s32x4, 3 ); | ||
430 | vst1q_s32( &NSQ->sAR2_Q14[ i ], t_s32x4 ); | ||
431 | } | ||
432 | |||
433 | for( ; i < (opus_int)( sizeof( NSQ->sAR2_Q14 ) / sizeof( NSQ->sAR2_Q14[ 0 ] ) ); i++ ) { | ||
434 | NSQ->sAR2_Q14[ i ] = psDelDec->sAR2_Q14[ i ][ Winner_ind ]; | ||
435 | } | ||
436 | |||
437 | /* Update states */ | ||
438 | NSQ->sLF_AR_shp_Q14 = psDelDec->LF_AR_Q14[ Winner_ind ]; | ||
439 | NSQ->sDiff_shp_Q14 = psDelDec->Diff_Q14[ Winner_ind ]; | ||
440 | NSQ->lagPrev = pitchL[ psEncC->nb_subfr - 1 ]; | ||
441 | |||
442 | /* Save quantized speech signal */ | ||
443 | silk_memmove( NSQ->xq, &NSQ->xq[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int16 ) ); | ||
444 | silk_memmove( NSQ->sLTP_shp_Q14, &NSQ->sLTP_shp_Q14[ psEncC->frame_length ], psEncC->ltp_mem_length * sizeof( opus_int32 ) ); | ||
445 | RESTORE_STACK; | ||
446 | } | ||
447 | |||
448 | #ifdef OPUS_CHECK_ASM | ||
449 | silk_assert( !memcmp( &NSQ_c, NSQ, sizeof( NSQ_c ) ) ); | ||
450 | silk_assert( !memcmp( &psIndices_c, psIndices, sizeof( psIndices_c ) ) ); | ||
451 | silk_assert( !memcmp( pulses_c, pulses_a, sizeof( pulses_c ) ) ); | ||
452 | #endif | ||
453 | } | ||
454 | |||
455 | /******************************************/ | ||
456 | /* Noise shape quantizer for one subframe */ | ||
457 | /******************************************/ | ||
458 | /* Note: Function silk_short_prediction_create_arch_coef_neon() defined in NSQ_neon.h is actually a hacking C function. */ | ||
459 | /* Therefore here we append "_local" to the NEON function name to avoid confusion. */ | ||
460 | static OPUS_INLINE void silk_short_prediction_create_arch_coef_neon_local(opus_int32 *out, const opus_int16 *in, opus_int order) | ||
461 | { | ||
462 | int16x8_t t_s16x8; | ||
463 | int32x4_t t0_s32x4, t1_s32x4, t2_s32x4, t3_s32x4; | ||
464 | silk_assert( order == 10 || order == 16 ); | ||
465 | |||
466 | t_s16x8 = vld1q_s16( in + 0 ); /* 7 6 5 4 3 2 1 0 */ | ||
467 | t_s16x8 = vrev64q_s16( t_s16x8 ); /* 4 5 6 7 0 1 2 3 */ | ||
468 | t2_s32x4 = vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ); /* 4 5 6 7 */ | ||
469 | t3_s32x4 = vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ); /* 0 1 2 3 */ | ||
470 | |||
471 | if( order == 16 ) { | ||
472 | t_s16x8 = vld1q_s16( in + 8 ); /* F E D C B A 9 8 */ | ||
473 | t_s16x8 = vrev64q_s16( t_s16x8 ); /* C D E F 8 9 A B */ | ||
474 | t0_s32x4 = vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ); /* C D E F */ | ||
475 | t1_s32x4 = vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ); /* 8 9 A B */ | ||
476 | } else { | ||
477 | int16x4_t t_s16x4; | ||
478 | |||
479 | t0_s32x4 = vdupq_n_s32( 0 ); /* zero zero zero zero */ | ||
480 | t_s16x4 = vld1_s16( in + 6 ); /* 9 8 7 6 */ | ||
481 | t_s16x4 = vrev64_s16( t_s16x4 ); /* 6 7 8 9 */ | ||
482 | t1_s32x4 = vshll_n_s16( t_s16x4, 15 ); | ||
483 | t1_s32x4 = vcombine_s32( vget_low_s32(t0_s32x4), vget_low_s32( t1_s32x4 ) ); /* 8 9 zero zero */ | ||
484 | } | ||
485 | vst1q_s32( out + 0, t0_s32x4 ); | ||
486 | vst1q_s32( out + 4, t1_s32x4 ); | ||
487 | vst1q_s32( out + 8, t2_s32x4 ); | ||
488 | vst1q_s32( out + 12, t3_s32x4 ); | ||
489 | } | ||
490 | |||
491 | static OPUS_INLINE int32x4_t silk_SMLAWB_lane0_neon( | ||
492 | const int32x4_t out_s32x4, | ||
493 | const int32x4_t in_s32x4, | ||
494 | const int32x2_t coef_s32x2 | ||
495 | ) | ||
496 | { | ||
497 | return vaddq_s32( out_s32x4, vqdmulhq_lane_s32( in_s32x4, coef_s32x2, 0 ) ); | ||
498 | } | ||
499 | |||
500 | static OPUS_INLINE int32x4_t silk_SMLAWB_lane1_neon( | ||
501 | const int32x4_t out_s32x4, | ||
502 | const int32x4_t in_s32x4, | ||
503 | const int32x2_t coef_s32x2 | ||
504 | ) | ||
505 | { | ||
506 | return vaddq_s32( out_s32x4, vqdmulhq_lane_s32( in_s32x4, coef_s32x2, 1 ) ); | ||
507 | } | ||
508 | |||
509 | /* Note: This function has different return value than silk_noise_shape_quantizer_short_prediction_neon(). */ | ||
510 | /* Therefore here we append "_local" to the function name to avoid confusion. */ | ||
511 | static OPUS_INLINE int32x4_t silk_noise_shape_quantizer_short_prediction_neon_local(const opus_int32 *buf32, const opus_int32 *a_Q12_arch, opus_int order) | ||
512 | { | ||
513 | const int32x4_t a_Q12_arch0_s32x4 = vld1q_s32( a_Q12_arch + 0 ); | ||
514 | const int32x4_t a_Q12_arch1_s32x4 = vld1q_s32( a_Q12_arch + 4 ); | ||
515 | const int32x4_t a_Q12_arch2_s32x4 = vld1q_s32( a_Q12_arch + 8 ); | ||
516 | const int32x4_t a_Q12_arch3_s32x4 = vld1q_s32( a_Q12_arch + 12 ); | ||
517 | int32x4_t LPC_pred_Q14_s32x4; | ||
518 | |||
519 | silk_assert( order == 10 || order == 16 ); | ||
520 | /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ | ||
521 | LPC_pred_Q14_s32x4 = vdupq_n_s32( silk_RSHIFT( order, 1 ) ); | ||
522 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 0 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch0_s32x4 ) ); | ||
523 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 1 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch0_s32x4 ) ); | ||
524 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 2 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch0_s32x4 ) ); | ||
525 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 3 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch0_s32x4 ) ); | ||
526 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 4 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch1_s32x4 ) ); | ||
527 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 5 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch1_s32x4 ) ); | ||
528 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 6 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch1_s32x4 ) ); | ||
529 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 7 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch1_s32x4 ) ); | ||
530 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 8 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch2_s32x4 ) ); | ||
531 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 9 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch2_s32x4 ) ); | ||
532 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 10 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch2_s32x4 ) ); | ||
533 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 11 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch2_s32x4 ) ); | ||
534 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 12 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch3_s32x4 ) ); | ||
535 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 13 * NEON_MAX_DEL_DEC_STATES ), vget_low_s32( a_Q12_arch3_s32x4 ) ); | ||
536 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane0_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 14 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch3_s32x4 ) ); | ||
537 | LPC_pred_Q14_s32x4 = silk_SMLAWB_lane1_neon( LPC_pred_Q14_s32x4, vld1q_s32( buf32 + 15 * NEON_MAX_DEL_DEC_STATES ), vget_high_s32( a_Q12_arch3_s32x4 ) ); | ||
538 | |||
539 | return LPC_pred_Q14_s32x4; | ||
540 | } | ||
541 | |||
542 | static OPUS_INLINE void silk_noise_shape_quantizer_del_dec_neon( | ||
543 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
544 | NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ | ||
545 | opus_int signalType, /* I Signal type */ | ||
546 | const opus_int32 x_Q10[], /* I */ | ||
547 | opus_int8 pulses[], /* O */ | ||
548 | opus_int16 xq[], /* O */ | ||
549 | opus_int32 sLTP_Q15[], /* I/O LTP filter state */ | ||
550 | opus_int32 delayedGain_Q10[], /* I/O Gain delay buffer */ | ||
551 | const opus_int16 a_Q12[], /* I Short term prediction coefs */ | ||
552 | const opus_int16 b_Q14[], /* I Long term prediction coefs */ | ||
553 | const opus_int16 AR_shp_Q13[], /* I Noise shaping coefs */ | ||
554 | opus_int lag, /* I Pitch lag */ | ||
555 | opus_int32 HarmShapeFIRPacked_Q14, /* I */ | ||
556 | opus_int Tilt_Q14, /* I Spectral tilt */ | ||
557 | opus_int32 LF_shp_Q14, /* I */ | ||
558 | opus_int32 Gain_Q16, /* I */ | ||
559 | opus_int Lambda_Q10, /* I */ | ||
560 | opus_int offset_Q10, /* I */ | ||
561 | opus_int length, /* I Input length */ | ||
562 | opus_int subfr, /* I Subframe number */ | ||
563 | opus_int shapingLPCOrder, /* I Shaping LPC filter order */ | ||
564 | opus_int predictLPCOrder, /* I Prediction filter order */ | ||
565 | opus_int warping_Q16, /* I */ | ||
566 | opus_int nStatesDelayedDecision, /* I Number of states in decision tree */ | ||
567 | opus_int *smpl_buf_idx, /* I/O Index to newest samples in buffers */ | ||
568 | opus_int decisionDelay /* I */ | ||
569 | ) | ||
570 | { | ||
571 | opus_int i, j, k, Winner_ind, RDmin_ind, RDmax_ind, last_smple_idx; | ||
572 | opus_int32 Winner_rand_state; | ||
573 | opus_int32 LTP_pred_Q14, n_LTP_Q14; | ||
574 | opus_int32 RDmin_Q10, RDmax_Q10; | ||
575 | opus_int32 Gain_Q10; | ||
576 | opus_int32 *pred_lag_ptr, *shp_lag_ptr; | ||
577 | opus_int32 a_Q12_arch[MAX_LPC_ORDER]; | ||
578 | const int32x2_t warping_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( warping_Q16, 16 ) >> 1 ); | ||
579 | const opus_int32 LF_shp_Q29 = silk_LSHIFT32( LF_shp_Q14, 16 ) >> 1; | ||
580 | opus_int32 AR_shp_Q28[ MAX_SHAPE_LPC_ORDER ]; | ||
581 | const uint32x4_t rand_multiplier_u32x4 = vdupq_n_u32( RAND_MULTIPLIER ); | ||
582 | const uint32x4_t rand_increment_u32x4 = vdupq_n_u32( RAND_INCREMENT ); | ||
583 | |||
584 | VARDECL( NSQ_samples_struct, psSampleState ); | ||
585 | SAVE_STACK; | ||
586 | |||
587 | silk_assert( nStatesDelayedDecision > 0 ); | ||
588 | silk_assert( ( shapingLPCOrder & 1 ) == 0 ); /* check that order is even */ | ||
589 | ALLOC( psSampleState, 2, NSQ_samples_struct ); | ||
590 | |||
591 | shp_lag_ptr = &NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - lag + HARM_SHAPE_FIR_TAPS / 2 ]; | ||
592 | pred_lag_ptr = &sLTP_Q15[ NSQ->sLTP_buf_idx - lag + LTP_ORDER / 2 ]; | ||
593 | Gain_Q10 = silk_RSHIFT( Gain_Q16, 6 ); | ||
594 | |||
595 | for( i = 0; i < ( MAX_SHAPE_LPC_ORDER - 7 ); i += 8 ) { | ||
596 | const int16x8_t t_s16x8 = vld1q_s16( AR_shp_Q13 + i ); | ||
597 | vst1q_s32( AR_shp_Q28 + i + 0, vshll_n_s16( vget_low_s16( t_s16x8 ), 15 ) ); | ||
598 | vst1q_s32( AR_shp_Q28 + i + 4, vshll_n_s16( vget_high_s16( t_s16x8 ), 15 ) ); | ||
599 | } | ||
600 | |||
601 | for( ; i < MAX_SHAPE_LPC_ORDER; i++ ) { | ||
602 | AR_shp_Q28[i] = silk_LSHIFT32( AR_shp_Q13[i], 15 ); | ||
603 | } | ||
604 | |||
605 | silk_short_prediction_create_arch_coef_neon_local( a_Q12_arch, a_Q12, predictLPCOrder ); | ||
606 | |||
607 | for( i = 0; i < length; i++ ) { | ||
608 | int32x4_t Seed_s32x4, LPC_pred_Q14_s32x4; | ||
609 | int32x4_t sign_s32x4, tmp1_s32x4, tmp2_s32x4; | ||
610 | int32x4_t n_AR_Q14_s32x4, n_LF_Q14_s32x4; | ||
611 | int32x2_t AR_shp_Q28_s32x2; | ||
612 | int16x4_t r_Q10_s16x4, rr_Q10_s16x4; | ||
613 | |||
614 | /* Perform common calculations used in all states */ | ||
615 | |||
616 | /* Long-term prediction */ | ||
617 | if( signalType == TYPE_VOICED ) { | ||
618 | /* Unrolled loop */ | ||
619 | /* Avoids introducing a bias because silk_SMLAWB() always rounds to -inf */ | ||
620 | LTP_pred_Q14 = 2; | ||
621 | LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ 0 ], b_Q14[ 0 ] ); | ||
622 | LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -1 ], b_Q14[ 1 ] ); | ||
623 | LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -2 ], b_Q14[ 2 ] ); | ||
624 | LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -3 ], b_Q14[ 3 ] ); | ||
625 | LTP_pred_Q14 = silk_SMLAWB( LTP_pred_Q14, pred_lag_ptr[ -4 ], b_Q14[ 4 ] ); | ||
626 | LTP_pred_Q14 = silk_LSHIFT( LTP_pred_Q14, 1 ); /* Q13 -> Q14 */ | ||
627 | pred_lag_ptr++; | ||
628 | } else { | ||
629 | LTP_pred_Q14 = 0; | ||
630 | } | ||
631 | |||
632 | /* Long-term shaping */ | ||
633 | if( lag > 0 ) { | ||
634 | /* Symmetric, packed FIR coefficients */ | ||
635 | n_LTP_Q14 = silk_SMULWB( silk_ADD32( shp_lag_ptr[ 0 ], shp_lag_ptr[ -2 ] ), HarmShapeFIRPacked_Q14 ); | ||
636 | n_LTP_Q14 = silk_SMLAWT( n_LTP_Q14, shp_lag_ptr[ -1 ], HarmShapeFIRPacked_Q14 ); | ||
637 | n_LTP_Q14 = silk_SUB_LSHIFT32( LTP_pred_Q14, n_LTP_Q14, 2 ); /* Q12 -> Q14 */ | ||
638 | shp_lag_ptr++; | ||
639 | } else { | ||
640 | n_LTP_Q14 = 0; | ||
641 | } | ||
642 | |||
643 | /* Generate dither */ | ||
644 | Seed_s32x4 = vld1q_s32( psDelDec->Seed ); | ||
645 | Seed_s32x4 = vreinterpretq_s32_u32( vmlaq_u32( rand_increment_u32x4, vreinterpretq_u32_s32( Seed_s32x4 ), rand_multiplier_u32x4 ) ); | ||
646 | vst1q_s32( psDelDec->Seed, Seed_s32x4 ); | ||
647 | |||
648 | /* Short-term prediction */ | ||
649 | LPC_pred_Q14_s32x4 = silk_noise_shape_quantizer_short_prediction_neon_local(psDelDec->sLPC_Q14[ NSQ_LPC_BUF_LENGTH - 16 + i ], a_Q12_arch, predictLPCOrder); | ||
650 | LPC_pred_Q14_s32x4 = vshlq_n_s32( LPC_pred_Q14_s32x4, 4 ); /* Q10 -> Q14 */ | ||
651 | |||
652 | /* Noise shape feedback */ | ||
653 | /* Output of lowpass section */ | ||
654 | tmp2_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->Diff_Q14 ), vld1q_s32( psDelDec->sAR2_Q14[ 0 ] ), warping_Q16_s32x2 ); | ||
655 | /* Output of allpass section */ | ||
656 | tmp1_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ 1 ] ), tmp2_s32x4 ); | ||
657 | tmp1_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ 0 ] ), tmp1_s32x4, warping_Q16_s32x2 ); | ||
658 | vst1q_s32( psDelDec->sAR2_Q14[ 0 ], tmp2_s32x4 ); | ||
659 | AR_shp_Q28_s32x2 = vld1_s32( AR_shp_Q28 ); | ||
660 | n_AR_Q14_s32x4 = vaddq_s32( vdupq_n_s32( silk_RSHIFT( shapingLPCOrder, 1 ) ), vqdmulhq_lane_s32( tmp2_s32x4, AR_shp_Q28_s32x2, 0 ) ); | ||
661 | |||
662 | /* Loop over allpass sections */ | ||
663 | for( j = 2; j < shapingLPCOrder; j += 2 ) { | ||
664 | /* Output of allpass section */ | ||
665 | tmp2_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ j + 0 ] ), tmp1_s32x4 ); | ||
666 | tmp2_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ j - 1 ] ), tmp2_s32x4, warping_Q16_s32x2 ); | ||
667 | vst1q_s32( psDelDec->sAR2_Q14[ j - 1 ], tmp1_s32x4 ); | ||
668 | n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp1_s32x4, AR_shp_Q28_s32x2, 1 ) ); | ||
669 | /* Output of allpass section */ | ||
670 | tmp1_s32x4 = vsubq_s32( vld1q_s32( psDelDec->sAR2_Q14[ j + 1 ] ), tmp2_s32x4 ); | ||
671 | tmp1_s32x4 = silk_SMLAWB_lane0_neon( vld1q_s32( psDelDec->sAR2_Q14[ j + 0 ] ), tmp1_s32x4, warping_Q16_s32x2 ); | ||
672 | vst1q_s32( psDelDec->sAR2_Q14[ j + 0 ], tmp2_s32x4 ); | ||
673 | AR_shp_Q28_s32x2 = vld1_s32( &AR_shp_Q28[ j ] ); | ||
674 | n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp2_s32x4, AR_shp_Q28_s32x2, 0 ) ); | ||
675 | } | ||
676 | vst1q_s32( psDelDec->sAR2_Q14[ shapingLPCOrder - 1 ], tmp1_s32x4 ); | ||
677 | n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_lane_s32( tmp1_s32x4, AR_shp_Q28_s32x2, 1 ) ); | ||
678 | n_AR_Q14_s32x4 = vshlq_n_s32( n_AR_Q14_s32x4, 1 ); /* Q11 -> Q12 */ | ||
679 | n_AR_Q14_s32x4 = vaddq_s32( n_AR_Q14_s32x4, vqdmulhq_n_s32( vld1q_s32( psDelDec->LF_AR_Q14 ), silk_LSHIFT32( Tilt_Q14, 16 ) >> 1 ) ); /* Q12 */ | ||
680 | n_AR_Q14_s32x4 = vshlq_n_s32( n_AR_Q14_s32x4, 2 ); /* Q12 -> Q14 */ | ||
681 | n_LF_Q14_s32x4 = vqdmulhq_n_s32( vld1q_s32( psDelDec->Shape_Q14[ *smpl_buf_idx ] ), LF_shp_Q29 ); /* Q12 */ | ||
682 | n_LF_Q14_s32x4 = vaddq_s32( n_LF_Q14_s32x4, vqdmulhq_n_s32( vld1q_s32( psDelDec->LF_AR_Q14 ), silk_LSHIFT32( LF_shp_Q14 >> 16 , 15 ) ) ); /* Q12 */ | ||
683 | n_LF_Q14_s32x4 = vshlq_n_s32( n_LF_Q14_s32x4, 2 ); /* Q12 -> Q14 */ | ||
684 | |||
685 | /* Input minus prediction plus noise feedback */ | ||
686 | /* r = x[ i ] - LTP_pred - LPC_pred + n_AR + n_Tilt + n_LF + n_LTP */ | ||
687 | tmp1_s32x4 = vaddq_s32( n_AR_Q14_s32x4, n_LF_Q14_s32x4 ); /* Q14 */ | ||
688 | tmp2_s32x4 = vaddq_s32( vdupq_n_s32( n_LTP_Q14 ), LPC_pred_Q14_s32x4 ); /* Q13 */ | ||
689 | tmp1_s32x4 = vsubq_s32( tmp2_s32x4, tmp1_s32x4 ); /* Q13 */ | ||
690 | tmp1_s32x4 = vrshrq_n_s32( tmp1_s32x4, 4 ); /* Q10 */ | ||
691 | tmp1_s32x4 = vsubq_s32( vdupq_n_s32( x_Q10[ i ] ), tmp1_s32x4 ); /* residual error Q10 */ | ||
692 | |||
693 | /* Flip sign depending on dither */ | ||
694 | sign_s32x4 = vreinterpretq_s32_u32( vcltq_s32( Seed_s32x4, vdupq_n_s32( 0 ) ) ); | ||
695 | tmp1_s32x4 = veorq_s32( tmp1_s32x4, sign_s32x4 ); | ||
696 | tmp1_s32x4 = vsubq_s32( tmp1_s32x4, sign_s32x4 ); | ||
697 | tmp1_s32x4 = vmaxq_s32( tmp1_s32x4, vdupq_n_s32( -( 31 << 10 ) ) ); | ||
698 | tmp1_s32x4 = vminq_s32( tmp1_s32x4, vdupq_n_s32( 30 << 10 ) ); | ||
699 | r_Q10_s16x4 = vmovn_s32( tmp1_s32x4 ); | ||
700 | |||
701 | /* Find two quantization level candidates and measure their rate-distortion */ | ||
702 | { | ||
703 | int16x4_t q1_Q10_s16x4 = vsub_s16( r_Q10_s16x4, vdup_n_s16( offset_Q10 ) ); | ||
704 | int16x4_t q1_Q0_s16x4 = vshr_n_s16( q1_Q10_s16x4, 10 ); | ||
705 | int16x4_t q2_Q10_s16x4; | ||
706 | int32x4_t rd1_Q10_s32x4, rd2_Q10_s32x4; | ||
707 | uint32x4_t t_u32x4; | ||
708 | |||
709 | if( Lambda_Q10 > 2048 ) { | ||
710 | /* For aggressive RDO, the bias becomes more than one pulse. */ | ||
711 | const int rdo_offset = Lambda_Q10/2 - 512; | ||
712 | const uint16x4_t greaterThanRdo = vcgt_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ); | ||
713 | const uint16x4_t lessThanMinusRdo = vclt_s16( q1_Q10_s16x4, vdup_n_s16( -rdo_offset ) ); | ||
714 | /* If Lambda_Q10 > 32767, then q1_Q0, q1_Q10 and q2_Q10 must change to 32-bit. */ | ||
715 | silk_assert( Lambda_Q10 <= 32767 ); | ||
716 | |||
717 | q1_Q0_s16x4 = vreinterpret_s16_u16( vclt_s16( q1_Q10_s16x4, vdup_n_s16( 0 ) ) ); | ||
718 | q1_Q0_s16x4 = vbsl_s16( greaterThanRdo, vsub_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ), q1_Q0_s16x4 ); | ||
719 | q1_Q0_s16x4 = vbsl_s16( lessThanMinusRdo, vadd_s16( q1_Q10_s16x4, vdup_n_s16( rdo_offset ) ), q1_Q0_s16x4 ); | ||
720 | q1_Q0_s16x4 = vshr_n_s16( q1_Q0_s16x4, 10 ); | ||
721 | } | ||
722 | { | ||
723 | const uint16x4_t equal0_u16x4 = vceq_s16( q1_Q0_s16x4, vdup_n_s16( 0 ) ); | ||
724 | const uint16x4_t equalMinus1_u16x4 = vceq_s16( q1_Q0_s16x4, vdup_n_s16( -1 ) ); | ||
725 | const uint16x4_t lessThanMinus1_u16x4 = vclt_s16( q1_Q0_s16x4, vdup_n_s16( -1 ) ); | ||
726 | int16x4_t tmp1_s16x4, tmp2_s16x4; | ||
727 | |||
728 | q1_Q10_s16x4 = vshl_n_s16( q1_Q0_s16x4, 10 ); | ||
729 | tmp1_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( offset_Q10 - QUANT_LEVEL_ADJUST_Q10 ) ); | ||
730 | q1_Q10_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( offset_Q10 + QUANT_LEVEL_ADJUST_Q10 ) ); | ||
731 | q1_Q10_s16x4 = vbsl_s16( lessThanMinus1_u16x4, q1_Q10_s16x4, tmp1_s16x4 ); | ||
732 | q1_Q10_s16x4 = vbsl_s16( equal0_u16x4, vdup_n_s16( offset_Q10 ), q1_Q10_s16x4 ); | ||
733 | q1_Q10_s16x4 = vbsl_s16( equalMinus1_u16x4, vdup_n_s16( offset_Q10 - ( 1024 - QUANT_LEVEL_ADJUST_Q10 ) ), q1_Q10_s16x4 ); | ||
734 | q2_Q10_s16x4 = vadd_s16( q1_Q10_s16x4, vdup_n_s16( 1024 ) ); | ||
735 | q2_Q10_s16x4 = vbsl_s16( equal0_u16x4, vdup_n_s16( offset_Q10 + 1024 - QUANT_LEVEL_ADJUST_Q10 ), q2_Q10_s16x4 ); | ||
736 | q2_Q10_s16x4 = vbsl_s16( equalMinus1_u16x4, vdup_n_s16( offset_Q10 ), q2_Q10_s16x4 ); | ||
737 | tmp1_s16x4 = q1_Q10_s16x4; | ||
738 | tmp2_s16x4 = q2_Q10_s16x4; | ||
739 | tmp1_s16x4 = vbsl_s16( vorr_u16( equalMinus1_u16x4, lessThanMinus1_u16x4 ), vneg_s16( tmp1_s16x4 ), tmp1_s16x4 ); | ||
740 | tmp2_s16x4 = vbsl_s16( lessThanMinus1_u16x4, vneg_s16( tmp2_s16x4 ), tmp2_s16x4 ); | ||
741 | rd1_Q10_s32x4 = vmull_s16( tmp1_s16x4, vdup_n_s16( Lambda_Q10 ) ); | ||
742 | rd2_Q10_s32x4 = vmull_s16( tmp2_s16x4, vdup_n_s16( Lambda_Q10 ) ); | ||
743 | } | ||
744 | |||
745 | rr_Q10_s16x4 = vsub_s16( r_Q10_s16x4, q1_Q10_s16x4 ); | ||
746 | rd1_Q10_s32x4 = vmlal_s16( rd1_Q10_s32x4, rr_Q10_s16x4, rr_Q10_s16x4 ); | ||
747 | rd1_Q10_s32x4 = vshrq_n_s32( rd1_Q10_s32x4, 10 ); | ||
748 | |||
749 | rr_Q10_s16x4 = vsub_s16( r_Q10_s16x4, q2_Q10_s16x4 ); | ||
750 | rd2_Q10_s32x4 = vmlal_s16( rd2_Q10_s32x4, rr_Q10_s16x4, rr_Q10_s16x4 ); | ||
751 | rd2_Q10_s32x4 = vshrq_n_s32( rd2_Q10_s32x4, 10 ); | ||
752 | |||
753 | tmp2_s32x4 = vld1q_s32( psDelDec->RD_Q10 ); | ||
754 | tmp1_s32x4 = vaddq_s32( tmp2_s32x4, vminq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ) ); | ||
755 | tmp2_s32x4 = vaddq_s32( tmp2_s32x4, vmaxq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ) ); | ||
756 | vst1q_s32( psSampleState[ 0 ].RD_Q10, tmp1_s32x4 ); | ||
757 | vst1q_s32( psSampleState[ 1 ].RD_Q10, tmp2_s32x4 ); | ||
758 | t_u32x4 = vcltq_s32( rd1_Q10_s32x4, rd2_Q10_s32x4 ); | ||
759 | tmp1_s32x4 = vbslq_s32( t_u32x4, vmovl_s16( q1_Q10_s16x4 ), vmovl_s16( q2_Q10_s16x4 ) ); | ||
760 | tmp2_s32x4 = vbslq_s32( t_u32x4, vmovl_s16( q2_Q10_s16x4 ), vmovl_s16( q1_Q10_s16x4 ) ); | ||
761 | vst1q_s32( psSampleState[ 0 ].Q_Q10, tmp1_s32x4 ); | ||
762 | vst1q_s32( psSampleState[ 1 ].Q_Q10, tmp2_s32x4 ); | ||
763 | } | ||
764 | |||
765 | { | ||
766 | /* Update states for best quantization */ | ||
767 | int32x4_t exc_Q14_s32x4, LPC_exc_Q14_s32x4, xq_Q14_s32x4, sLF_AR_shp_Q14_s32x4; | ||
768 | |||
769 | /* Quantized excitation */ | ||
770 | exc_Q14_s32x4 = vshlq_n_s32( tmp1_s32x4, 4 ); | ||
771 | exc_Q14_s32x4 = veorq_s32( exc_Q14_s32x4, sign_s32x4 ); | ||
772 | exc_Q14_s32x4 = vsubq_s32( exc_Q14_s32x4, sign_s32x4 ); | ||
773 | |||
774 | /* Add predictions */ | ||
775 | LPC_exc_Q14_s32x4 = vaddq_s32( exc_Q14_s32x4, vdupq_n_s32( LTP_pred_Q14 ) ); | ||
776 | xq_Q14_s32x4 = vaddq_s32( LPC_exc_Q14_s32x4, LPC_pred_Q14_s32x4 ); | ||
777 | |||
778 | /* Update states */ | ||
779 | tmp1_s32x4 = vsubq_s32( xq_Q14_s32x4, vshlq_n_s32( vdupq_n_s32( x_Q10[ i ] ), 4 ) ); | ||
780 | vst1q_s32( psSampleState[ 0 ].Diff_Q14, tmp1_s32x4 ); | ||
781 | sLF_AR_shp_Q14_s32x4 = vsubq_s32( tmp1_s32x4, n_AR_Q14_s32x4 ); | ||
782 | vst1q_s32( psSampleState[ 0 ].sLTP_shp_Q14, vsubq_s32( sLF_AR_shp_Q14_s32x4, n_LF_Q14_s32x4 ) ); | ||
783 | vst1q_s32( psSampleState[ 0 ].LF_AR_Q14, sLF_AR_shp_Q14_s32x4 ); | ||
784 | vst1q_s32( psSampleState[ 0 ].LPC_exc_Q14, LPC_exc_Q14_s32x4 ); | ||
785 | vst1q_s32( psSampleState[ 0 ].xq_Q14, xq_Q14_s32x4 ); | ||
786 | |||
787 | /* Quantized excitation */ | ||
788 | exc_Q14_s32x4 = vshlq_n_s32( tmp2_s32x4, 4 ); | ||
789 | exc_Q14_s32x4 = veorq_s32( exc_Q14_s32x4, sign_s32x4 ); | ||
790 | exc_Q14_s32x4 = vsubq_s32( exc_Q14_s32x4, sign_s32x4 ); | ||
791 | |||
792 | /* Add predictions */ | ||
793 | LPC_exc_Q14_s32x4 = vaddq_s32( exc_Q14_s32x4, vdupq_n_s32( LTP_pred_Q14 ) ); | ||
794 | xq_Q14_s32x4 = vaddq_s32( LPC_exc_Q14_s32x4, LPC_pred_Q14_s32x4 ); | ||
795 | |||
796 | /* Update states */ | ||
797 | tmp1_s32x4 = vsubq_s32( xq_Q14_s32x4, vshlq_n_s32( vdupq_n_s32( x_Q10[ i ] ), 4 ) ); | ||
798 | vst1q_s32( psSampleState[ 1 ].Diff_Q14, tmp1_s32x4 ); | ||
799 | sLF_AR_shp_Q14_s32x4 = vsubq_s32( tmp1_s32x4, n_AR_Q14_s32x4 ); | ||
800 | vst1q_s32( psSampleState[ 1 ].sLTP_shp_Q14, vsubq_s32( sLF_AR_shp_Q14_s32x4, n_LF_Q14_s32x4 ) ); | ||
801 | vst1q_s32( psSampleState[ 1 ].LF_AR_Q14, sLF_AR_shp_Q14_s32x4 ); | ||
802 | vst1q_s32( psSampleState[ 1 ].LPC_exc_Q14, LPC_exc_Q14_s32x4 ); | ||
803 | vst1q_s32( psSampleState[ 1 ].xq_Q14, xq_Q14_s32x4 ); | ||
804 | } | ||
805 | |||
806 | *smpl_buf_idx = *smpl_buf_idx ? ( *smpl_buf_idx - 1 ) : ( DECISION_DELAY - 1); | ||
807 | last_smple_idx = *smpl_buf_idx + decisionDelay + DECISION_DELAY; | ||
808 | if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; | ||
809 | if( last_smple_idx >= DECISION_DELAY ) last_smple_idx -= DECISION_DELAY; | ||
810 | |||
811 | /* Find winner */ | ||
812 | RDmin_Q10 = psSampleState[ 0 ].RD_Q10[ 0 ]; | ||
813 | Winner_ind = 0; | ||
814 | for( k = 1; k < nStatesDelayedDecision; k++ ) { | ||
815 | if( psSampleState[ 0 ].RD_Q10[ k ] < RDmin_Q10 ) { | ||
816 | RDmin_Q10 = psSampleState[ 0 ].RD_Q10[ k ]; | ||
817 | Winner_ind = k; | ||
818 | } | ||
819 | } | ||
820 | |||
821 | /* Increase RD values of expired states */ | ||
822 | { | ||
823 | uint32x4_t t_u32x4; | ||
824 | Winner_rand_state = psDelDec->RandState[ last_smple_idx ][ Winner_ind ]; | ||
825 | t_u32x4 = vceqq_s32( vld1q_s32( psDelDec->RandState[ last_smple_idx ] ), vdupq_n_s32( Winner_rand_state ) ); | ||
826 | t_u32x4 = vmvnq_u32( t_u32x4 ); | ||
827 | t_u32x4 = vshrq_n_u32( t_u32x4, 5 ); | ||
828 | tmp1_s32x4 = vld1q_s32( psSampleState[ 0 ].RD_Q10 ); | ||
829 | tmp2_s32x4 = vld1q_s32( psSampleState[ 1 ].RD_Q10 ); | ||
830 | tmp1_s32x4 = vaddq_s32( tmp1_s32x4, vreinterpretq_s32_u32( t_u32x4 ) ); | ||
831 | tmp2_s32x4 = vaddq_s32( tmp2_s32x4, vreinterpretq_s32_u32( t_u32x4 ) ); | ||
832 | vst1q_s32( psSampleState[ 0 ].RD_Q10, tmp1_s32x4 ); | ||
833 | vst1q_s32( psSampleState[ 1 ].RD_Q10, tmp2_s32x4 ); | ||
834 | |||
835 | /* Find worst in first set and best in second set */ | ||
836 | RDmax_Q10 = psSampleState[ 0 ].RD_Q10[ 0 ]; | ||
837 | RDmin_Q10 = psSampleState[ 1 ].RD_Q10[ 0 ]; | ||
838 | RDmax_ind = 0; | ||
839 | RDmin_ind = 0; | ||
840 | for( k = 1; k < nStatesDelayedDecision; k++ ) { | ||
841 | /* find worst in first set */ | ||
842 | if( psSampleState[ 0 ].RD_Q10[ k ] > RDmax_Q10 ) { | ||
843 | RDmax_Q10 = psSampleState[ 0 ].RD_Q10[ k ]; | ||
844 | RDmax_ind = k; | ||
845 | } | ||
846 | /* find best in second set */ | ||
847 | if( psSampleState[ 1 ].RD_Q10[ k ] < RDmin_Q10 ) { | ||
848 | RDmin_Q10 = psSampleState[ 1 ].RD_Q10[ k ]; | ||
849 | RDmin_ind = k; | ||
850 | } | ||
851 | } | ||
852 | } | ||
853 | |||
854 | /* Replace a state if best from second set outperforms worst in first set */ | ||
855 | if( RDmin_Q10 < RDmax_Q10 ) { | ||
856 | opus_int32 (*ptr)[NEON_MAX_DEL_DEC_STATES] = psDelDec->RandState; | ||
857 | const int numOthers = (int)( ( sizeof( NSQ_del_decs_struct ) - sizeof( ( (NSQ_del_decs_struct *)0 )->sLPC_Q14 ) ) | ||
858 | / ( NEON_MAX_DEL_DEC_STATES * sizeof( opus_int32 ) ) ); | ||
859 | /* Only ( predictLPCOrder - 1 ) of sLPC_Q14 buffer need to be updated, though the first several */ | ||
860 | /* useless sLPC_Q14[] will be different comparing with C when predictLPCOrder < NSQ_LPC_BUF_LENGTH. */ | ||
861 | /* Here just update constant ( NSQ_LPC_BUF_LENGTH - 1 ) for simplicity. */ | ||
862 | for( j = i + 1; j < i + NSQ_LPC_BUF_LENGTH; j++ ) { | ||
863 | psDelDec->sLPC_Q14[ j ][ RDmax_ind ] = psDelDec->sLPC_Q14[ j ][ RDmin_ind ]; | ||
864 | } | ||
865 | for( j = 0; j < numOthers; j++ ) { | ||
866 | ptr[ j ][ RDmax_ind ] = ptr[ j ][ RDmin_ind ]; | ||
867 | } | ||
868 | |||
869 | psSampleState[ 0 ].Q_Q10[ RDmax_ind ] = psSampleState[ 1 ].Q_Q10[ RDmin_ind ]; | ||
870 | psSampleState[ 0 ].RD_Q10[ RDmax_ind ] = psSampleState[ 1 ].RD_Q10[ RDmin_ind ]; | ||
871 | psSampleState[ 0 ].xq_Q14[ RDmax_ind ] = psSampleState[ 1 ].xq_Q14[ RDmin_ind ]; | ||
872 | psSampleState[ 0 ].LF_AR_Q14[ RDmax_ind ] = psSampleState[ 1 ].LF_AR_Q14[ RDmin_ind ]; | ||
873 | psSampleState[ 0 ].Diff_Q14[ RDmax_ind ] = psSampleState[ 1 ].Diff_Q14[ RDmin_ind ]; | ||
874 | psSampleState[ 0 ].sLTP_shp_Q14[ RDmax_ind ] = psSampleState[ 1 ].sLTP_shp_Q14[ RDmin_ind ]; | ||
875 | psSampleState[ 0 ].LPC_exc_Q14[ RDmax_ind ] = psSampleState[ 1 ].LPC_exc_Q14[ RDmin_ind ]; | ||
876 | } | ||
877 | |||
878 | /* Write samples from winner to output and long-term filter states */ | ||
879 | if( subfr > 0 || i >= decisionDelay ) { | ||
880 | pulses[ i - decisionDelay ] = (opus_int8)silk_RSHIFT_ROUND( psDelDec->Q_Q10[ last_smple_idx ][ Winner_ind ], 10 ); | ||
881 | xq[ i - decisionDelay ] = (opus_int16)silk_SAT16( silk_RSHIFT_ROUND( | ||
882 | silk_SMULWW( psDelDec->Xq_Q14[ last_smple_idx ][ Winner_ind ], delayedGain_Q10[ last_smple_idx ] ), 8 ) ); | ||
883 | NSQ->sLTP_shp_Q14[ NSQ->sLTP_shp_buf_idx - decisionDelay ] = psDelDec->Shape_Q14[ last_smple_idx ][ Winner_ind ]; | ||
884 | sLTP_Q15[ NSQ->sLTP_buf_idx - decisionDelay ] = psDelDec->Pred_Q15[ last_smple_idx ][ Winner_ind ]; | ||
885 | } | ||
886 | NSQ->sLTP_shp_buf_idx++; | ||
887 | NSQ->sLTP_buf_idx++; | ||
888 | |||
889 | /* Update states */ | ||
890 | vst1q_s32( psDelDec->LF_AR_Q14, vld1q_s32( psSampleState[ 0 ].LF_AR_Q14 ) ); | ||
891 | vst1q_s32( psDelDec->Diff_Q14, vld1q_s32( psSampleState[ 0 ].Diff_Q14 ) ); | ||
892 | vst1q_s32( psDelDec->sLPC_Q14[ NSQ_LPC_BUF_LENGTH + i ], vld1q_s32( psSampleState[ 0 ].xq_Q14 ) ); | ||
893 | vst1q_s32( psDelDec->Xq_Q14[ *smpl_buf_idx ], vld1q_s32( psSampleState[ 0 ].xq_Q14 ) ); | ||
894 | tmp1_s32x4 = vld1q_s32( psSampleState[ 0 ].Q_Q10 ); | ||
895 | vst1q_s32( psDelDec->Q_Q10[ *smpl_buf_idx ], tmp1_s32x4 ); | ||
896 | vst1q_s32( psDelDec->Pred_Q15[ *smpl_buf_idx ], vshlq_n_s32( vld1q_s32( psSampleState[ 0 ].LPC_exc_Q14 ), 1 ) ); | ||
897 | vst1q_s32( psDelDec->Shape_Q14[ *smpl_buf_idx ], vld1q_s32( psSampleState[ 0 ].sLTP_shp_Q14 ) ); | ||
898 | tmp1_s32x4 = vrshrq_n_s32( tmp1_s32x4, 10 ); | ||
899 | tmp1_s32x4 = vaddq_s32( vld1q_s32( psDelDec->Seed ), tmp1_s32x4 ); | ||
900 | vst1q_s32( psDelDec->Seed, tmp1_s32x4 ); | ||
901 | vst1q_s32( psDelDec->RandState[ *smpl_buf_idx ], tmp1_s32x4 ); | ||
902 | vst1q_s32( psDelDec->RD_Q10, vld1q_s32( psSampleState[ 0 ].RD_Q10 ) ); | ||
903 | delayedGain_Q10[ *smpl_buf_idx ] = Gain_Q10; | ||
904 | } | ||
905 | /* Update LPC states */ | ||
906 | silk_memcpy( psDelDec->sLPC_Q14[ 0 ], psDelDec->sLPC_Q14[ length ], NEON_MAX_DEL_DEC_STATES * NSQ_LPC_BUF_LENGTH * sizeof( opus_int32 ) ); | ||
907 | |||
908 | RESTORE_STACK; | ||
909 | } | ||
910 | |||
911 | static OPUS_INLINE void silk_SMULWB_8_neon( | ||
912 | const opus_int16 *a, | ||
913 | const int32x2_t b, | ||
914 | opus_int32 *o | ||
915 | ) | ||
916 | { | ||
917 | const int16x8_t a_s16x8 = vld1q_s16( a ); | ||
918 | int32x4_t o0_s32x4, o1_s32x4; | ||
919 | |||
920 | o0_s32x4 = vshll_n_s16( vget_low_s16( a_s16x8 ), 15 ); | ||
921 | o1_s32x4 = vshll_n_s16( vget_high_s16( a_s16x8 ), 15 ); | ||
922 | o0_s32x4 = vqdmulhq_lane_s32( o0_s32x4, b, 0 ); | ||
923 | o1_s32x4 = vqdmulhq_lane_s32( o1_s32x4, b, 0 ); | ||
924 | vst1q_s32( o, o0_s32x4 ); | ||
925 | vst1q_s32( o + 4, o1_s32x4 ); | ||
926 | } | ||
927 | |||
928 | /* Only works when ( b >= -65536 ) && ( b < 65536 ). */ | ||
929 | static OPUS_INLINE void silk_SMULWW_small_b_4_neon( | ||
930 | opus_int32 *a, | ||
931 | const int32x2_t b_s32x2) | ||
932 | { | ||
933 | int32x4_t o_s32x4; | ||
934 | |||
935 | o_s32x4 = vld1q_s32( a ); | ||
936 | o_s32x4 = vqdmulhq_lane_s32( o_s32x4, b_s32x2, 0 ); | ||
937 | vst1q_s32( a, o_s32x4 ); | ||
938 | } | ||
939 | |||
940 | /* Only works when ( b >= -65536 ) && ( b < 65536 ). */ | ||
941 | static OPUS_INLINE void silk_SMULWW_small_b_8_neon( | ||
942 | opus_int32 *a, | ||
943 | const int32x2_t b_s32x2 | ||
944 | ) | ||
945 | { | ||
946 | int32x4_t o0_s32x4, o1_s32x4; | ||
947 | |||
948 | o0_s32x4 = vld1q_s32( a ); | ||
949 | o1_s32x4 = vld1q_s32( a + 4 ); | ||
950 | o0_s32x4 = vqdmulhq_lane_s32( o0_s32x4, b_s32x2, 0 ); | ||
951 | o1_s32x4 = vqdmulhq_lane_s32( o1_s32x4, b_s32x2, 0 ); | ||
952 | vst1q_s32( a, o0_s32x4 ); | ||
953 | vst1q_s32( a + 4, o1_s32x4 ); | ||
954 | } | ||
955 | |||
956 | static OPUS_INLINE void silk_SMULWW_4_neon( | ||
957 | opus_int32 *a, | ||
958 | const int32x2_t b_s32x2) | ||
959 | { | ||
960 | int32x4_t a_s32x4, o_s32x4; | ||
961 | |||
962 | a_s32x4 = vld1q_s32( a ); | ||
963 | o_s32x4 = vqdmulhq_lane_s32( a_s32x4, b_s32x2, 0 ); | ||
964 | o_s32x4 = vmlaq_lane_s32( o_s32x4, a_s32x4, b_s32x2, 1 ); | ||
965 | vst1q_s32( a, o_s32x4 ); | ||
966 | } | ||
967 | |||
968 | static OPUS_INLINE void silk_SMULWW_8_neon( | ||
969 | opus_int32 *a, | ||
970 | const int32x2_t b_s32x2 | ||
971 | ) | ||
972 | { | ||
973 | int32x4_t a0_s32x4, a1_s32x4, o0_s32x4, o1_s32x4; | ||
974 | |||
975 | a0_s32x4 = vld1q_s32( a ); | ||
976 | a1_s32x4 = vld1q_s32( a + 4 ); | ||
977 | o0_s32x4 = vqdmulhq_lane_s32( a0_s32x4, b_s32x2, 0 ); | ||
978 | o1_s32x4 = vqdmulhq_lane_s32( a1_s32x4, b_s32x2, 0 ); | ||
979 | o0_s32x4 = vmlaq_lane_s32( o0_s32x4, a0_s32x4, b_s32x2, 1 ); | ||
980 | o1_s32x4 = vmlaq_lane_s32( o1_s32x4, a1_s32x4, b_s32x2, 1 ); | ||
981 | vst1q_s32( a, o0_s32x4 ); | ||
982 | vst1q_s32( a + 4, o1_s32x4 ); | ||
983 | } | ||
984 | |||
985 | static OPUS_INLINE void silk_SMULWW_loop_neon( | ||
986 | const opus_int16 *a, | ||
987 | const opus_int32 b, | ||
988 | opus_int32 *o, | ||
989 | const opus_int loop_num | ||
990 | ) | ||
991 | { | ||
992 | opus_int i; | ||
993 | int32x2_t b_s32x2; | ||
994 | |||
995 | b_s32x2 = vdup_n_s32( b ); | ||
996 | for( i = 0; i < loop_num - 7; i += 8 ) { | ||
997 | silk_SMULWB_8_neon( a + i, b_s32x2, o + i ); | ||
998 | } | ||
999 | for( ; i < loop_num; i++ ) { | ||
1000 | o[ i ] = silk_SMULWW( a[ i ], b ); | ||
1001 | } | ||
1002 | } | ||
1003 | |||
1004 | static OPUS_INLINE void silk_nsq_del_dec_scale_states_neon( | ||
1005 | const silk_encoder_state *psEncC, /* I Encoder State */ | ||
1006 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
1007 | NSQ_del_decs_struct psDelDec[], /* I/O Delayed decision states */ | ||
1008 | const opus_int16 x16[], /* I Input */ | ||
1009 | opus_int32 x_sc_Q10[], /* O Input scaled with 1/Gain in Q10 */ | ||
1010 | const opus_int16 sLTP[], /* I Re-whitened LTP state in Q0 */ | ||
1011 | opus_int32 sLTP_Q15[], /* O LTP state matching scaled input */ | ||
1012 | opus_int subfr, /* I Subframe number */ | ||
1013 | const opus_int LTP_scale_Q14, /* I LTP state scaling */ | ||
1014 | const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I */ | ||
1015 | const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lag */ | ||
1016 | const opus_int signal_type, /* I Signal type */ | ||
1017 | const opus_int decisionDelay /* I Decision delay */ | ||
1018 | ) | ||
1019 | { | ||
1020 | opus_int i, lag; | ||
1021 | opus_int32 gain_adj_Q16, inv_gain_Q31, inv_gain_Q26; | ||
1022 | |||
1023 | lag = pitchL[ subfr ]; | ||
1024 | inv_gain_Q31 = silk_INVERSE32_varQ( silk_max( Gains_Q16[ subfr ], 1 ), 47 ); | ||
1025 | silk_assert( inv_gain_Q31 != 0 ); | ||
1026 | |||
1027 | /* Scale input */ | ||
1028 | inv_gain_Q26 = silk_RSHIFT_ROUND( inv_gain_Q31, 5 ); | ||
1029 | silk_SMULWW_loop_neon( x16, inv_gain_Q26, x_sc_Q10, psEncC->subfr_length ); | ||
1030 | |||
1031 | /* After rewhitening the LTP state is un-scaled, so scale with inv_gain_Q16 */ | ||
1032 | if( NSQ->rewhite_flag ) { | ||
1033 | if( subfr == 0 ) { | ||
1034 | /* Do LTP downscaling */ | ||
1035 | inv_gain_Q31 = silk_LSHIFT( silk_SMULWB( inv_gain_Q31, LTP_scale_Q14 ), 2 ); | ||
1036 | } | ||
1037 | silk_SMULWW_loop_neon( sLTP + NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2, inv_gain_Q31, sLTP_Q15 + NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2, lag + LTP_ORDER / 2 ); | ||
1038 | } | ||
1039 | |||
1040 | /* Adjust for changing gain */ | ||
1041 | if( Gains_Q16[ subfr ] != NSQ->prev_gain_Q16 ) { | ||
1042 | int32x2_t gain_adj_Q16_s32x2; | ||
1043 | gain_adj_Q16 = silk_DIV32_varQ( NSQ->prev_gain_Q16, Gains_Q16[ subfr ], 16 ); | ||
1044 | |||
1045 | /* Scale long-term shaping state */ | ||
1046 | if( ( gain_adj_Q16 >= -65536 ) && ( gain_adj_Q16 < 65536 ) ) { | ||
1047 | gain_adj_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( gain_adj_Q16, 15 ) ); | ||
1048 | for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 7; i += 8 ) { | ||
1049 | silk_SMULWW_small_b_8_neon( NSQ->sLTP_shp_Q14 + i, gain_adj_Q16_s32x2 ); | ||
1050 | } | ||
1051 | for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { | ||
1052 | NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); | ||
1053 | } | ||
1054 | |||
1055 | /* Scale long-term prediction state */ | ||
1056 | if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { | ||
1057 | for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay - 7; i += 8 ) { | ||
1058 | silk_SMULWW_small_b_8_neon( sLTP_Q15 + i, gain_adj_Q16_s32x2 ); | ||
1059 | } | ||
1060 | for( ; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { | ||
1061 | sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); | ||
1062 | } | ||
1063 | } | ||
1064 | |||
1065 | /* Scale scalar states */ | ||
1066 | silk_SMULWW_small_b_4_neon( psDelDec->LF_AR_Q14, gain_adj_Q16_s32x2 ); | ||
1067 | silk_SMULWW_small_b_4_neon( psDelDec->Diff_Q14, gain_adj_Q16_s32x2 ); | ||
1068 | |||
1069 | /* Scale short-term prediction and shaping states */ | ||
1070 | for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { | ||
1071 | silk_SMULWW_small_b_4_neon( psDelDec->sLPC_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1072 | } | ||
1073 | |||
1074 | for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { | ||
1075 | silk_SMULWW_small_b_4_neon( psDelDec->sAR2_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1076 | } | ||
1077 | |||
1078 | for( i = 0; i < DECISION_DELAY; i++ ) { | ||
1079 | silk_SMULWW_small_b_4_neon( psDelDec->Pred_Q15[ i ], gain_adj_Q16_s32x2 ); | ||
1080 | silk_SMULWW_small_b_4_neon( psDelDec->Shape_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1081 | } | ||
1082 | } else { | ||
1083 | gain_adj_Q16_s32x2 = vdup_n_s32( silk_LSHIFT32( gain_adj_Q16 & 0x0000FFFF, 15 ) ); | ||
1084 | gain_adj_Q16_s32x2 = vset_lane_s32( gain_adj_Q16 >> 16, gain_adj_Q16_s32x2, 1 ); | ||
1085 | for( i = NSQ->sLTP_shp_buf_idx - psEncC->ltp_mem_length; i < NSQ->sLTP_shp_buf_idx - 7; i += 8 ) { | ||
1086 | silk_SMULWW_8_neon( NSQ->sLTP_shp_Q14 + i, gain_adj_Q16_s32x2 ); | ||
1087 | } | ||
1088 | for( ; i < NSQ->sLTP_shp_buf_idx; i++ ) { | ||
1089 | NSQ->sLTP_shp_Q14[ i ] = silk_SMULWW( gain_adj_Q16, NSQ->sLTP_shp_Q14[ i ] ); | ||
1090 | } | ||
1091 | |||
1092 | /* Scale long-term prediction state */ | ||
1093 | if( signal_type == TYPE_VOICED && NSQ->rewhite_flag == 0 ) { | ||
1094 | for( i = NSQ->sLTP_buf_idx - lag - LTP_ORDER / 2; i < NSQ->sLTP_buf_idx - decisionDelay - 7; i += 8 ) { | ||
1095 | silk_SMULWW_8_neon( sLTP_Q15 + i, gain_adj_Q16_s32x2 ); | ||
1096 | } | ||
1097 | for( ; i < NSQ->sLTP_buf_idx - decisionDelay; i++ ) { | ||
1098 | sLTP_Q15[ i ] = silk_SMULWW( gain_adj_Q16, sLTP_Q15[ i ] ); | ||
1099 | } | ||
1100 | } | ||
1101 | |||
1102 | /* Scale scalar states */ | ||
1103 | silk_SMULWW_4_neon( psDelDec->LF_AR_Q14, gain_adj_Q16_s32x2 ); | ||
1104 | silk_SMULWW_4_neon( psDelDec->Diff_Q14, gain_adj_Q16_s32x2 ); | ||
1105 | |||
1106 | /* Scale short-term prediction and shaping states */ | ||
1107 | for( i = 0; i < NSQ_LPC_BUF_LENGTH; i++ ) { | ||
1108 | silk_SMULWW_4_neon( psDelDec->sLPC_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1109 | } | ||
1110 | |||
1111 | for( i = 0; i < MAX_SHAPE_LPC_ORDER; i++ ) { | ||
1112 | silk_SMULWW_4_neon( psDelDec->sAR2_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1113 | } | ||
1114 | |||
1115 | for( i = 0; i < DECISION_DELAY; i++ ) { | ||
1116 | silk_SMULWW_4_neon( psDelDec->Pred_Q15[ i ], gain_adj_Q16_s32x2 ); | ||
1117 | silk_SMULWW_4_neon( psDelDec->Shape_Q14[ i ], gain_adj_Q16_s32x2 ); | ||
1118 | } | ||
1119 | } | ||
1120 | |||
1121 | /* Save inverse gain */ | ||
1122 | NSQ->prev_gain_Q16 = Gains_Q16[ subfr ]; | ||
1123 | } | ||
1124 | } | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.c b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.c new file mode 100644 index 0000000000..9642529973 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.c | |||
@@ -0,0 +1,112 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (C) 2014 Vidyo | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | #ifdef HAVE_CONFIG_H | ||
28 | #include "config.h" | ||
29 | #endif | ||
30 | |||
31 | #include <arm_neon.h> | ||
32 | #include "main.h" | ||
33 | #include "stack_alloc.h" | ||
34 | #include "NSQ.h" | ||
35 | #include "celt/cpu_support.h" | ||
36 | #include "celt/arm/armcpu.h" | ||
37 | |||
38 | opus_int32 silk_noise_shape_quantizer_short_prediction_neon(const opus_int32 *buf32, const opus_int32 *coef32, opus_int order) | ||
39 | { | ||
40 | int32x4_t coef0 = vld1q_s32(coef32); | ||
41 | int32x4_t coef1 = vld1q_s32(coef32 + 4); | ||
42 | int32x4_t coef2 = vld1q_s32(coef32 + 8); | ||
43 | int32x4_t coef3 = vld1q_s32(coef32 + 12); | ||
44 | |||
45 | int32x4_t a0 = vld1q_s32(buf32 - 15); | ||
46 | int32x4_t a1 = vld1q_s32(buf32 - 11); | ||
47 | int32x4_t a2 = vld1q_s32(buf32 - 7); | ||
48 | int32x4_t a3 = vld1q_s32(buf32 - 3); | ||
49 | |||
50 | int32x4_t b0 = vqdmulhq_s32(coef0, a0); | ||
51 | int32x4_t b1 = vqdmulhq_s32(coef1, a1); | ||
52 | int32x4_t b2 = vqdmulhq_s32(coef2, a2); | ||
53 | int32x4_t b3 = vqdmulhq_s32(coef3, a3); | ||
54 | |||
55 | int32x4_t c0 = vaddq_s32(b0, b1); | ||
56 | int32x4_t c1 = vaddq_s32(b2, b3); | ||
57 | |||
58 | int32x4_t d = vaddq_s32(c0, c1); | ||
59 | |||
60 | int64x2_t e = vpaddlq_s32(d); | ||
61 | |||
62 | int64x1_t f = vadd_s64(vget_low_s64(e), vget_high_s64(e)); | ||
63 | |||
64 | opus_int32 out = vget_lane_s32(vreinterpret_s32_s64(f), 0); | ||
65 | |||
66 | out += silk_RSHIFT( order, 1 ); | ||
67 | |||
68 | return out; | ||
69 | } | ||
70 | |||
71 | |||
72 | opus_int32 silk_NSQ_noise_shape_feedback_loop_neon(const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, opus_int order) | ||
73 | { | ||
74 | opus_int32 out; | ||
75 | if (order == 8) | ||
76 | { | ||
77 | int32x4_t a00 = vdupq_n_s32(data0[0]); | ||
78 | int32x4_t a01 = vld1q_s32(data1); /* data1[0] ... [3] */ | ||
79 | |||
80 | int32x4_t a0 = vextq_s32 (a00, a01, 3); /* data0[0] data1[0] ...[2] */ | ||
81 | int32x4_t a1 = vld1q_s32(data1 + 3); /* data1[3] ... [6] */ | ||
82 | |||
83 | /*TODO: Convert these once in advance instead of once per sample, like | ||
84 | silk_noise_shape_quantizer_short_prediction_neon() does.*/ | ||
85 | int16x8_t coef16 = vld1q_s16(coef); | ||
86 | int32x4_t coef0 = vmovl_s16(vget_low_s16(coef16)); | ||
87 | int32x4_t coef1 = vmovl_s16(vget_high_s16(coef16)); | ||
88 | |||
89 | /*This is not bit-exact with the C version, since we do not drop the | ||
90 | lower 16 bits of each multiply, but wait until the end to truncate | ||
91 | precision. This is an encoder-specific calculation (and unlike | ||
92 | silk_noise_shape_quantizer_short_prediction_neon(), is not meant to | ||
93 | simulate what the decoder will do). We still could use vqdmulhq_s32() | ||
94 | like silk_noise_shape_quantizer_short_prediction_neon() and save | ||
95 | half the multiplies, but the speed difference is not large, since we | ||
96 | then need two extra adds.*/ | ||
97 | int64x2_t b0 = vmull_s32(vget_low_s32(a0), vget_low_s32(coef0)); | ||
98 | int64x2_t b1 = vmlal_s32(b0, vget_high_s32(a0), vget_high_s32(coef0)); | ||
99 | int64x2_t b2 = vmlal_s32(b1, vget_low_s32(a1), vget_low_s32(coef1)); | ||
100 | int64x2_t b3 = vmlal_s32(b2, vget_high_s32(a1), vget_high_s32(coef1)); | ||
101 | |||
102 | int64x1_t c = vadd_s64(vget_low_s64(b3), vget_high_s64(b3)); | ||
103 | int64x1_t cS = vrshr_n_s64(c, 15); | ||
104 | int32x2_t d = vreinterpret_s32_s64(cS); | ||
105 | |||
106 | out = vget_lane_s32(d, 0); | ||
107 | vst1q_s32(data1, a0); | ||
108 | vst1q_s32(data1 + 4, a1); | ||
109 | return out; | ||
110 | } | ||
111 | return silk_NSQ_noise_shape_feedback_loop_c(data0, data1, coef, order); | ||
112 | } | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.h b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.h new file mode 100644 index 0000000000..b31d9442d6 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/NSQ_neon.h | |||
@@ -0,0 +1,114 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (C) 2014 Vidyo | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | #ifndef SILK_NSQ_NEON_H | ||
28 | #define SILK_NSQ_NEON_H | ||
29 | |||
30 | #include "cpu_support.h" | ||
31 | #include "SigProc_FIX.h" | ||
32 | |||
33 | #undef silk_short_prediction_create_arch_coef | ||
34 | /* For vectorized calc, reverse a_Q12 coefs, convert to 32-bit, and shift for vqdmulhq_s32. */ | ||
35 | static OPUS_INLINE void silk_short_prediction_create_arch_coef_neon(opus_int32 *out, const opus_int16 *in, opus_int order) | ||
36 | { | ||
37 | out[15] = silk_LSHIFT32(in[0], 15); | ||
38 | out[14] = silk_LSHIFT32(in[1], 15); | ||
39 | out[13] = silk_LSHIFT32(in[2], 15); | ||
40 | out[12] = silk_LSHIFT32(in[3], 15); | ||
41 | out[11] = silk_LSHIFT32(in[4], 15); | ||
42 | out[10] = silk_LSHIFT32(in[5], 15); | ||
43 | out[9] = silk_LSHIFT32(in[6], 15); | ||
44 | out[8] = silk_LSHIFT32(in[7], 15); | ||
45 | out[7] = silk_LSHIFT32(in[8], 15); | ||
46 | out[6] = silk_LSHIFT32(in[9], 15); | ||
47 | |||
48 | if (order == 16) | ||
49 | { | ||
50 | out[5] = silk_LSHIFT32(in[10], 15); | ||
51 | out[4] = silk_LSHIFT32(in[11], 15); | ||
52 | out[3] = silk_LSHIFT32(in[12], 15); | ||
53 | out[2] = silk_LSHIFT32(in[13], 15); | ||
54 | out[1] = silk_LSHIFT32(in[14], 15); | ||
55 | out[0] = silk_LSHIFT32(in[15], 15); | ||
56 | } | ||
57 | else | ||
58 | { | ||
59 | out[5] = 0; | ||
60 | out[4] = 0; | ||
61 | out[3] = 0; | ||
62 | out[2] = 0; | ||
63 | out[1] = 0; | ||
64 | out[0] = 0; | ||
65 | } | ||
66 | } | ||
67 | |||
68 | #if defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
69 | |||
70 | #define silk_short_prediction_create_arch_coef(out, in, order) \ | ||
71 | (silk_short_prediction_create_arch_coef_neon(out, in, order)) | ||
72 | |||
73 | #elif defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_MAY_HAVE_NEON_INTR) | ||
74 | |||
75 | #define silk_short_prediction_create_arch_coef(out, in, order) \ | ||
76 | do { if (arch == OPUS_ARCH_ARM_NEON) { silk_short_prediction_create_arch_coef_neon(out, in, order); } } while (0) | ||
77 | |||
78 | #endif | ||
79 | |||
80 | opus_int32 silk_noise_shape_quantizer_short_prediction_neon(const opus_int32 *buf32, const opus_int32 *coef32, opus_int order); | ||
81 | |||
82 | opus_int32 silk_NSQ_noise_shape_feedback_loop_neon(const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, opus_int order); | ||
83 | |||
84 | #if defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
85 | #undef silk_noise_shape_quantizer_short_prediction | ||
86 | #define silk_noise_shape_quantizer_short_prediction(in, coef, coefRev, order, arch) \ | ||
87 | ((void)arch,silk_noise_shape_quantizer_short_prediction_neon(in, coefRev, order)) | ||
88 | |||
89 | #undef silk_NSQ_noise_shape_feedback_loop | ||
90 | #define silk_NSQ_noise_shape_feedback_loop(data0, data1, coef, order, arch) ((void)arch,silk_NSQ_noise_shape_feedback_loop_neon(data0, data1, coef, order)) | ||
91 | |||
92 | #elif defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_MAY_HAVE_NEON_INTR) | ||
93 | |||
94 | /* silk_noise_shape_quantizer_short_prediction implementations take different parameters based on arch | ||
95 | (coef vs. coefRev) so can't use the usual IMPL table implementation */ | ||
96 | #undef silk_noise_shape_quantizer_short_prediction | ||
97 | #define silk_noise_shape_quantizer_short_prediction(in, coef, coefRev, order, arch) \ | ||
98 | (arch == OPUS_ARCH_ARM_NEON ? \ | ||
99 | silk_noise_shape_quantizer_short_prediction_neon(in, coefRev, order) : \ | ||
100 | silk_noise_shape_quantizer_short_prediction_c(in, coef, order)) | ||
101 | |||
102 | extern opus_int32 | ||
103 | (*const SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[OPUS_ARCHMASK+1])( | ||
104 | const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, | ||
105 | opus_int order); | ||
106 | |||
107 | #undef silk_NSQ_noise_shape_feedback_loop | ||
108 | #define silk_NSQ_noise_shape_feedback_loop(data0, data1, coef, order, arch) \ | ||
109 | (SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[(arch)&OPUS_ARCHMASK](data0, data1, \ | ||
110 | coef, order)) | ||
111 | |||
112 | #endif | ||
113 | |||
114 | #endif /* SILK_NSQ_NEON_H */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/arm_silk_map.c b/lib/rbcodec/codecs/libopus/silk/arm/arm_silk_map.c new file mode 100644 index 0000000000..0b9bfec2ca --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/arm_silk_map.c | |||
@@ -0,0 +1,123 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (C) 2014 Vidyo | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | #ifdef HAVE_CONFIG_H | ||
28 | # include "config.h" | ||
29 | #endif | ||
30 | |||
31 | #include "main_FIX.h" | ||
32 | #include "NSQ.h" | ||
33 | #include "SigProc_FIX.h" | ||
34 | |||
35 | #if defined(OPUS_HAVE_RTCD) | ||
36 | |||
37 | # if (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && \ | ||
38 | !defined(OPUS_ARM_PRESUME_NEON_INTR)) | ||
39 | |||
40 | void (*const SILK_BIQUAD_ALT_STRIDE2_IMPL[OPUS_ARCHMASK + 1])( | ||
41 | const opus_int16 *in, /* I input signal */ | ||
42 | const opus_int32 *B_Q28, /* I MA coefficients [3] */ | ||
43 | const opus_int32 *A_Q28, /* I AR coefficients [2] */ | ||
44 | opus_int32 *S, /* I/O State vector [4] */ | ||
45 | opus_int16 *out, /* O output signal */ | ||
46 | const opus_int32 len /* I signal length (must be even) */ | ||
47 | ) = { | ||
48 | silk_biquad_alt_stride2_c, /* ARMv4 */ | ||
49 | silk_biquad_alt_stride2_c, /* EDSP */ | ||
50 | silk_biquad_alt_stride2_c, /* Media */ | ||
51 | silk_biquad_alt_stride2_neon, /* Neon */ | ||
52 | }; | ||
53 | |||
54 | opus_int32 (*const SILK_LPC_INVERSE_PRED_GAIN_IMPL[OPUS_ARCHMASK + 1])( /* O Returns inverse prediction gain in energy domain, Q30 */ | ||
55 | const opus_int16 *A_Q12, /* I Prediction coefficients, Q12 [order] */ | ||
56 | const opus_int order /* I Prediction order */ | ||
57 | ) = { | ||
58 | silk_LPC_inverse_pred_gain_c, /* ARMv4 */ | ||
59 | silk_LPC_inverse_pred_gain_c, /* EDSP */ | ||
60 | silk_LPC_inverse_pred_gain_c, /* Media */ | ||
61 | silk_LPC_inverse_pred_gain_neon, /* Neon */ | ||
62 | }; | ||
63 | |||
64 | void (*const SILK_NSQ_DEL_DEC_IMPL[OPUS_ARCHMASK + 1])( | ||
65 | const silk_encoder_state *psEncC, /* I Encoder State */ | ||
66 | silk_nsq_state *NSQ, /* I/O NSQ state */ | ||
67 | SideInfoIndices *psIndices, /* I/O Quantization Indices */ | ||
68 | const opus_int16 x16[], /* I Input */ | ||
69 | opus_int8 pulses[], /* O Quantized pulse signal */ | ||
70 | const opus_int16 PredCoef_Q12[ 2 * MAX_LPC_ORDER ], /* I Short term prediction coefs */ | ||
71 | const opus_int16 LTPCoef_Q14[ LTP_ORDER * MAX_NB_SUBFR ], /* I Long term prediction coefs */ | ||
72 | const opus_int16 AR_Q13[ MAX_NB_SUBFR * MAX_SHAPE_LPC_ORDER ], /* I Noise shaping coefs */ | ||
73 | const opus_int HarmShapeGain_Q14[ MAX_NB_SUBFR ], /* I Long term shaping coefs */ | ||
74 | const opus_int Tilt_Q14[ MAX_NB_SUBFR ], /* I Spectral tilt */ | ||
75 | const opus_int32 LF_shp_Q14[ MAX_NB_SUBFR ], /* I Low frequency shaping coefs */ | ||
76 | const opus_int32 Gains_Q16[ MAX_NB_SUBFR ], /* I Quantization step sizes */ | ||
77 | const opus_int pitchL[ MAX_NB_SUBFR ], /* I Pitch lags */ | ||
78 | const opus_int Lambda_Q10, /* I Rate/distortion tradeoff */ | ||
79 | const opus_int LTP_scale_Q14 /* I LTP state scaling */ | ||
80 | ) = { | ||
81 | silk_NSQ_del_dec_c, /* ARMv4 */ | ||
82 | silk_NSQ_del_dec_c, /* EDSP */ | ||
83 | silk_NSQ_del_dec_c, /* Media */ | ||
84 | silk_NSQ_del_dec_neon, /* Neon */ | ||
85 | }; | ||
86 | |||
87 | /*There is no table for silk_noise_shape_quantizer_short_prediction because the | ||
88 | NEON version takes different parameters than the C version. | ||
89 | Instead RTCD is done via if statements at the call sites. | ||
90 | See NSQ_neon.h for details.*/ | ||
91 | |||
92 | opus_int32 | ||
93 | (*const SILK_NSQ_NOISE_SHAPE_FEEDBACK_LOOP_IMPL[OPUS_ARCHMASK+1])( | ||
94 | const opus_int32 *data0, opus_int32 *data1, const opus_int16 *coef, | ||
95 | opus_int order) = { | ||
96 | silk_NSQ_noise_shape_feedback_loop_c, /* ARMv4 */ | ||
97 | silk_NSQ_noise_shape_feedback_loop_c, /* EDSP */ | ||
98 | silk_NSQ_noise_shape_feedback_loop_c, /* Media */ | ||
99 | silk_NSQ_noise_shape_feedback_loop_neon, /* NEON */ | ||
100 | }; | ||
101 | |||
102 | # endif | ||
103 | |||
104 | # if defined(FIXED_POINT) && \ | ||
105 | defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
106 | |||
107 | void (*const SILK_WARPED_AUTOCORRELATION_FIX_IMPL[OPUS_ARCHMASK + 1])( | ||
108 | opus_int32 *corr, /* O Result [order + 1] */ | ||
109 | opus_int *scale, /* O Scaling of the correlation vector */ | ||
110 | const opus_int16 *input, /* I Input data to correlate */ | ||
111 | const opus_int warping_Q16, /* I Warping coefficient */ | ||
112 | const opus_int length, /* I Length of input */ | ||
113 | const opus_int order /* I Correlation order (even) */ | ||
114 | ) = { | ||
115 | silk_warped_autocorrelation_FIX_c, /* ARMv4 */ | ||
116 | silk_warped_autocorrelation_FIX_c, /* EDSP */ | ||
117 | silk_warped_autocorrelation_FIX_c, /* Media */ | ||
118 | silk_warped_autocorrelation_FIX_neon, /* Neon */ | ||
119 | }; | ||
120 | |||
121 | # endif | ||
122 | |||
123 | #endif /* OPUS_HAVE_RTCD */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_arm.h b/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_arm.h new file mode 100644 index 0000000000..66ea9f43dd --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_arm.h | |||
@@ -0,0 +1,68 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifndef SILK_BIQUAD_ALT_ARM_H | ||
29 | # define SILK_BIQUAD_ALT_ARM_H | ||
30 | |||
31 | # include "celt/arm/armcpu.h" | ||
32 | |||
33 | # if defined(OPUS_ARM_MAY_HAVE_NEON_INTR) | ||
34 | void silk_biquad_alt_stride2_neon( | ||
35 | const opus_int16 *in, /* I input signal */ | ||
36 | const opus_int32 *B_Q28, /* I MA coefficients [3] */ | ||
37 | const opus_int32 *A_Q28, /* I AR coefficients [2] */ | ||
38 | opus_int32 *S, /* I/O State vector [4] */ | ||
39 | opus_int16 *out, /* O output signal */ | ||
40 | const opus_int32 len /* I signal length (must be even) */ | ||
41 | ); | ||
42 | |||
43 | # if !defined(OPUS_HAVE_RTCD) && defined(OPUS_ARM_PRESUME_NEON) | ||
44 | # define OVERRIDE_silk_biquad_alt_stride2 (1) | ||
45 | # define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((void)(arch), PRESUME_NEON(silk_biquad_alt_stride2)(in, B_Q28, A_Q28, S, out, len)) | ||
46 | # endif | ||
47 | # endif | ||
48 | |||
49 | # if !defined(OVERRIDE_silk_biquad_alt_stride2) | ||
50 | /*Is run-time CPU detection enabled on this platform?*/ | ||
51 | # if defined(OPUS_HAVE_RTCD) && (defined(OPUS_ARM_MAY_HAVE_NEON_INTR) && !defined(OPUS_ARM_PRESUME_NEON_INTR)) | ||
52 | extern void (*const SILK_BIQUAD_ALT_STRIDE2_IMPL[OPUS_ARCHMASK+1])( | ||
53 | const opus_int16 *in, /* I input signal */ | ||
54 | const opus_int32 *B_Q28, /* I MA coefficients [3] */ | ||
55 | const opus_int32 *A_Q28, /* I AR coefficients [2] */ | ||
56 | opus_int32 *S, /* I/O State vector [4] */ | ||
57 | opus_int16 *out, /* O output signal */ | ||
58 | const opus_int32 len /* I signal length (must be even) */ | ||
59 | ); | ||
60 | # define OVERRIDE_silk_biquad_alt_stride2 (1) | ||
61 | # define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((*SILK_BIQUAD_ALT_STRIDE2_IMPL[(arch)&OPUS_ARCHMASK])(in, B_Q28, A_Q28, S, out, len)) | ||
62 | # elif defined(OPUS_ARM_PRESUME_NEON_INTR) | ||
63 | # define OVERRIDE_silk_biquad_alt_stride2 (1) | ||
64 | # define silk_biquad_alt_stride2(in, B_Q28, A_Q28, S, out, len, arch) ((void)(arch), silk_biquad_alt_stride2_neon(in, B_Q28, A_Q28, S, out, len)) | ||
65 | # endif | ||
66 | # endif | ||
67 | |||
68 | #endif /* end SILK_BIQUAD_ALT_ARM_H */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_neon_intr.c b/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_neon_intr.c new file mode 100644 index 0000000000..9715733185 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/biquad_alt_neon_intr.c | |||
@@ -0,0 +1,156 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (c) 2017 Google Inc. | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifdef HAVE_CONFIG_H | ||
29 | #include "config.h" | ||
30 | #endif | ||
31 | |||
32 | #include <arm_neon.h> | ||
33 | #ifdef OPUS_CHECK_ASM | ||
34 | # include <string.h> | ||
35 | # include "stack_alloc.h" | ||
36 | #endif | ||
37 | #include "SigProc_FIX.h" | ||
38 | |||
39 | static inline void silk_biquad_alt_stride2_kernel( const int32x4_t A_L_s32x4, const int32x4_t A_U_s32x4, const int32x4_t B_Q28_s32x4, const int32x2_t t_s32x2, const int32x4_t in_s32x4, int32x4_t *S_s32x4, int32x2_t *out32_Q14_s32x2 ) | ||
40 | { | ||
41 | int32x4_t t_s32x4, out32_Q14_s32x4; | ||
42 | |||
43 | *out32_Q14_s32x2 = vadd_s32( vget_low_s32( *S_s32x4 ), t_s32x2 ); /* silk_SMLAWB( S{0,1}, B_Q28[ 0 ], in{0,1} ) */ | ||
44 | *S_s32x4 = vcombine_s32( vget_high_s32( *S_s32x4 ), vdup_n_s32( 0 ) ); /* S{0,1} = S{2,3}; S{2,3} = 0; */ | ||
45 | *out32_Q14_s32x2 = vshl_n_s32( *out32_Q14_s32x2, 2 ); /* out32_Q14_{0,1} = silk_LSHIFT( silk_SMLAWB( S{0,1}, B_Q28[ 0 ], in{0,1} ), 2 ); */ | ||
46 | out32_Q14_s32x4 = vcombine_s32( *out32_Q14_s32x2, *out32_Q14_s32x2 ); /* out32_Q14_{0,1,0,1} */ | ||
47 | t_s32x4 = vqdmulhq_s32( out32_Q14_s32x4, A_L_s32x4 ); /* silk_SMULWB( out32_Q14_{0,1,0,1}, A{0,0,1,1}_L_Q28 ) */ | ||
48 | *S_s32x4 = vrsraq_n_s32( *S_s32x4, t_s32x4, 14 ); /* S{0,1} = S{2,3} + silk_RSHIFT_ROUND(); S{2,3} = silk_RSHIFT_ROUND(); */ | ||
49 | t_s32x4 = vqdmulhq_s32( out32_Q14_s32x4, A_U_s32x4 ); /* silk_SMULWB( out32_Q14_{0,1,0,1}, A{0,0,1,1}_U_Q28 ) */ | ||
50 | *S_s32x4 = vaddq_s32( *S_s32x4, t_s32x4 ); /* S0 = silk_SMLAWB( S{0,1,2,3}, out32_Q14_{0,1,0,1}, A{0,0,1,1}_U_Q28 ); */ | ||
51 | t_s32x4 = vqdmulhq_s32( in_s32x4, B_Q28_s32x4 ); /* silk_SMULWB( B_Q28[ {1,1,2,2} ], in{0,1,0,1} ) */ | ||
52 | *S_s32x4 = vaddq_s32( *S_s32x4, t_s32x4 ); /* S0 = silk_SMLAWB( S0, B_Q28[ {1,1,2,2} ], in{0,1,0,1} ); */ | ||
53 | } | ||
54 | |||
55 | void silk_biquad_alt_stride2_neon( | ||
56 | const opus_int16 *in, /* I input signal */ | ||
57 | const opus_int32 *B_Q28, /* I MA coefficients [3] */ | ||
58 | const opus_int32 *A_Q28, /* I AR coefficients [2] */ | ||
59 | opus_int32 *S, /* I/O State vector [4] */ | ||
60 | opus_int16 *out, /* O output signal */ | ||
61 | const opus_int32 len /* I signal length (must be even) */ | ||
62 | ) | ||
63 | { | ||
64 | /* DIRECT FORM II TRANSPOSED (uses 2 element state vector) */ | ||
65 | opus_int k = 0; | ||
66 | const int32x2_t offset_s32x2 = vdup_n_s32( (1<<14) - 1 ); | ||
67 | const int32x4_t offset_s32x4 = vcombine_s32( offset_s32x2, offset_s32x2 ); | ||
68 | int16x4_t in_s16x4 = vdup_n_s16( 0 ); | ||
69 | int16x4_t out_s16x4; | ||
70 | int32x2_t A_Q28_s32x2, A_L_s32x2, A_U_s32x2, B_Q28_s32x2, t_s32x2; | ||
71 | int32x4_t A_L_s32x4, A_U_s32x4, B_Q28_s32x4, S_s32x4, out32_Q14_s32x4; | ||
72 | int32x2x2_t t0_s32x2x2, t1_s32x2x2, t2_s32x2x2, S_s32x2x2; | ||
73 | |||
74 | #ifdef OPUS_CHECK_ASM | ||
75 | opus_int32 S_c[ 4 ]; | ||
76 | VARDECL( opus_int16, out_c ); | ||
77 | SAVE_STACK; | ||
78 | ALLOC( out_c, 2 * len, opus_int16 ); | ||
79 | |||
80 | silk_memcpy( &S_c, S, sizeof( S_c ) ); | ||
81 | silk_biquad_alt_stride2_c( in, B_Q28, A_Q28, S_c, out_c, len ); | ||
82 | #endif | ||
83 | |||
84 | /* Negate A_Q28 values and split in two parts */ | ||
85 | A_Q28_s32x2 = vld1_s32( A_Q28 ); | ||
86 | A_Q28_s32x2 = vneg_s32( A_Q28_s32x2 ); | ||
87 | A_L_s32x2 = vshl_n_s32( A_Q28_s32x2, 18 ); /* ( -A_Q28[] & 0x00003FFF ) << 18 */ | ||
88 | A_L_s32x2 = vreinterpret_s32_u32( vshr_n_u32( vreinterpret_u32_s32( A_L_s32x2 ), 3 ) ); /* ( -A_Q28[] & 0x00003FFF ) << 15 */ | ||
89 | A_U_s32x2 = vshr_n_s32( A_Q28_s32x2, 14 ); /* silk_RSHIFT( -A_Q28[], 14 ) */ | ||
90 | A_U_s32x2 = vshl_n_s32( A_U_s32x2, 16 ); /* silk_RSHIFT( -A_Q28[], 14 ) << 16 (Clip two leading bits to conform to C function.) */ | ||
91 | A_U_s32x2 = vshr_n_s32( A_U_s32x2, 1 ); /* silk_RSHIFT( -A_Q28[], 14 ) << 15 */ | ||
92 | |||
93 | B_Q28_s32x2 = vld1_s32( B_Q28 ); | ||
94 | t_s32x2 = vld1_s32( B_Q28 + 1 ); | ||
95 | t0_s32x2x2 = vzip_s32( A_L_s32x2, A_L_s32x2 ); | ||
96 | t1_s32x2x2 = vzip_s32( A_U_s32x2, A_U_s32x2 ); | ||
97 | t2_s32x2x2 = vzip_s32( t_s32x2, t_s32x2 ); | ||
98 | A_L_s32x4 = vcombine_s32( t0_s32x2x2.val[ 0 ], t0_s32x2x2.val[ 1 ] ); /* A{0,0,1,1}_L_Q28 */ | ||
99 | A_U_s32x4 = vcombine_s32( t1_s32x2x2.val[ 0 ], t1_s32x2x2.val[ 1 ] ); /* A{0,0,1,1}_U_Q28 */ | ||
100 | B_Q28_s32x4 = vcombine_s32( t2_s32x2x2.val[ 0 ], t2_s32x2x2.val[ 1 ] ); /* B_Q28[ {1,1,2,2} ] */ | ||
101 | S_s32x4 = vld1q_s32( S ); /* S0 = S[ 0 ]; S3 = S[ 3 ]; */ | ||
102 | S_s32x2x2 = vtrn_s32( vget_low_s32( S_s32x4 ), vget_high_s32( S_s32x4 ) ); /* S2 = S[ 1 ]; S1 = S[ 2 ]; */ | ||
103 | S_s32x4 = vcombine_s32( S_s32x2x2.val[ 0 ], S_s32x2x2.val[ 1 ] ); | ||
104 | |||
105 | for( ; k < len - 1; k += 2 ) { | ||
106 | int32x4_t in_s32x4[ 2 ], t_s32x4; | ||
107 | int32x2_t out32_Q14_s32x2[ 2 ]; | ||
108 | |||
109 | /* S[ 2 * i + 0 ], S[ 2 * i + 1 ], S[ 2 * i + 2 ], S[ 2 * i + 3 ]: Q12 */ | ||
110 | in_s16x4 = vld1_s16( &in[ 2 * k ] ); /* in{0,1,2,3} = in[ 2 * k + {0,1,2,3} ]; */ | ||
111 | in_s32x4[ 0 ] = vshll_n_s16( in_s16x4, 15 ); /* in{0,1,2,3} << 15 */ | ||
112 | t_s32x4 = vqdmulhq_lane_s32( in_s32x4[ 0 ], B_Q28_s32x2, 0 ); /* silk_SMULWB( B_Q28[ 0 ], in{0,1,2,3} ) */ | ||
113 | in_s32x4[ 1 ] = vcombine_s32( vget_high_s32( in_s32x4[ 0 ] ), vget_high_s32( in_s32x4[ 0 ] ) ); /* in{2,3,2,3} << 15 */ | ||
114 | in_s32x4[ 0 ] = vcombine_s32( vget_low_s32 ( in_s32x4[ 0 ] ), vget_low_s32 ( in_s32x4[ 0 ] ) ); /* in{0,1,0,1} << 15 */ | ||
115 | silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, vget_low_s32 ( t_s32x4 ), in_s32x4[ 0 ], &S_s32x4, &out32_Q14_s32x2[ 0 ] ); | ||
116 | silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, vget_high_s32( t_s32x4 ), in_s32x4[ 1 ], &S_s32x4, &out32_Q14_s32x2[ 1 ] ); | ||
117 | |||
118 | /* Scale back to Q0 and saturate */ | ||
119 | out32_Q14_s32x4 = vcombine_s32( out32_Q14_s32x2[ 0 ], out32_Q14_s32x2[ 1 ] ); /* out32_Q14_{0,1,2,3} */ | ||
120 | out32_Q14_s32x4 = vaddq_s32( out32_Q14_s32x4, offset_s32x4 ); /* out32_Q14_{0,1,2,3} + (1<<14) - 1 */ | ||
121 | out_s16x4 = vqshrn_n_s32( out32_Q14_s32x4, 14 ); /* (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,2,3} + (1<<14) - 1, 14 ) ) */ | ||
122 | vst1_s16( &out[ 2 * k ], out_s16x4 ); /* out[ 2 * k + {0,1,2,3} ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,2,3} + (1<<14) - 1, 14 ) ); */ | ||
123 | } | ||
124 | |||
125 | /* Process leftover. */ | ||
126 | if( k < len ) { | ||
127 | int32x4_t in_s32x4; | ||
128 | int32x2_t out32_Q14_s32x2; | ||
129 | |||
130 | /* S[ 2 * i + 0 ], S[ 2 * i + 1 ]: Q12 */ | ||
131 | in_s16x4 = vld1_lane_s16( &in[ 2 * k + 0 ], in_s16x4, 0 ); /* in{0,1} = in[ 2 * k + {0,1} ]; */ | ||
132 | in_s16x4 = vld1_lane_s16( &in[ 2 * k + 1 ], in_s16x4, 1 ); /* in{0,1} = in[ 2 * k + {0,1} ]; */ | ||
133 | in_s32x4 = vshll_n_s16( in_s16x4, 15 ); /* in{0,1} << 15 */ | ||
134 | t_s32x2 = vqdmulh_lane_s32( vget_low_s32( in_s32x4 ), B_Q28_s32x2, 0 ); /* silk_SMULWB( B_Q28[ 0 ], in{0,1} ) */ | ||
135 | in_s32x4 = vcombine_s32( vget_low_s32( in_s32x4 ), vget_low_s32( in_s32x4 ) ); /* in{0,1,0,1} << 15 */ | ||
136 | silk_biquad_alt_stride2_kernel( A_L_s32x4, A_U_s32x4, B_Q28_s32x4, t_s32x2, in_s32x4, &S_s32x4, &out32_Q14_s32x2 ); | ||
137 | |||
138 | /* Scale back to Q0 and saturate */ | ||
139 | out32_Q14_s32x2 = vadd_s32( out32_Q14_s32x2, offset_s32x2 ); /* out32_Q14_{0,1} + (1<<14) - 1 */ | ||
140 | out32_Q14_s32x4 = vcombine_s32( out32_Q14_s32x2, out32_Q14_s32x2 ); /* out32_Q14_{0,1,0,1} + (1<<14) - 1 */ | ||
141 | out_s16x4 = vqshrn_n_s32( out32_Q14_s32x4, 14 ); /* (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_{0,1,0,1} + (1<<14) - 1, 14 ) ) */ | ||
142 | vst1_lane_s16( &out[ 2 * k + 0 ], out_s16x4, 0 ); /* out[ 2 * k + 0 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_0 + (1<<14) - 1, 14 ) ); */ | ||
143 | vst1_lane_s16( &out[ 2 * k + 1 ], out_s16x4, 1 ); /* out[ 2 * k + 1 ] = (opus_int16)silk_SAT16( silk_RSHIFT( out32_Q14_1 + (1<<14) - 1, 14 ) ); */ | ||
144 | } | ||
145 | |||
146 | vst1q_lane_s32( &S[ 0 ], S_s32x4, 0 ); /* S[ 0 ] = S0; */ | ||
147 | vst1q_lane_s32( &S[ 1 ], S_s32x4, 2 ); /* S[ 1 ] = S2; */ | ||
148 | vst1q_lane_s32( &S[ 2 ], S_s32x4, 1 ); /* S[ 2 ] = S1; */ | ||
149 | vst1q_lane_s32( &S[ 3 ], S_s32x4, 3 ); /* S[ 3 ] = S3; */ | ||
150 | |||
151 | #ifdef OPUS_CHECK_ASM | ||
152 | silk_assert( !memcmp( S_c, S, sizeof( S_c ) ) ); | ||
153 | silk_assert( !memcmp( out_c, out, 2 * len * sizeof( opus_int16 ) ) ); | ||
154 | RESTORE_STACK; | ||
155 | #endif | ||
156 | } | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/macros_arm64.h b/lib/rbcodec/codecs/libopus/silk/arm/macros_arm64.h new file mode 100644 index 0000000000..ed030413c5 --- /dev/null +++ b/lib/rbcodec/codecs/libopus/silk/arm/macros_arm64.h | |||
@@ -0,0 +1,39 @@ | |||
1 | /*********************************************************************** | ||
2 | Copyright (C) 2015 Vidyo | ||
3 | Redistribution and use in source and binary forms, with or without | ||
4 | modification, are permitted provided that the following conditions | ||
5 | are met: | ||
6 | - Redistributions of source code must retain the above copyright notice, | ||
7 | this list of conditions and the following disclaimer. | ||
8 | - Redistributions in binary form must reproduce the above copyright | ||
9 | notice, this list of conditions and the following disclaimer in the | ||
10 | documentation and/or other materials provided with the distribution. | ||
11 | - Neither the name of Internet Society, IETF or IETF Trust, nor the | ||
12 | names of specific contributors, may be used to endorse or promote | ||
13 | products derived from this software without specific prior written | ||
14 | permission. | ||
15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" | ||
16 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
17 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
18 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
19 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
20 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
21 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
22 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
23 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
24 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
25 | POSSIBILITY OF SUCH DAMAGE. | ||
26 | ***********************************************************************/ | ||
27 | |||
28 | #ifndef SILK_MACROS_ARM64_H | ||
29 | #define SILK_MACROS_ARM64_H | ||
30 | |||
31 | #include <arm_neon.h> | ||
32 | |||
33 | #undef silk_ADD_SAT32 | ||
34 | #define silk_ADD_SAT32(a, b) (vqadds_s32((a), (b))) | ||
35 | |||
36 | #undef silk_SUB_SAT32 | ||
37 | #define silk_SUB_SAT32(a, b) (vqsubs_s32((a), (b))) | ||
38 | |||
39 | #endif /* SILK_MACROS_ARM64_H */ | ||
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/macros_armv4.h b/lib/rbcodec/codecs/libopus/silk/arm/macros_armv4.h index 3f30e97288..877eb18dd5 100644 --- a/lib/rbcodec/codecs/libopus/silk/arm/macros_armv4.h +++ b/lib/rbcodec/codecs/libopus/silk/arm/macros_armv4.h | |||
@@ -28,6 +28,11 @@ POSSIBILITY OF SUCH DAMAGE. | |||
28 | #ifndef SILK_MACROS_ARMv4_H | 28 | #ifndef SILK_MACROS_ARMv4_H |
29 | #define SILK_MACROS_ARMv4_H | 29 | #define SILK_MACROS_ARMv4_H |
30 | 30 | ||
31 | /* This macro only avoids the undefined behaviour from a left shift of | ||
32 | a negative value. It should only be used in macros that can't include | ||
33 | SigProc_FIX.h. In other cases, use silk_LSHIFT32(). */ | ||
34 | #define SAFE_SHL(a,b) ((opus_int32)((opus_uint32)(a) << (b))) | ||
35 | |||
31 | /* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ | 36 | /* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ |
32 | #undef silk_SMULWB | 37 | #undef silk_SMULWB |
33 | static OPUS_INLINE opus_int32 silk_SMULWB_armv4(opus_int32 a, opus_int16 b) | 38 | static OPUS_INLINE opus_int32 silk_SMULWB_armv4(opus_int32 a, opus_int16 b) |
@@ -38,7 +43,7 @@ static OPUS_INLINE opus_int32 silk_SMULWB_armv4(opus_int32 a, opus_int16 b) | |||
38 | "#silk_SMULWB\n\t" | 43 | "#silk_SMULWB\n\t" |
39 | "smull %0, %1, %2, %3\n\t" | 44 | "smull %0, %1, %2, %3\n\t" |
40 | : "=&r"(rd_lo), "=&r"(rd_hi) | 45 | : "=&r"(rd_lo), "=&r"(rd_hi) |
41 | : "%r"(a), "r"(b<<16) | 46 | : "%r"(a), "r"(SAFE_SHL(b,16)) |
42 | ); | 47 | ); |
43 | return rd_hi; | 48 | return rd_hi; |
44 | } | 49 | } |
@@ -80,7 +85,7 @@ static OPUS_INLINE opus_int32 silk_SMULWW_armv4(opus_int32 a, opus_int32 b) | |||
80 | : "=&r"(rd_lo), "=&r"(rd_hi) | 85 | : "=&r"(rd_lo), "=&r"(rd_hi) |
81 | : "%r"(a), "r"(b) | 86 | : "%r"(a), "r"(b) |
82 | ); | 87 | ); |
83 | return (rd_hi<<16)+(rd_lo>>16); | 88 | return SAFE_SHL(rd_hi,16)+(rd_lo>>16); |
84 | } | 89 | } |
85 | #define silk_SMULWW(a, b) (silk_SMULWW_armv4(a, b)) | 90 | #define silk_SMULWW(a, b) (silk_SMULWW_armv4(a, b)) |
86 | 91 | ||
@@ -96,8 +101,10 @@ static OPUS_INLINE opus_int32 silk_SMLAWW_armv4(opus_int32 a, opus_int32 b, | |||
96 | : "=&r"(rd_lo), "=&r"(rd_hi) | 101 | : "=&r"(rd_lo), "=&r"(rd_hi) |
97 | : "%r"(b), "r"(c) | 102 | : "%r"(b), "r"(c) |
98 | ); | 103 | ); |
99 | return a+(rd_hi<<16)+(rd_lo>>16); | 104 | return a+SAFE_SHL(rd_hi,16)+(rd_lo>>16); |
100 | } | 105 | } |
101 | #define silk_SMLAWW(a, b, c) (silk_SMLAWW_armv4(a, b, c)) | 106 | #define silk_SMLAWW(a, b, c) (silk_SMLAWW_armv4(a, b, c)) |
102 | 107 | ||
108 | #undef SAFE_SHL | ||
109 | |||
103 | #endif /* SILK_MACROS_ARMv4_H */ | 110 | #endif /* SILK_MACROS_ARMv4_H */ |
diff --git a/lib/rbcodec/codecs/libopus/silk/arm/macros_armv5e.h b/lib/rbcodec/codecs/libopus/silk/arm/macros_armv5e.h index aad4117e46..b14ec65ddb 100644 --- a/lib/rbcodec/codecs/libopus/silk/arm/macros_armv5e.h +++ b/lib/rbcodec/codecs/libopus/silk/arm/macros_armv5e.h | |||
@@ -29,6 +29,11 @@ POSSIBILITY OF SUCH DAMAGE. | |||
29 | #ifndef SILK_MACROS_ARMv5E_H | 29 | #ifndef SILK_MACROS_ARMv5E_H |
30 | #define SILK_MACROS_ARMv5E_H | 30 | #define SILK_MACROS_ARMv5E_H |
31 | 31 | ||
32 | /* This macro only avoids the undefined behaviour from a left shift of | ||
33 | a negative value. It should only be used in macros that can't include | ||
34 | SigProc_FIX.h. In other cases, use silk_LSHIFT32(). */ | ||
35 | #define SAFE_SHL(a,b) ((opus_int32)((opus_uint32)(a) << (b))) | ||
36 | |||
32 | /* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ | 37 | /* (a32 * (opus_int32)((opus_int16)(b32))) >> 16 output have to be 32bit int */ |
33 | #undef silk_SMULWB | 38 | #undef silk_SMULWB |
34 | static OPUS_INLINE opus_int32 silk_SMULWB_armv5e(opus_int32 a, opus_int16 b) | 39 | static OPUS_INLINE opus_int32 silk_SMULWB_armv5e(opus_int32 a, opus_int16 b) |
@@ -190,7 +195,7 @@ static OPUS_INLINE opus_int32 silk_CLZ16_armv5(opus_int16 in16) | |||
190 | "#silk_CLZ16\n\t" | 195 | "#silk_CLZ16\n\t" |
191 | "clz %0, %1;\n" | 196 | "clz %0, %1;\n" |
192 | : "=r"(res) | 197 | : "=r"(res) |
193 | : "r"(in16<<16|0x8000) | 198 | : "r"(SAFE_SHL(in16,16)|0x8000) |
194 | ); | 199 | ); |
195 | return res; | 200 | return res; |
196 | } | 201 | } |
@@ -210,4 +215,6 @@ static OPUS_INLINE opus_int32 silk_CLZ32_armv5(opus_int32 in32) | |||
210 | } | 215 | } |
211 | #define silk_CLZ32(in32) (silk_CLZ32_armv5(in32)) | 216 | #define silk_CLZ32(in32) (silk_CLZ32_armv5(in32)) |
212 | 217 | ||
218 | #undef SAFE_SHL | ||
219 | |||
213 | #endif /* SILK_MACROS_ARMv5E_H */ | 220 | #endif /* SILK_MACROS_ARMv5E_H */ |