opus - Mar 2015 - [RFC PATCH v1] armv7(float): Optimize decode usecase using NE10 library

Hello All,

I extended the libNE10 optimizations for float towards
mdct_backwards/opus_ifft.

I am able to get about 14.26% improvement for Decode use
case now on my Beaglebone Black. Please see [1] for measurements.

Questions
1. Since this patch needs to go in after Encode [2] patch)
   should I submit this as patch series?
2. Since Jonathan Lennox posted intrinsics cleanup [3]
   patch, should I rebase this patch on Jonathan's patch
3. I put on "what would Timothy say" hat while reviewing
   my own patch and hope I don't have any silly indentation
   issues in this patch. Hope there is a checkpatch.pl kind
   of script that I can automatically check for these
   mundane mistakes.

Either way, please let me know soon, I am starting to work on
fixed point optimizations and would like to reduce rebases I
may have to do if these patches don't go in soon.

[1]:
https://docs.google.com/a/linaro.org/document/d/1avz20b3DOnD3IwxiKTmUfyUK89hUwL9K2PYMh7dlkNg/edit#heading=h.d7ydpmyrwvsv
[2]: http://lists.xiph.org/pipermail/opus/2015-March/002904.html
[3]: http://lists.xiph.org/pipermail/opus/2015-March/002899.html

Viswanath Puttagunta (1):
  armv7(float): Optimize decode usecase using NE10 library

 celt/arm/arm_celt_map.c     |  22 ++++++++++
 celt/arm/celt_ne10_fft.c    |  26 +++++++++++
 celt/arm/celt_ne10_mdct.c   | 102 ++++++++++++++++++++++++++++++++++++++++++++
 celt/arm/fft_arm.h          |   8 ++++
 celt/arm/mdct_arm.h         |   7 +++
 celt/celt_decoder.c         |  18 ++++----
 celt/celt_encoder.c         |   3 +-
 celt/kiss_fft.c             |   4 +-
 celt/kiss_fft.h             |  13 +++++-
 celt/mdct.c                 |   5 ++-
 celt/mdct.h                 |  22 ++++++++--
 celt/tests/test_unit_dft.c  |   2 +-
 celt/tests/test_unit_mdct.c |   2 +-
 13 files changed, 214 insertions(+), 20 deletions(-)

-- 
1.9.1

Optimize opus decode (float only) use case using ARM NE10.
Mainly effects opus_ifft and ctl_mdct_backward and related
functions.

Work based on previous Encode optimization using ARM NE10
library.

TBD: Add commit id of upstream Encode NE10 optimization patch
so that users have reference of how to enable this optimization

Signed-off-by: Viswanath Puttagunta <viswanath.puttagunta at linaro.org>
---
 celt/arm/arm_celt_map.c     |  22 ++++++++++
 celt/arm/celt_ne10_fft.c    |  26 +++++++++++
 celt/arm/celt_ne10_mdct.c   | 102 ++++++++++++++++++++++++++++++++++++++++++++
 celt/arm/fft_arm.h          |   8 ++++
 celt/arm/mdct_arm.h         |   7 +++
 celt/celt_decoder.c         |  18 ++++----
 celt/celt_encoder.c         |   3 +-
 celt/kiss_fft.c             |   4 +-
 celt/kiss_fft.h             |  13 +++++-
 celt/mdct.c                 |   5 ++-
 celt/mdct.h                 |  22 ++++++++--
 celt/tests/test_unit_dft.c  |   2 +-
 celt/tests/test_unit_mdct.c |   2 +-
 13 files changed, 214 insertions(+), 20 deletions(-)

diff --git a/celt/arm/arm_celt_map.c b/celt/arm/arm_celt_map.c
index 3b49f90..f132fe1 100644
--- a/celt/arm/arm_celt_map.c
+++ b/celt/arm/arm_celt_map.c
@@ -79,6 +79,15 @@ void (*const OPUS_FFT[OPUS_ARCHMASK+1])(const kiss_fft_state
*cfg,
    opus_fft_float_neon           /* Neon with NE10 */
 };
 
+void (*const OPUS_IFFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg,
+                                         const kiss_fft_cpx *fin,
+                                         kiss_fft_cpx *fout) = {
+   opus_ifft_c,                   /* ARMv4 */
+   opus_ifft_c,                   /* EDSP */
+   opus_ifft_c,                   /* Media */
+   opus_ifft_float_neon           /* Neon with NE10 */
+};
+
 void (*const CLT_MDCT_FORWARD_IMPL[OPUS_ARCHMASK+1])(const mdct_lookup *l,
                                                      kiss_fft_scalar *in,
                                                      kiss_fft_scalar *
OPUS_RESTRICT out,
@@ -90,6 +99,19 @@ void (*const CLT_MDCT_FORWARD_IMPL[OPUS_ARCHMASK+1])(const
mdct_lookup *l,
    clt_mdct_forward_c,           /* Media */
    clt_mdct_forward_float_neon   /* Neon with NE10 */
 };
+
+void (*const CLT_MDCT_BACKWARD_IMPL[OPUS_ARCHMASK+1])(const mdct_lookup *l,
+                                                      kiss_fft_scalar *in,
+                                                      kiss_fft_scalar *
OPUS_RESTRICT out,
+                                                      const opus_val16 *window,
+                                                      int overlap, int shift,
+                                                      int stride, int arch) = {
+   clt_mdct_backward_c,           /* ARMv4 */
+   clt_mdct_backward_c,           /* EDSP */
+   clt_mdct_backward_c,           /* Media */
+   clt_mdct_backward_float_neon   /* Neon with NE10 */
+};
+
 #endif /* HAVE_ARM_NE10 */
 #  endif /* OPUS_ARM_NEON_INTR */
 # endif /* FIXED_POINT */
diff --git a/celt/arm/celt_ne10_fft.c b/celt/arm/celt_ne10_fft.c
index b592f19..d354502 100644
--- a/celt/arm/celt_ne10_fft.c
+++ b/celt/arm/celt_ne10_fft.c
@@ -118,3 +118,29 @@ void opus_fft_float_neon(const kiss_fft_state *st,
    }
    RESTORE_STACK;
 }
+
+void opus_ifft_float_neon(const kiss_fft_state *st,
+                          const kiss_fft_cpx *fin,
+                          kiss_fft_cpx *fout)
+{
+   ne10_fft_state_float32_t state;
+   ne10_fft_cfg_float32_t cfg = &state;
+   VARDECL(ne10_fft_cpx_float32_t, buffer);
+   SAVE_STACK;
+   ALLOC(buffer, st->nfft, ne10_fft_cpx_float32_t);
+
+   if (!st->arch_fft->is_supported) {
+      /* This nfft length (scaled fft) not supported in NE10 */
+      opus_ifft_c(st, fin, fout);
+   }
+   else {
+      memcpy((void *)cfg, st->arch_fft->priv,
sizeof(ne10_fft_state_float32_t));
+      state.buffer = (ne10_fft_cpx_float32_t *)&buffer[0];
+      state.is_backward_scaled = 0;
+
+      ne10_fft_c2c_1d_float32_neon((ne10_fft_cpx_float32_t *)fout,
+                                   (ne10_fft_cpx_float32_t *)fin,
+                                   cfg, 1);
+   }
+   RESTORE_STACK;
+}
diff --git a/celt/arm/celt_ne10_mdct.c b/celt/arm/celt_ne10_mdct.c
index cf175cb..0979cbe 100644
--- a/celt/arm/celt_ne10_mdct.c
+++ b/celt/arm/celt_ne10_mdct.c
@@ -156,3 +156,105 @@ void clt_mdct_forward_float_neon(const mdct_lookup *l,
    }
    RESTORE_STACK;
 }
+
+void clt_mdct_backward_float_neon(const mdct_lookup *l,
+                                  kiss_fft_scalar *in,
+                                  kiss_fft_scalar * OPUS_RESTRICT out,
+                                  const opus_val16 * OPUS_RESTRICT window,
+                                  int overlap, int shift, int stride, int arch)
+{
+   int i;
+   int N, N2, N4;
+   VARDECL(kiss_fft_scalar, f);
+   const kiss_twiddle_scalar *trig;
+   const kiss_fft_state *st = l->kfft[shift];
+
+   N = l->n;
+   trig = l->trig;
+   for (i=0;i<shift;i++)
+   {
+      N >>= 1;
+      trig += N;
+   }
+   N2 = N>>1;
+   N4 = N>>2;
+
+   ALLOC(f, N2, kiss_fft_scalar);
+
+   /* Pre-rotate */
+   {
+      /* Temp pointers to make it really clear to the compiler what we're
doing */
+      const kiss_fft_scalar * OPUS_RESTRICT xp1 = in;
+      const kiss_fft_scalar * OPUS_RESTRICT xp2 = in+stride*(N2-1);
+      kiss_fft_scalar * OPUS_RESTRICT yp = f;
+      const kiss_twiddle_scalar * OPUS_RESTRICT t = &trig[0];
+      for(i=0;i<N4;i++)
+      {
+         kiss_fft_scalar yr, yi;
+         yr = S_MUL(*xp2, t[i]) + S_MUL(*xp1, t[N4+i]);
+         yi = S_MUL(*xp1, t[i]) - S_MUL(*xp2, t[N4+i]);
+         yp[2*i] = yr;
+         yp[2*i+1] = yi;
+         xp1+=2*stride;
+         xp2-=2*stride;
+      }
+   }
+
+   opus_ifft(st, (kiss_fft_cpx *)f, (kiss_fft_cpx*)(out+(overlap>>1)),
arch);
+
+   /* Post-rotate and de-shuffle from both ends of the buffer at once to make
+      it in-place. */
+   {
+      kiss_fft_scalar * yp0 = out+(overlap>>1);
+      kiss_fft_scalar * yp1 = out+(overlap>>1)+N2-2;
+      const kiss_twiddle_scalar *t = &trig[0];
+      /* Loop to (N4+1)>>1 to handle odd N4. When N4 is odd, the
+         middle pair will be computed twice. */
+      for(i=0;i<(N4+1)>>1;i++)
+      {
+         kiss_fft_scalar re, im, yr, yi;
+         kiss_twiddle_scalar t0, t1;
+         re = yp0[0];
+         im = yp0[1];
+         t0 = t[i];
+         t1 = t[N4+i];
+         /* We'd scale up by 2 here, but instead it's done when mixing
the windows */
+         yr = S_MUL(re,t0) + S_MUL(im,t1);
+         yi = S_MUL(re,t1) - S_MUL(im,t0);
+         re = yp1[0];
+         im = yp1[1];
+         yp0[0] = yr;
+         yp1[1] = yi;
+
+         t0 = t[(N4-i-1)];
+         t1 = t[(N2-i-1)];
+         /* We'd scale up by 2 here, but instead it's done when mixing
the windows */
+         yr = S_MUL(re,t0) + S_MUL(im,t1);
+         yi = S_MUL(re,t1) - S_MUL(im,t0);
+         yp1[0] = yr;
+         yp0[1] = yi;
+         yp0 += 2;
+         yp1 -= 2;
+      }
+   }
+
+   /* Mirror on both sides for TDAC */
+   {
+      kiss_fft_scalar * OPUS_RESTRICT xp1 = out+overlap-1;
+      kiss_fft_scalar * OPUS_RESTRICT yp1 = out;
+      const opus_val16 * OPUS_RESTRICT wp1 = window;
+      const opus_val16 * OPUS_RESTRICT wp2 = window+overlap-1;
+
+      for(i = 0; i < overlap/2; i++)
+      {
+         kiss_fft_scalar x1, x2;
+         x1 = *xp1;
+         x2 = *yp1;
+         *yp1++ = MULT16_32_Q15(*wp2, x2) - MULT16_32_Q15(*wp1, x1);
+         *xp1-- = MULT16_32_Q15(*wp1, x2) + MULT16_32_Q15(*wp2, x1);
+         wp1++;
+         wp2--;
+      }
+   }
+   RESTORE_STACK;
+}
diff --git a/celt/arm/fft_arm.h b/celt/arm/fft_arm.h
index e7a30d6..e57b0aa 100644
--- a/celt/arm/fft_arm.h
+++ b/celt/arm/fft_arm.h
@@ -46,6 +46,11 @@ void opus_fft_free_arm_float_neon(kiss_fft_state *st);
 void opus_fft_float_neon(const kiss_fft_state *st,
                          const kiss_fft_cpx *fin,
                          kiss_fft_cpx *fout);
+
+void opus_ifft_float_neon(const kiss_fft_state *st,
+                         const kiss_fft_cpx *fin,
+                         kiss_fft_cpx *fout);
+
 #if !defined(OPUS_HAVE_RTCD)
 #define OVERRIDE_OPUS_FFT (1)
 
@@ -58,6 +63,9 @@ void opus_fft_float_neon(const kiss_fft_state *st,
 #define opus_fft(_st, _fin, _fout, arch) \
    ((void)(arch), opus_fft_float_neon(_st, _fin, _fout))
 
+#define opus_ifft(_st, _fin, _fout, arch) \
+   ((void)(arch), opus_ifft_float_neon(_st, _fin, _fout))
+
 #endif /* OPUS_HAVE_RTCD */
 
 #endif /* HAVE_ARM_NE10 */
diff --git a/celt/arm/mdct_arm.h b/celt/arm/mdct_arm.h
index 33f7bb6..80bf53a 100644
--- a/celt/arm/mdct_arm.h
+++ b/celt/arm/mdct_arm.h
@@ -43,10 +43,17 @@ void clt_mdct_forward_float_neon(const mdct_lookup *l,
kiss_fft_scalar *in,
                                  const opus_val16 *window, int overlap,
                                  int shift, int stride, int arch);
 
+void clt_mdct_backward_float_neon(const mdct_lookup *l, kiss_fft_scalar *in,
+                                  kiss_fft_scalar * OPUS_RESTRICT out,
+                                  const opus_val16 *window, int overlap,
+                                  int shift, int stride, int arch);
 #if !defined(OPUS_HAVE_RTCD)
 #define OVERRIDE_OPUS_MDCT (1)
 #define clt_mdct_forward(_l, _in, _out, _window, _int, _shift, _stride, _arch)
\
       clt_mdct_forward_float_neon((_l, _in, _out, _window, _int, _shift,
_stride, _arch)
+
+#define clt_mdct_backward(_l, _in, _out, _window, _int, _shift, _stride, _arch)
\
+      clt_mdct_backward_float_neon((_l, _in, _out, _window, _int, _shift,
_stride, _arch)
 #endif /* OPUS_HAVE_RTCD */
 #endif /* !defined(FIXED_POINT) && defined(HAVE_ARM_NE10) */
 
diff --git a/celt/celt_decoder.c b/celt/celt_decoder.c
index 4304a3e..304f334 100644
--- a/celt/celt_decoder.c
+++ b/celt/celt_decoder.c
@@ -278,8 +278,9 @@ void deemphasis(celt_sig *in[], opus_val16 *pcm, int N, int
C, int downsample, c
 static
 #endif
 void celt_synthesis(const CELTMode *mode, celt_norm *X, celt_sig * out_syn[],
-      opus_val16 *oldBandE, int start, int effEnd, int C, int CC, int
isTransient,
-      int LM, int downsample, int silence)
+                    opus_val16 *oldBandE, int start, int effEnd, int C, int CC,
+                    int isTransient, int LM, int downsample,
+                    int silence, int arch)
 {
    int c, i;
    int M;
@@ -319,9 +320,9 @@ void celt_synthesis(const CELTMode *mode, celt_norm *X,
celt_sig * out_syn[],
       freq2 = out_syn[1]+overlap/2;
       OPUS_COPY(freq2, freq, N);
       for (b=0;b<B;b++)
-         clt_mdct_backward(&mode->mdct, &freq2[b], out_syn[0]+NB*b,
mode->window, overlap, shift, B);
+         clt_mdct_backward(&mode->mdct, &freq2[b], out_syn[0]+NB*b,
mode->window, overlap, shift, B, arch);
       for (b=0;b<B;b++)
-         clt_mdct_backward(&mode->mdct, &freq[b], out_syn[1]+NB*b,
mode->window, overlap, shift, B);
+         clt_mdct_backward(&mode->mdct, &freq[b], out_syn[1]+NB*b,
mode->window, overlap, shift, B, arch);
    } else if (CC==1&&C==2)
    {
       /* Downmixing a stereo stream to mono */
@@ -335,14 +336,14 @@ void celt_synthesis(const CELTMode *mode, celt_norm *X,
celt_sig * out_syn[],
       for (i=0;i<N;i++)
          freq[i] = HALF32(ADD32(freq[i],freq2[i]));
       for (b=0;b<B;b++)
-         clt_mdct_backward(&mode->mdct, &freq[b], out_syn[0]+NB*b,
mode->window, overlap, shift, B);
+         clt_mdct_backward(&mode->mdct, &freq[b], out_syn[0]+NB*b,
mode->window, overlap, shift, B, arch);
    } else {
       /* Normal case (mono or stereo) */
       c=0; do {
          denormalise_bands(mode, X+c*N, freq, oldBandE+c*nbEBands, start,
effEnd, M,
                downsample, silence);
          for (b=0;b<B;b++)
-            clt_mdct_backward(&mode->mdct, &freq[b],
out_syn[c]+NB*b, mode->window, overlap, shift, B);
+            clt_mdct_backward(&mode->mdct, &freq[b],
out_syn[c]+NB*b, mode->window, overlap, shift, B, arch);
       } while (++c<CC);
    }
    RESTORE_STACK;
@@ -509,7 +510,7 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st,
int N, int LM)
                DECODE_BUFFER_SIZE-N+(overlap>>1));
       } while (++c<C);
 
-      celt_synthesis(mode, X, out_syn, plcLogE, start, effEnd, C, C, 0, LM,
st->downsample, 0);
+      celt_synthesis(mode, X, out_syn, plcLogE, start, effEnd, C, C, 0, LM,
st->downsample, 0, st->arch);
    } else {
       /* Pitch-based PLC */
       const opus_val16 *window;
@@ -1002,7 +1003,8 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st,
const unsigned char *dat
          oldBandE[i] = -QCONST16(28.f,DB_SHIFT);
    }
 
-   celt_synthesis(mode, X, out_syn, oldBandE, start, effEnd, C, CC,
isTransient, LM, st->downsample, silence);
+   celt_synthesis(mode, X, out_syn, oldBandE, start, effEnd,
+                  C, CC, isTransient, LM, st->downsample, silence,
st->arch);
 
    c=0; do {
       st->postfilter_period=IMAX(st->postfilter_period,
COMBFILTER_MINPERIOD);
diff --git a/celt/celt_encoder.c b/celt/celt_encoder.c
index 7a2c71b..5f48638 100644
--- a/celt/celt_encoder.c
+++ b/celt/celt_encoder.c
@@ -2072,7 +2072,8 @@ int celt_encode_with_ec(CELTEncoder * OPUS_RESTRICT st,
const opus_val16 * pcm,
          out_mem[c] = st->syn_mem[c]+2*MAX_PERIOD-N;
       } while (++c<CC);
 
-      celt_synthesis(mode, X, out_mem, oldBandE, start, effEnd, C, CC,
isTransient, LM, st->upsample, silence);
+      celt_synthesis(mode, X, out_mem, oldBandE, start, effEnd,
+                     C, CC, isTransient, LM, st->upsample, silence,
st->arch);
 
       c=0; do {
          st->prefilter_period=IMAX(st->prefilter_period,
COMBFILTER_MINPERIOD);
diff --git a/celt/kiss_fft.c b/celt/kiss_fft.c
index 38fd4fb..4ed37d2 100644
--- a/celt/kiss_fft.c
+++ b/celt/kiss_fft.c
@@ -589,8 +589,7 @@ void opus_fft_c(const kiss_fft_state *st,const kiss_fft_cpx
*fin,kiss_fft_cpx *f
 }
 
 
-#ifdef TEST_UNIT_DFT_C
-void opus_ifft(const kiss_fft_state *st,const kiss_fft_cpx *fin,kiss_fft_cpx
*fout)
+void opus_ifft_c(const kiss_fft_state *st,const kiss_fft_cpx *fin,kiss_fft_cpx
*fout)
 {
    int i;
    celt_assert2 (fin != fout, "In-place FFT not supported");
@@ -603,4 +602,3 @@ void opus_ifft(const kiss_fft_state *st,const kiss_fft_cpx
*fin,kiss_fft_cpx *fo
    for (i=0;i<st->nfft;i++)
       fout[i].i = -fout[i].i;
 }
-#endif
diff --git a/celt/kiss_fft.h b/celt/kiss_fft.h
index d3724e9..25b1ebc 100644
--- a/celt/kiss_fft.h
+++ b/celt/kiss_fft.h
@@ -140,7 +140,7 @@ kiss_fft_state *opus_fft_alloc(int nfft,void * mem,size_t *
lenmem, int arch);
     f[k].r and f[k].i
  * */
 void opus_fft_c(const kiss_fft_state *cfg,const kiss_fft_cpx *fin,kiss_fft_cpx
*fout);
-void opus_ifft(const kiss_fft_state *cfg,const kiss_fft_cpx *fin,kiss_fft_cpx
*fout);
+void opus_ifft_c(const kiss_fft_state *cfg,const kiss_fft_cpx *fin,kiss_fft_cpx
*fout);
 
 void opus_fft_impl(const kiss_fft_state *st,kiss_fft_cpx *fout);
 void opus_ifft_impl(const kiss_fft_state *st,kiss_fft_cpx *fout);
@@ -169,6 +169,13 @@ void (*const OPUS_FFT[OPUS_ARCHMASK+1])(const
kiss_fft_state *cfg,
                                         kiss_fft_cpx *fout);
 #define opus_fft(_cfg, _fin, _fout, arch) \
    ((*OPUS_FFT[(arch)&OPUS_ARCHMASK])(_cfg, _fin, _fout))
+
+void (*const OPUS_IFFT[OPUS_ARCHMASK+1])(const kiss_fft_state *cfg,
+                                         const kiss_fft_cpx *fin,
+                                         kiss_fft_cpx *fout);
+#define opus_ifft(_cfg, _fin, _fout, arch) \
+   ((*OPUS_IFFT[(arch)&OPUS_ARCHMASK])(_cfg, _fin, _fout))
+
 #else /* else for if defined(OPUS_HAVE_RTCD) &&
(defined(HAVE_ARM_NE10)) */
 
 #define opus_fft_alloc_arch(_st, arch) \
@@ -179,6 +186,10 @@ void (*const OPUS_FFT[OPUS_ARCHMASK+1])(const
kiss_fft_state *cfg,
 
 #define opus_fft(_cfg, _fin, _fout, arch) \
          ((void)(arch), opus_fft_c(_cfg, _fin, _fout))
+
+#define opus_ifft(_cfg, _fin, _fout, arch) \
+         ((void)(arch), opus_ifft_c(_cfg, _fin, _fout))
+
 #endif /* end if defined(OPUS_HAVE_RTCD) && (defined(HAVE_ARM_NE10)) */
 #endif /* end if !defined(OVERRIDE_OPUS_FFT) */
 
diff --git a/celt/mdct.c b/celt/mdct.c
index ee6d80e..5315ad1 100644
--- a/celt/mdct.c
+++ b/celt/mdct.c
@@ -239,12 +239,13 @@ void clt_mdct_forward_c(const mdct_lookup *l,
kiss_fft_scalar *in, kiss_fft_scal
 #endif /* OVERRIDE_clt_mdct_forward */
 
 #ifndef OVERRIDE_clt_mdct_backward
-void clt_mdct_backward(const mdct_lookup *l, kiss_fft_scalar *in,
kiss_fft_scalar * OPUS_RESTRICT out,
-      const opus_val16 * OPUS_RESTRICT window, int overlap, int shift, int
stride)
+void clt_mdct_backward_c(const mdct_lookup *l, kiss_fft_scalar *in,
kiss_fft_scalar * OPUS_RESTRICT out,
+      const opus_val16 * OPUS_RESTRICT window, int overlap, int shift, int
stride, int arch)
 {
    int i;
    int N, N2, N4;
    const kiss_twiddle_scalar *trig;
+   (void) arch;
 
    N = l->n;
    trig = l->trig;
diff --git a/celt/mdct.h b/celt/mdct.h
index cbaf679..5349ccf 100644
--- a/celt/mdct.h
+++ b/celt/mdct.h
@@ -69,9 +69,10 @@ void clt_mdct_forward_c(const mdct_lookup *l, kiss_fft_scalar
*in,
 
 /** Compute a backward MDCT (no scaling) and performs weighted overlap-add
     (scales implicitly by 1/2) */
-void clt_mdct_backward(const mdct_lookup *l, kiss_fft_scalar *in,
-      kiss_fft_scalar * OPUS_RESTRICT out,
-      const opus_val16 * OPUS_RESTRICT window, int overlap, int shift, int
stride);
+void clt_mdct_backward_c(const mdct_lookup *l, kiss_fft_scalar *in,
+                         kiss_fft_scalar * OPUS_RESTRICT out,
+                         const opus_val16 * OPUS_RESTRICT window,
+                         int overlap, int shift, int stride, int arch);
 
 #if !defined(OVERRIDE_OPUS_MDCT)
 /* Is run-time CPU detection enabled on this platform? */
@@ -88,11 +89,26 @@ void (*const CLT_MDCT_FORWARD_IMPL[OPUS_ARCHMASK+1])(const
mdct_lookup *l,
    ((*CLT_MDCT_FORWARD_IMPL[(arch)&OPUS_ARCHMASK])(_l, _in, _out, \
                                                    _window, _overlap, _shift, \
                                                    _stride, _arch))
+
+void (*const CLT_MDCT_BACKWARD_IMPL[OPUS_ARCHMASK+1])(const mdct_lookup *l,
+                                                      kiss_fft_scalar *in,
+                                                      kiss_fft_scalar *
OPUS_RESTRICT out,
+                                                      const opus_val16 *window,
+                                                      int overlap, int shift,
+                                                      int stride, int arch);
+
+#define clt_mdct_backward(_l, _in, _out, _window, _overlap, _shift, _stride,
_arch) \
+   (*CLT_MDCT_BACKWARD_IMPL[(arch)&OPUS_ARCHMASK])(_l, _in, _out, \
+                                                   _window, _overlap, _shift, \
+                                                   _stride, _arch)
 #else /* else for if defined(OPUS_HAVE_RTCD) &&
(defined(HAVE_ARM_NE10)) */
 
 #define clt_mdct_forward(_l, _in, _out, _window, _overlap, _shift, _stride,
_arch) \
    clt_mdct_forward_c(_l, _in, _out, _window, _overlap, _shift, _stride, _arch)
 
+#define clt_mdct_backward(_l, _in, _out, _window, _overlap, _shift, _stride,
_arch) \
+   clt_mdct_backward_c(_l, _in, _out, _window, _overlap, _shift, _stride,
_arch)
+
 #endif /* end if defined(OPUS_HAVE_RTCD) && (defined(HAVE_ARM_NE10)) */
 #endif /* end if !defined(OVERRIDE_OPUS_MDCT) */
 
diff --git a/celt/tests/test_unit_dft.c b/celt/tests/test_unit_dft.c
index badc30c..84f69bd 100644
--- a/celt/tests/test_unit_dft.c
+++ b/celt/tests/test_unit_dft.c
@@ -139,7 +139,7 @@ void test1d(int nfft,int isinverse,int arch)
     /*for (k=0;k<nfft;++k) printf("%d %d ", in[k].r,
in[k].i);printf("\n");*/
 
     if (isinverse)
-       opus_ifft(cfg,in,out);
+       opus_ifft(cfg,in,out, arch);
     else
        opus_fft(cfg,in,out, arch);
 
diff --git a/celt/tests/test_unit_mdct.c b/celt/tests/test_unit_mdct.c
index 72e4418..c64cac2 100644
--- a/celt/tests/test_unit_mdct.c
+++ b/celt/tests/test_unit_mdct.c
@@ -166,7 +166,7 @@ void test1d(int nfft,int isinverse,int arch)
     {
        for (k=0;k<nfft;++k)
           out[k] = 0;
-       clt_mdct_backward(&cfg,in,out, window, nfft/2, 0, 1);
+       clt_mdct_backward(&cfg,in,out, window, nfft/2, 0, 1, arch);
        /* apply TDAC because clt_mdct_backward() no longer does that */
        for (k=0;k<nfft/4;++k)
           out[nfft-k-1] = out[nfft/2+k];
-- 
1.9.1