Rearrange and remove some manual register assignments, make use of pointer address increments for free on arm, and remove macros in favour of explicit inline fns. also add memory clobbers to all uses of stm in inline asm. appears to resolve issues with codeclib failing to work as expected when using ARM_ASM optimisations on targets with native position independent code (e.g. android rockbox targets, but also tremor on non-rockbox targets that has been patched with the fastermdct patches)

git-svn-id: svn://svn.rockbox.org/rockbox/trunk@28262 a1c6a512-1295-4272-9138-f99709370657

2 files changed, 319 insertions, 203 deletions

diff --git a/apps/codecs/lib/fft-ffmpeg.c b/apps/codecs/lib/fft-ffmpeg.c
index a5ffab9086..c00abde694 100644
--- a/apps/codecs/lib/fft-ffmpeg.c
+++ b/apps/codecs/lib/fft-ffmpeg.c
@@ -202,7 +202,7 @@ static void ff_fft_permute_c(FFTContext *s, FFTComplex *z)
 */
 
 #ifndef FFT_FFMPEG_INCL_OPTIMISED_TRANSFORM
-static inline void TRANSFORM(FFTComplex * z, unsigned int n, FFTSample wre, FFTSample wim)
+static inline FFTComplex* TRANSFORM(FFTComplex * z, unsigned int n, FFTSample wre, FFTSample wim)
 {
     register FFTSample t1,t2,t5,t6,r_re,r_im;
     r_re = z[n*2].re;
@@ -212,9 +212,10 @@ static inline void TRANSFORM(FFTComplex * z, unsigned int n, FFTSample wre, FFTS
     r_im = z[n*3].im;
     XNPROD31_R(r_re, r_im, wre, wim, t5,t6);
     BUTTERFLIES(z[0],z[n],z[n*2],z[n*3]);
+    return z+1;
 }
 
-static inline void TRANSFORM_W01(FFTComplex * z, unsigned int n, const FFTSample * w)
+static inline FFTComplex* TRANSFORM_W01(FFTComplex * z, unsigned int n, const FFTSample * w)
 {
     register const FFTSample wre=w[0],wim=w[1];
     register FFTSample t1,t2,t5,t6,r_re,r_im;
@@ -225,9 +226,10 @@ static inline void TRANSFORM_W01(FFTComplex * z, unsigned int n, const FFTSample
     r_im = z[n*3].im;
     XNPROD31_R(r_re, r_im, wre, wim, t5,t6);
     BUTTERFLIES(z[0],z[n],z[n*2],z[n*3]);
+    return z+1;
 }
 
-static inline void TRANSFORM_W10(FFTComplex * z, unsigned int n, const FFTSample * w)
+static inline FFTComplex* TRANSFORM_W10(FFTComplex * z, unsigned int n, const FFTSample * w)
 {
     register const FFTSample wim=w[0],wre=w[1];
     register FFTSample t1,t2,t5,t6,r_re,r_im;
@@ -238,9 +240,10 @@ static inline void TRANSFORM_W10(FFTComplex * z, unsigned int n, const FFTSample
     r_im = z[n*3].im;
     XNPROD31_R(r_re, r_im, wre, wim, t5,t6);
     BUTTERFLIES(z[0],z[n],z[n*2],z[n*3]);
+    return z+1;
 }
 
-static inline void TRANSFORM_EQUAL(FFTComplex * z, unsigned int n)
+static inline FFTComplex* TRANSFORM_EQUAL(FFTComplex * z, unsigned int n)
 {
     register FFTSample t1,t2,t5,t6,temp1,temp2;
     register FFTSample * my_z = (FFTSample *)(z);
@@ -256,9 +259,10 @@ static inline void TRANSFORM_EQUAL(FFTComplex * z, unsigned int n)
     t5 = ( temp2 - t5 );
     my_z -= n*6;
     BUTTERFLIES(z[0],z[n],z[n*2],z[n*3]);
+    return z+1;
 }
 
-static inline void TRANSFORM_ZERO(FFTComplex * z, unsigned int n)
+static inline FFTComplex* TRANSFORM_ZERO(FFTComplex * z, unsigned int n)
 {
     FFTSample t1,t2,t5,t6;
     t1 = z[n*2].re;
@@ -266,6 +270,7 @@ static inline void TRANSFORM_ZERO(FFTComplex * z, unsigned int n)
     t5 = z[n*3].re;
     t6 = z[n*3].im;
     BUTTERFLIES(z[0],z[n],z[n*2],z[n*3]);
+    return z+1;
 }
 #endif
 
@@ -282,17 +287,14 @@ void pass(FFTComplex *z_arg, unsigned int STEP_arg, unsigned int n_arg)
     register const FFTSample *w_end = sincos_lookup0+1024;
 
     /* first two are special (well, first one is special, but we need to do pairs) */
-    TRANSFORM_ZERO(z,n);
-    z++;
-    TRANSFORM_W10(z,n,w);
+    z = TRANSFORM_ZERO(z,n);
+    z = TRANSFORM_W10(z,n,w);
     w += STEP;
     /* first pass forwards through sincos_lookup0*/
     do {
-        z++;
-        TRANSFORM_W10(z,n,w);
+        z = TRANSFORM_W10(z,n,w);
         w += STEP;
-        z++;
-        TRANSFORM_W10(z,n,w);
+        z = TRANSFORM_W10(z,n,w);
         w += STEP;
     } while(LIKELY(w < w_end));
     /* second half: pass backwards through sincos_lookup0*/
@@ -300,11 +302,9 @@ void pass(FFTComplex *z_arg, unsigned int STEP_arg, unsigned int n_arg)
     w_end=sincos_lookup0;
     while(LIKELY(w>w_end))
     {
-        z++;
-        TRANSFORM_W01(z,n,w);
+        z = TRANSFORM_W01(z,n,w);
         w -= STEP;
-        z++;
-        TRANSFORM_W01(z,n,w);
+        z = TRANSFORM_W01(z,n,w);
         w -= STEP;
     }
 }
diff --git a/apps/codecs/lib/fft-ffmpeg_arm.h b/apps/codecs/lib/fft-ffmpeg_arm.h
index 9d396a3fc0..073ad8ee46 100644
--- a/apps/codecs/lib/fft-ffmpeg_arm.h
+++ b/apps/codecs/lib/fft-ffmpeg_arm.h
@@ -43,6 +43,7 @@
     y = x - (b<<1);\
 }
 
+
 /* standard BUTTERFLIES package.  Note, we actually manually inline this
    in all the TRANSFORM macros below anyway */
 #define FFT_FFMPEG_INCL_OPTIMISED_BUTTERFLIES
@@ -59,198 +60,314 @@
 
 #define FFT_FFMPEG_INCL_OPTIMISED_TRANSFORM
 
-/* on ARM, all the TRANSFORM_etc inlines use the following registers:
-   r5,r6,r7,r8,r9,r10,r4,r12
-   
-   inputs are: z, n, STEP
-   
-   NOTE THAT THESE MACROS ACTUALLY CHANGE z INPUT INPLACE-
-   so sequential actions, z += n*3, z -= n*2 etc etc matter
-*/
-   
+static inline FFTComplex* TRANSFORM( FFTComplex* z, int n, FFTSample wre, FFTSample wim )
+{
+    register FFTSample t1,t2 asm("r5"),t5 asm("r6"),t6 asm("r7"),r_re asm("r8"),r_im asm("r9");
+    z += n*2; /* z[o2] */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XPROD31_R(r_re, r_im, wre, wim, t1,t2);
+    
+    z += n; /* z[o3] */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XNPROD31_R(r_re, r_im, wre, wim, t5,t6);
+    
+    BF_OPT(t1, t5, t5, t1);
+    BF_OPT(t6, t2, t2, t6);
 
-#define TRANSFORM_POST_STORE( z, n ) {\
-    /*{*/\
-    /*   BF_OPT(t1, t5, t5, t1);*/\
-    /*    BF_OPT(t6, t2, t2, t6);*/\
-    /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/\
-    /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/\
-    /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/\
-    /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/\
-    /*}*/\
-    z -= n*3;\
-    /* r_re = my_z[0]; r_im = my_z[1]; */\
-    {\
-        register FFTSample rt0temp asm("r4");\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        BF_OPT(rt0temp, r_re, r_re, t5);\
-        BF_OPT(t2,      r_im, r_im, t2);\
-        /* my_z[0] = r_re; my_z[1] = r_im; */\
-        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im));\
-        z += n;\
-        /* r_re = my_z[0]; r_im = my_z[1]; */\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        BF_OPT(t5, r_re, r_re, t6);\
-        BF_OPT(t6, r_im, r_im, t1);\
-        /* my_z[0] = r_re; my_z[1] = r_im; */\
-        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im));\
-        z += n;\
-        /* my_z[0] = rt0temp; my_z[1] = t2; */\
-        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2));\
-        z += n;\
-    }\
-    /* my_z[0] = t5; my_z[1] = t6; */\
-    asm volatile( "stmia %[my_z], {%[t5],%[t6]}\n\t"::[my_z] "r" (z), [t5] "r" (t5), [t6] "r" (t6));\
-    z -= n*3;\
+    {    
+        register FFTSample rt0temp asm("r4");
+        /*{*/
+        /*   BF_OPT(t1, t5, t5, t1);*/
+        /*    BF_OPT(t6, t2, t2, t6);*/
+        /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/
+        /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/
+        /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/
+        /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/
+        /*}*/
+        z -= n*3;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(rt0temp, r_re, r_re, t5);
+        BF_OPT(t2,      r_im, r_im, t2);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory" );
+        z += n;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(t5, r_re, r_re, t6);
+        BF_OPT(t6, r_im, r_im, t1);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* my_z[0] = rt0temp; my_z[1] = t2; */
+        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2):"memory");
+    }
+    z += n;
+   
+    /* my_z[0] = t5; my_z[1] = t6; */
+    asm volatile( "stmia %[my_z]!, {%[t5],%[t6]}\n\t":[my_z] "+r" (z) : [t5] "r" (t5), [t6] "r" (t6):"memory");
+    z -= n*3;
+    return(z);
 }
 
-#define TRANSFORM( z, n, wre_arg, wim_arg )\
-{\
-    FFTSample wre = wre_arg, wim = wim_arg;\
-    register FFTSample t1 asm("r5"),t2 asm("r6"),t5 asm("r7"),t6 asm("r8"),r_re asm("r9"),r_im asm("r10");\
-    z += n*2; /* z[o2] */\
-    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-    XPROD31_R(r_re, r_im, wre, wim, t1,t2);\
-    \
-    z += n; /* z[o3] */\
-    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-    XNPROD31_R(r_re, r_im, wre, wim, t5,t6);\
-    \
-    BF_OPT(t1, t5, t5, t1);\
-    BF_OPT(t6, t2, t2, t6);\
-    TRANSFORM_POST_STORE( z, n );\
-}
+static inline FFTComplex* TRANSFORM_W01( FFTComplex* z, int n, const FFTSample* w )
+{
+    register FFTSample t1,t2 asm("r5"),t5 asm("r6"),t6 asm("r7"),r_re asm("r8"),r_im asm("r9");
+    
+    /* load wre,wim into t5,t6 */
+    asm volatile( "ldmia %[w], {%[wre], %[wim]}\n\t":[wre] "=r" (t5), [wim] "=r" (t6):[w] "r" (w));
+    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XPROD31_R(r_re, r_im, t5 /*wre*/, t6 /*wim*/, t1,t2);
 
-#define TRANSFORM_W01( z, n, w )\
-{\
-    register FFTSample t1 asm("r5"),t2 asm("r6"),t5 asm("r7"),t6 asm("r8"),r_re asm("r9"),r_im asm("r10");\
-    \
-    {\
-        register FFTSample wre asm("r4"),wim asm("r12");\
-        asm volatile( "ldmia %[w], {%[wre], %[wim]}\n\t":[wre] "=r" (wre), [wim] "=r" (wim):[w] "r" (w));\
-        z += n*2; /* z[o2] -- 2n * 2 since complex numbers */\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        XPROD31_R(r_re, r_im, wre, wim, t1,t2);\
-\
-        z += n; /* z[o3] */\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        XNPROD31_R(r_re, r_im, wre, wim, t5,t6);\
-    }\
-    \
-    BF_OPT(t1, t5, t5, t1);\
-    BF_OPT(t6, t2, t2, t6);\
-    TRANSFORM_POST_STORE( z, n );\
+    z += n; /* z[o3] */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XNPROD31_R(r_re, r_im, t5 /*wre*/, t6 /*wim*/, t5,t6);
+    
+    BF_OPT(t1, t5, t5, t1);
+    BF_OPT(t6, t2, t2, t6);
+    {
+        register FFTSample rt0temp asm("r4");
+        /*{*/
+        /*   BF_OPT(t1, t5, t5, t1);*/
+        /*    BF_OPT(t6, t2, t2, t6);*/
+        /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/
+        /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/
+        /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/
+        /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/
+        /*}*/
+        z -= n*3;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(rt0temp, r_re, r_re, t5);
+        BF_OPT(t2,      r_im, r_im, t2);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(t5, r_re, r_re, t6);
+        BF_OPT(t6, r_im, r_im, t1);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* my_z[0] = rt0temp; my_z[1] = t2; */
+        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2):"memory");
+    }
+    z += n;
+
+    /* my_z[0] = t5; my_z[1] = t6; */
+    asm volatile( "stmia %[my_z]!, {%[t5],%[t6]}\n\t":[my_z] "+r" (z) : [t5] "r" (t5), [t6] "r" (t6):"memory");
+    z -= n*3;
+    return(z);
 }
 
-//static inline void TRANSFORM_W10(int32_t * z, unsigned int n, const int32_t * w)
-#define TRANSFORM_W10( z, n, w )\
-{\
-    register FFTSample t1 asm("r5"),t2 asm("r6"),t5 asm("r7"),t6 asm("r8"),r_re asm("r9"),r_im asm("r10");\
-    \
-    {\
-        register FFTSample wim asm("r4"),wre asm("r12");\
-        asm volatile( "ldmia %[w], {%[wim], %[wre]}\n\t":[wim] "=r" (wim), [wre] "=r" (wre):[w] "r" (w));\
-        z += n*2; /* z[o2] -- 2n * 2 since complex numbers */\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        XPROD31_R(r_re, r_im, wre, wim, t1,t2);\
-\
-        z += n; /* z[o3] */\
-        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-        XNPROD31_R(r_re, r_im, wre, wim, t5,t6);\
-    }\
-    \
-    BF_OPT(t1, t5, t5, t1);\
-    BF_OPT(t6, t2, t2, t6);\
-    TRANSFORM_POST_STORE( z, n );\
+static inline FFTComplex* TRANSFORM_W10( FFTComplex* z, int n, const FFTSample* w )
+{
+    register FFTSample t1,t2 asm("r5"),t5 asm("r6"),t6 asm("r7"),r_re asm("r8"),r_im asm("r9");
+    
+    /* load wim,wre into t5,t6 */
+    asm volatile( "ldmia %[w], {%[wim], %[wre]}\n\t":[wim] "=r" (t5), [wre] "=r" (t6):[w] "r" (w));
+    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XPROD31_R(r_re, r_im, t6 /*wim*/, t5 /*wre*/, t1,t2);
+
+    z += n; /* z[o3] */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    XNPROD31_R(r_re, r_im, t6 /*wim*/, t5 /*wre*/, t5,t6);
+    
+    BF_OPT(t1, t5, t5, t1);
+    BF_OPT(t6, t2, t2, t6);
+    {
+        register FFTSample rt0temp asm("r4");
+        /*{*/
+        /*   BF_OPT(t1, t5, t5, t1);*/
+        /*    BF_OPT(t6, t2, t2, t6);*/
+        /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/
+        /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/
+        /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/
+        /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/
+        /*}*/
+        z -= n*3;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(rt0temp, r_re, r_re, t5);
+        BF_OPT(t2,      r_im, r_im, t2);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(t5, r_re, r_re, t6);
+        BF_OPT(t6, r_im, r_im, t1);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* my_z[0] = rt0temp; my_z[1] = t2; */
+        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2):"memory");
+    }
+    z += n;
+
+    /* my_z[0] = t5; my_z[1] = t6; */
+    asm volatile( "stmia %[my_z]!, {%[t5],%[t6]}\n\t":[my_z] "+r" (z) : [t5] "r" (t5), [t6] "r" (t6):"memory");
+    z -= n*3;
+    return(z);
 }
 
-#define TRANSFORM_EQUAL( z, n )\
-{\
-    register FFTSample t1 asm("r5"),t2 asm("r6"),t5 asm("r7"),t6 asm("r8"),r_re asm("r9"),r_im asm("r10");\
-\
-    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */\
-    asm volatile( "ldmia %[my_z], {%[t5],%[t6]}\n\t":[t5] "=r" (t5), [t6] "=r" (t6):[my_z] "r" (z));\
-    z += n; /* z[o3] */\
-    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));\
-\
-/**/\
-/*t2 = MULT32(cPI2_8, t5);*/\
-/*t1 = MULT31(cPI2_8, t6);*/\
-/*t6 = MULT31(cPI2_8, r_re);*/\
-/*t5 = MULT32(cPI2_8, r_im);*/\
-\  
-/*t1 = ( t1 + (t2<<1) );*/\
-/*t2 = ( t1 - (t2<<2) );*/\
-/*t6 = ( t6 + (t5<<1) );*/\
-/*t5 = ( t6 - (t5<<2) );*/\
-/**/\
-    t2   = MULT31(cPI2_8, t5);\
-    t6   = MULT31(cPI2_8, t6);\
-    r_re = MULT31(cPI2_8, r_re);\
-    t5   = MULT31(cPI2_8, r_im);\
-    \
-    t1 = ( t6 + t2 );\
-    t2 = ( t6 - t2 );\
-    t6 = ( r_re + t5 );\
-    t5 = ( r_re - t5 );\
-    \
-    BF_OPT(t1, t5, t5, t1);\
-    BF_OPT(t6, t2, t2, t6);\
-    TRANSFORM_POST_STORE( z, n );\
+static inline FFTComplex* TRANSFORM_EQUAL( FFTComplex* z, int n )
+{
+    register FFTSample t1,t2 asm("r5"),t5 asm("r6"),t6 asm("r7"),r_re asm("r8"),r_im asm("r9");
+
+    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */
+    asm volatile( "ldmia %[my_z], {%[t5],%[t6]}\n\t":[t5] "=r" (t5), [t6] "=r" (t6):[my_z] "r" (z));
+    z += n; /* z[o3] */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+
+/**/
+/*t2 = MULT32(cPI2_8, t5);*/
+/*t1 = MULT31(cPI2_8, t6);*/
+/*t6 = MULT31(cPI2_8, r_re);*/
+/*t5 = MULT32(cPI2_8, r_im);*/
+
+/*t1 = ( t1 + (t2<<1) );*/
+/*t2 = ( t1 - (t2<<2) );*/
+/*t6 = ( t6 + (t5<<1) );*/
+/*t5 = ( t6 - (t5<<2) );*/
+/**/
+    t2   = MULT31(cPI2_8, t5);
+    t6   = MULT31(cPI2_8, t6);
+    r_re = MULT31(cPI2_8, r_re);
+    t5   = MULT31(cPI2_8, r_im);
+    
+    t1 = ( t6 + t2 );
+    t2 = ( t6 - t2 );
+    t6 = ( r_re + t5 );
+    t5 = ( r_re - t5 );
+    
+    BF_OPT(t1, t5, t5, t1);
+    BF_OPT(t6, t2, t2, t6);
+    {
+        register FFTSample rt0temp asm("r4");
+        /*{*/
+        /*   BF_OPT(t1, t5, t5, t1);*/
+        /*    BF_OPT(t6, t2, t2, t6);*/
+        /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/
+        /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/
+        /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/
+        /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/
+        /*}*/
+        z -= n*3;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(rt0temp, r_re, r_re, t5);
+        BF_OPT(t2,      r_im, r_im, t2);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(t5, r_re, r_re, t6);
+        BF_OPT(t6, r_im, r_im, t1);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* my_z[0] = rt0temp; my_z[1] = t2; */
+        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2):"memory");
+    }
+    z += n;
+
+    /* my_z[0] = t5; my_z[1] = t6; */
+    asm volatile( "stmia %[my_z]!, {%[t5],%[t6]}\n\t":[my_z] "+r" (z) : [t5] "r" (t5), [t6] "r" (t6):"memory");
+    z -= n*3;
+    return(z);
 }
 
-#define TRANSFORM_ZERO( z,n )\
-{\
-    register FFTSample t1 asm("r5"),t2 asm("r6"),t5 asm("r7"),t6 asm("r8"),r_re asm("r9"),r_im asm("r10");\
-\
-    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */\
-    asm volatile( "ldmia %[my_z], {%[t1],%[t2]}\n\t":[t1] "=r" (t1), [t2] "=r" (t2):[my_z] "r" (z));\
-    z += n; /* z[o3] */\
-    asm volatile( "ldmia %[my_z], {%[t5],%[t6]}\n\t":[t5] "=r" (t5), [t6] "=r" (t6):[my_z] "r" (z));\
-\
-    BF_OPT(t1, t5, t5, t1);\
-    BF_OPT(t6, t2, t2, t6);\
-    TRANSFORM_POST_STORE( z, n );\
+static inline FFTComplex* TRANSFORM_ZERO( FFTComplex* z, int n )
+{
+    register FFTSample t1,t2 asm("r5"),t5 asm("r6"),t6 asm("r7"), r_re asm("r8"), r_im asm("r9");
+
+    z += n*2; /* z[o2] -- 2n * 2 since complex numbers */
+    asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+    z += n; /* z[o3] */
+    asm volatile( "ldmia %[my_z], {%[t5],%[t6]}\n\t":[t5] "=r" (t5), [t6] "=r" (t6):[my_z] "r" (z));
+
+    BF_OPT(t1, t5, t5, r_re);
+    BF_OPT(t6, t2, r_im, t6);
+    {
+        register FFTSample rt0temp asm("r4");
+        /*{*/
+        /*   BF_OPT(t1, t5, t5, t1);*/
+        /*    BF_OPT(t6, t2, t2, t6);*/
+        /*    BF_OPT(a2.re, a0.re, a0.re, t5);*/
+        /*    BF_OPT(a2.im, a0.im, a0.im, t2);*/
+        /*    BF_OPT(a3.re, a1.re, a1.re, t6);*/
+        /*    BF_OPT(a3.im, a1.im, a1.im, t1);*/
+        /*}*/
+        z -= n*3;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(rt0temp, r_re, r_re, t5);
+        BF_OPT(t2,      r_im, r_im, t2);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* r_re = my_z[0]; r_im = my_z[1]; */
+        asm volatile( "ldmia %[my_z], {%[r_re],%[r_im]}\n\t":[r_re] "=r" (r_re), [r_im] "=r" (r_im):[my_z] "r" (z));
+        BF_OPT(t5, r_re, r_re, t6);
+        BF_OPT(t6, r_im, r_im, t1);
+        /* my_z[0] = r_re; my_z[1] = r_im; */
+        asm volatile( "stmia %[my_z], {%[r_re],%[r_im]}\n\t"::[my_z] "r" (z), [r_re] "r" (r_re), [r_im] "r" (r_im):"memory");
+        z += n;
+        /* my_z[0] = rt0temp; my_z[1] = t2; */
+        asm volatile( "stmia %[my_z], {%[rt0temp],%[t2]}\n\t"::[my_z] "r" (z), [rt0temp] "r" (rt0temp), [t2] "r" (t2):"memory");
+    }
+    z += n;
+
+    /* my_z[0] = t5; my_z[1] = t6; */
+    asm volatile( "stmia %[my_z]!, {%[t5],%[t6]}\n\t":[my_z] "+r" (z) : [t5] "r" (t5), [t6] "r" (t6):"memory");
+    z -= n*3;
+    return(z);
 }
 
 #define FFT_FFMPEG_INCL_OPTIMISED_FFT4
-#define fft4(z_arg)\
-{\
-    /* input[0..7] -> output[0..7] */\
-    fixed32 * m = (fixed32 *) ( ( z_arg ) );\
-    /* load r1=z[0],r2=z[1],...,r8=z[7] */\
-    asm volatile(\
-      "ldmia %[z], {r1-r8}\n\t"\
-      "add r1,r1,r3\n\t"         /* r1 :=t1 */\
-      "sub r3,r1,r3, lsl #1\n\t" /* r3 :=t3 */\
-      "sub r7,r7,r5\n\t"         /* r10:=t8 */\
-      "add r5,r7,r5, lsl #1\n\t" /* r5 :=t6 */\
-      \
-      "add r1,r1,r5\n\t"                 /* r1 = o[0] */\
-      "sub r5,r1,r5, lsl #1\n\t"         /* r5 = o[4] */\
-      \
-      "add r2,r2,r4\n\t"         /* r2 :=t2 */\
-      "sub r4,r2,r4, lsl #1\n\t" /* r9 :=t4 */\
-      \
-      "add r12,r6,r8\n\t"        /* r10:=t5 */\
-      "sub r6,r6,r8\n\t"         /* r6 :=t7 */\
-      \
-      "sub r8,r4,r7\n\t"                 /* r8 = o[7]*/ \
-      "add r4,r4,r7\n\t"                 /* r4 = o[3]*/ \
-      "sub r7,r3,r6\n\t"                 /* r7 = o[6]*/ \
-      "add r3,r3,r6\n\t"                 /* r3 = o[2]*/ \
-      "sub r6,r2,r12\n\t"                /* r6 = o[5]*/ \
-      "add r2,r2,r12\n\t"                /* r2 = o[1]*/ \
-      \
-      "stmia %[z], {r1-r8}\n\t"\
-      : /* outputs */\
-      : /* inputs */ [z] "r" (m)\
-      : /* clobbers */\
-      "r1","r2","r3","r4","r5","r6","r7","r8","r12","memory"\
-   );\
+static inline FFTComplex* fft4(FFTComplex * z)
+{
+    FFTSample temp;
+    
+    /* input[0..7] -> output[0..7] */
+    /* load r1=z[0],r2=z[1],...,r8=z[7] */
+    asm volatile(
+      "ldmia %[z], {r1-r8}\n\t"
+      "add r1,r1,r3\n\t"         /* r1 :=t1 */
+      "sub r3,r1,r3, lsl #1\n\t" /* r3 :=t3 */
+      "sub r7,r7,r5\n\t"         /* r10:=t8 */
+      "add r5,r7,r5, lsl #1\n\t" /* r5 :=t6 */
+      
+      "add r1,r1,r5\n\t"                 /* r1 = o[0] */
+      "sub r5,r1,r5, lsl #1\n\t"         /* r5 = o[4] */
+      
+      "add r2,r2,r4\n\t"         /* r2 :=t2 */
+      "sub r4,r2,r4, lsl #1\n\t" /* r9 :=t4 */
+      
+      "add %[temp],r6,r8\n\t"        /* r10:=t5 */
+      "sub r6,r6,r8\n\t"         /* r6 :=t7 */
+      
+      "sub r8,r4,r7\n\t"                 /* r8 = o[7]*/ 
+      "add r4,r4,r7\n\t"                 /* r4 = o[3]*/ 
+      "sub r7,r3,r6\n\t"                 /* r7 = o[6]*/ 
+      "add r3,r3,r6\n\t"                 /* r3 = o[2]*/ 
+      "sub r6,r2,%[temp]\n\t"                /* r6 = o[5]*/ 
+      "add r2,r2,%[temp]\n\t"                /* r2 = o[1]*/ 
+      
+      "stmia %[z]!, {r1-r8}\n\t"
+      : /* outputs */ [z] "+r" (z), [temp] "=r" (temp)
+      : /* inputs */
+      : /* clobbers */
+      "r1","r2","r3","r4","r5","r6","r7","r8","memory"
+   );
+   return z;
 }
 
-
 #define FFT_FFMPEG_INCL_OPTIMISED_FFT8
         /* The chunk of asm below is equivalent to the following:
         
@@ -279,12 +396,14 @@
         // Finally save out z[4].re, z[4].im, z[0].re and z[0].im
         // ...
         */
-static inline void fft8( FFTComplex * z )
+static inline void fft8(FFTComplex * z)
 {
-    fft4(z);
+    FFTComplex* m4 = fft4(z);
     {
-        FFTSample temp;
-        fixed32 * m4 = (fixed32 *)(&(z[4].re));
+        /* note that we increment z_ptr on the final stmia, which 
+           leaves z_ptr pointing to z[1].re ready for the Transform step */
+           
+        register FFTSample temp;
 
         asm volatile(
             /* read in z[4].re thru z[7].im */
@@ -323,18 +442,15 @@ static inline void fft8( FFTComplex * z )
             "add r8,r8,r2\n\t"
             "sub r2,r8,r2,lsl #1\n\t"
 
-            "stmia %[z_ptr],{r7,r8}\n\t" /* write out z[0].re, z[0].im */
+            "stmia %[z_ptr]!,{r7,r8}\n\t" /* write out z[0].re, z[0].im */
             "stmdb %[z4_ptr], {r1,r2}\n\t" /* write out z[4].re, z[4].im */
-            : [z4_ptr] "+r" (m4), [temp] "=r" (temp)
-            : [z_ptr] "r" (z)
+            : [z4_ptr] "+r" (m4), [temp] "=r" (temp), [z_ptr] "+r" (z)
+            :
             : "r1","r2","r3","r4","r5","r6","r7","r8","memory"
         );
     }
 
-    z++;
     TRANSFORM_EQUAL(z,2);
 }
 
-
 #endif // CPU_ARM
-