FFmpeg: libavcodec/x86/simple

00001 /*
00002  * Simple IDCT MMX
00003  *
00004  * Copyright (c) 2001, 2002 Michael Niedermayer <michaelni@gmx.at>
00005  *
00006  * This file is part of FFmpeg.
00007  *
00008  * FFmpeg is free software; you can redistribute it and/or
00009  * modify it under the terms of the GNU Lesser General Public
00010  * License as published by the Free Software Foundation; either
00011  * version 2.1 of the License, or (at your option) any later version.
00012  *
00013  * FFmpeg is distributed in the hope that it will be useful,
00014  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00015  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00016  * Lesser General Public License for more details.
00017  *
00018  * You should have received a copy of the GNU Lesser General Public
00019  * License along with FFmpeg; if not, write to the Free Software
00020  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00021  */
00022 #include "libavcodec/dsputil.h"
00023 #include "libavcodec/simple_idct.h"
00024 #include "libavutil/mem.h"
00025 #include "dsputil_mmx.h"
00026 
00027 #if HAVE_INLINE_ASM
00028 
00029 /*
00030 23170.475006
00031 22725.260826
00032 21406.727617
00033 19265.545870
00034 16384.000000
00035 12872.826198
00036 8866.956905
00037 4520.335430
00038 */
00039 #define C0 23170 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00040 #define C1 22725 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00041 #define C2 21407 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00042 #define C3 19266 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00043 #define C4 16383 //cos(i*M_PI/16)*sqrt(2)*(1<<14) - 0.5
00044 #define C5 12873 //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00045 #define C6 8867  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00046 #define C7 4520  //cos(i*M_PI/16)*sqrt(2)*(1<<14) + 0.5
00047 
00048 #define ROW_SHIFT 11
00049 #define COL_SHIFT 20 // 6
00050 
00051 DECLARE_ASM_CONST(8, uint64_t, wm1010)= 0xFFFF0000FFFF0000ULL;
00052 DECLARE_ASM_CONST(8, uint64_t, d40000)= 0x0000000000040000ULL;
00053 
00054 DECLARE_ALIGNED(8, static const int16_t, coeffs)[]= {
00055         1<<(ROW_SHIFT-1), 0, 1<<(ROW_SHIFT-1), 0,
00056 //        1<<(COL_SHIFT-1), 0, 1<<(COL_SHIFT-1), 0,
00057 //        0, 1<<(COL_SHIFT-1-16), 0, 1<<(COL_SHIFT-1-16),
00058         1<<(ROW_SHIFT-1), 1, 1<<(ROW_SHIFT-1), 0,
00059         // the 1 = ((1<<(COL_SHIFT-1))/C4)<<ROW_SHIFT :)
00060 //        0, 0, 0, 0,
00061 //        0, 0, 0, 0,
00062 
00063  C4,  C4,  C4,  C4,
00064  C4, -C4,  C4, -C4,
00065 
00066  C2,  C6,  C2,  C6,
00067  C6, -C2,  C6, -C2,
00068 
00069  C1,  C3,  C1,  C3,
00070  C5,  C7,  C5,  C7,
00071 
00072  C3, -C7,  C3, -C7,
00073 -C1, -C5, -C1, -C5,
00074 
00075  C5, -C1,  C5, -C1,
00076  C7,  C3,  C7,  C3,
00077 
00078  C7, -C5,  C7, -C5,
00079  C3, -C1,  C3, -C1
00080 };
00081 
00082 static inline void idct(int16_t *block)
00083 {
00084         DECLARE_ALIGNED(8, int64_t, align_tmp)[16];
00085         int16_t * const temp= (int16_t*)align_tmp;
00086 
00087         __asm__ volatile(
00088 #if 0 //Alternative, simpler variant
00089 
00090 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
00091         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00092         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00093         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00094         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00095         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00096         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00097         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00098         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00099         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00100         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00101         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00102         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00103         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00104         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00105         #rounder ", %%mm4               \n\t"\
00106         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00107         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00108         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00109         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
00110         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00111         #rounder ", %%mm0               \n\t"\
00112         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
00113         "paddd %%mm0, %%mm0             \n\t" \
00114         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
00115         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00116         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
00117         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
00118         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00119         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00120         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00121         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00122         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
00123         "psrad $" #shift ", %%mm7       \n\t"\
00124         "psrad $" #shift ", %%mm4       \n\t"\
00125         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
00126         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
00127         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00128         "psrad $" #shift ", %%mm1       \n\t"\
00129         "psrad $" #shift ", %%mm2       \n\t"\
00130         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
00131         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
00132         "movq %%mm7, " #dst "           \n\t"\
00133         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
00134         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00135         "movq %%mm2, 24+" #dst "        \n\t"\
00136         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00137         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00138         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00139         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00140         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
00141         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00142         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00143         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00144         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
00145         "psrad $" #shift ", %%mm2       \n\t"\
00146         "psrad $" #shift ", %%mm0       \n\t"\
00147         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00148         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
00149         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00150         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00151         "psrad $" #shift ", %%mm6       \n\t"\
00152         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
00153         "movq %%mm2, 8+" #dst "         \n\t"\
00154         "psrad $" #shift ", %%mm4       \n\t"\
00155         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
00156         "movq %%mm4, 16+" #dst "        \n\t"\
00157 
00158 #define COL_IDCT(src0, src4, src1, src5, dst, shift) \
00159         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00160         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00161         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00162         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00163         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00164         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00165         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00166         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00167         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00168         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00169         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00170         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00171         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00172         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00173         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00174         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00175         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00176         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00177         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
00178         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
00179         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
00180         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00181         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00182         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
00183         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
00184         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00185         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00186         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00187         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00188         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
00189         "psrad $" #shift ", %%mm7       \n\t"\
00190         "psrad $" #shift ", %%mm4       \n\t"\
00191         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
00192         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00193         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00194         "psrad $" #shift ", %%mm0       \n\t"\
00195         "psrad $" #shift ", %%mm2       \n\t"\
00196         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
00197         "movd %%mm7, " #dst "           \n\t"\
00198         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00199         "movd %%mm0, 16+" #dst "        \n\t"\
00200         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
00201         "movd %%mm2, 96+" #dst "        \n\t"\
00202         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00203         "movd %%mm4, 112+" #dst "       \n\t"\
00204         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
00205         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00206         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00207         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00208         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00209         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00210         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
00211         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00212         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00213         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00214         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00215         "psrad $" #shift ", %%mm2       \n\t"\
00216         "psrad $" #shift ", %%mm5       \n\t"\
00217         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00218         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
00219         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00220         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00221         "psrad $" #shift ", %%mm6       \n\t"\
00222         "psrad $" #shift ", %%mm4       \n\t"\
00223         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
00224         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00225         "movd %%mm2, 32+" #dst "        \n\t"\
00226         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
00227         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00228         "movd %%mm6, 48+" #dst "        \n\t"\
00229         "movd %%mm4, 64+" #dst "        \n\t"\
00230         "movd %%mm5, 80+" #dst "        \n\t"\
00231 
00232 
00233 #define DC_COND_ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
00234         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00235         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00236         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00237         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00238         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
00239         "pand %%mm0, %%mm4              \n\t"\
00240         "por %%mm1, %%mm4               \n\t"\
00241         "por %%mm2, %%mm4               \n\t"\
00242         "por %%mm3, %%mm4               \n\t"\
00243         "packssdw %%mm4,%%mm4           \n\t"\
00244         "movd %%mm4, %%eax              \n\t"\
00245         "orl %%eax, %%eax               \n\t"\
00246         "jz 1f                          \n\t"\
00247         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00248         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00249         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00250         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00251         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00252         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00253         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00254         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00255         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00256         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00257         #rounder ", %%mm4               \n\t"\
00258         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00259         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00260         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00261         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
00262         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00263         #rounder ", %%mm0               \n\t"\
00264         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
00265         "paddd %%mm0, %%mm0             \n\t" \
00266         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
00267         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00268         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
00269         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
00270         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00271         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00272         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00273         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00274         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
00275         "psrad $" #shift ", %%mm7       \n\t"\
00276         "psrad $" #shift ", %%mm4       \n\t"\
00277         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
00278         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
00279         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00280         "psrad $" #shift ", %%mm1       \n\t"\
00281         "psrad $" #shift ", %%mm2       \n\t"\
00282         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
00283         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
00284         "movq %%mm7, " #dst "           \n\t"\
00285         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
00286         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00287         "movq %%mm2, 24+" #dst "        \n\t"\
00288         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00289         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00290         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00291         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00292         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
00293         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00294         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00295         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00296         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
00297         "psrad $" #shift ", %%mm2       \n\t"\
00298         "psrad $" #shift ", %%mm0       \n\t"\
00299         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00300         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
00301         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00302         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00303         "psrad $" #shift ", %%mm6       \n\t"\
00304         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
00305         "movq %%mm2, 8+" #dst "         \n\t"\
00306         "psrad $" #shift ", %%mm4       \n\t"\
00307         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
00308         "movq %%mm4, 16+" #dst "        \n\t"\
00309         "jmp 2f                         \n\t"\
00310         "1:                             \n\t"\
00311         "pslld $16, %%mm0               \n\t"\
00312         "#paddd "MANGLE(d40000)", %%mm0 \n\t"\
00313         "psrad $13, %%mm0               \n\t"\
00314         "packssdw %%mm0, %%mm0          \n\t"\
00315         "movq %%mm0, " #dst "           \n\t"\
00316         "movq %%mm0, 8+" #dst "         \n\t"\
00317         "movq %%mm0, 16+" #dst "        \n\t"\
00318         "movq %%mm0, 24+" #dst "        \n\t"\
00319         "2:                             \n\t"
00320 
00321 
00322 //IDCT(      src0,   src4,   src1,   src5,    dst,    rounder, shift)
00323 ROW_IDCT(    (%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
00324 /*ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1), paddd (%2), 11)
00325 ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1), paddd (%2), 11)
00326 ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1), paddd (%2), 11)*/
00327 
00328 DC_COND_ROW_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11)
00329 DC_COND_ROW_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11)
00330 DC_COND_ROW_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11)
00331 
00332 
00333 //IDCT(      src0,   src4,   src1,    src5,    dst, shift)
00334 COL_IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00335 COL_IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00336 COL_IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00337 COL_IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00338 
00339 #else
00340 
00341 #define DC_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
00342         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00343         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00344         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00345         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00346         "movq "MANGLE(wm1010)", %%mm4   \n\t"\
00347         "pand %%mm0, %%mm4              \n\t"\
00348         "por %%mm1, %%mm4               \n\t"\
00349         "por %%mm2, %%mm4               \n\t"\
00350         "por %%mm3, %%mm4               \n\t"\
00351         "packssdw %%mm4,%%mm4           \n\t"\
00352         "movd %%mm4, %%eax              \n\t"\
00353         "orl %%eax, %%eax               \n\t"\
00354         "jz 1f                          \n\t"\
00355         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00356         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00357         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00358         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00359         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00360         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00361         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00362         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00363         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00364         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00365         #rounder ", %%mm4               \n\t"\
00366         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00367         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00368         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00369         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
00370         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00371         #rounder ", %%mm0               \n\t"\
00372         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
00373         "paddd %%mm0, %%mm0             \n\t" \
00374         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
00375         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00376         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
00377         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
00378         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00379         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00380         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00381         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00382         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
00383         "psrad $" #shift ", %%mm7       \n\t"\
00384         "psrad $" #shift ", %%mm4       \n\t"\
00385         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
00386         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
00387         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00388         "psrad $" #shift ", %%mm1       \n\t"\
00389         "psrad $" #shift ", %%mm2       \n\t"\
00390         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
00391         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
00392         "movq %%mm7, " #dst "           \n\t"\
00393         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
00394         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00395         "movq %%mm2, 24+" #dst "        \n\t"\
00396         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00397         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00398         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00399         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00400         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
00401         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00402         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00403         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00404         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
00405         "psrad $" #shift ", %%mm2       \n\t"\
00406         "psrad $" #shift ", %%mm0       \n\t"\
00407         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00408         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
00409         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00410         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00411         "psrad $" #shift ", %%mm6       \n\t"\
00412         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
00413         "movq %%mm2, 8+" #dst "         \n\t"\
00414         "psrad $" #shift ", %%mm4       \n\t"\
00415         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
00416         "movq %%mm4, 16+" #dst "        \n\t"\
00417         "jmp 2f                         \n\t"\
00418         "1:                             \n\t"\
00419         "pslld $16, %%mm0               \n\t"\
00420         "paddd "MANGLE(d40000)", %%mm0  \n\t"\
00421         "psrad $13, %%mm0               \n\t"\
00422         "packssdw %%mm0, %%mm0          \n\t"\
00423         "movq %%mm0, " #dst "           \n\t"\
00424         "movq %%mm0, 8+" #dst "         \n\t"\
00425         "movq %%mm0, 16+" #dst "        \n\t"\
00426         "movq %%mm0, 24+" #dst "        \n\t"\
00427         "2:                             \n\t"
00428 
00429 #define Z_COND_IDCT(src0, src4, src1, src5, dst, rounder, shift, bt) \
00430         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00431         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00432         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00433         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00434         "movq %%mm0, %%mm4              \n\t"\
00435         "por %%mm1, %%mm4               \n\t"\
00436         "por %%mm2, %%mm4               \n\t"\
00437         "por %%mm3, %%mm4               \n\t"\
00438         "packssdw %%mm4,%%mm4           \n\t"\
00439         "movd %%mm4, %%eax              \n\t"\
00440         "orl %%eax, %%eax               \n\t"\
00441         "jz " #bt "                     \n\t"\
00442         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00443         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00444         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00445         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00446         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00447         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00448         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00449         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00450         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00451         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00452         #rounder ", %%mm4               \n\t"\
00453         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00454         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00455         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00456         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
00457         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00458         #rounder ", %%mm0               \n\t"\
00459         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
00460         "paddd %%mm0, %%mm0             \n\t" \
00461         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
00462         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00463         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
00464         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
00465         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00466         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00467         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00468         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00469         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
00470         "psrad $" #shift ", %%mm7       \n\t"\
00471         "psrad $" #shift ", %%mm4       \n\t"\
00472         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
00473         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
00474         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00475         "psrad $" #shift ", %%mm1       \n\t"\
00476         "psrad $" #shift ", %%mm2       \n\t"\
00477         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
00478         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
00479         "movq %%mm7, " #dst "           \n\t"\
00480         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
00481         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00482         "movq %%mm2, 24+" #dst "        \n\t"\
00483         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00484         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00485         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00486         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00487         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
00488         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00489         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00490         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00491         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
00492         "psrad $" #shift ", %%mm2       \n\t"\
00493         "psrad $" #shift ", %%mm0       \n\t"\
00494         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00495         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
00496         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00497         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00498         "psrad $" #shift ", %%mm6       \n\t"\
00499         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
00500         "movq %%mm2, 8+" #dst "         \n\t"\
00501         "psrad $" #shift ", %%mm4       \n\t"\
00502         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
00503         "movq %%mm4, 16+" #dst "        \n\t"\
00504 
00505 #define ROW_IDCT(src0, src4, src1, src5, dst, rounder, shift) \
00506         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00507         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00508         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00509         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00510         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00511         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00512         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00513         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00514         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00515         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00516         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00517         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00518         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00519         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00520         #rounder ", %%mm4               \n\t"\
00521         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00522         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00523         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00524         "movq 56(%2), %%mm5             \n\t" /* C7     C5      C7      C5 */\
00525         "pmaddwd %%mm3, %%mm5           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00526         #rounder ", %%mm0               \n\t"\
00527         "paddd %%mm0, %%mm1             \n\t" /* A1             a1 */\
00528         "paddd %%mm0, %%mm0             \n\t" \
00529         "psubd %%mm1, %%mm0             \n\t" /* A2             a2 */\
00530         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00531         "paddd %%mm5, %%mm7             \n\t" /* B0             b0 */\
00532         "movq 72(%2), %%mm5             \n\t" /* -C5    -C1     -C5     -C1 */\
00533         "pmaddwd %%mm3, %%mm5           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00534         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00535         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00536         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00537         "paddd %%mm2, %%mm5             \n\t" /* B1             b1 */\
00538         "psrad $" #shift ", %%mm7       \n\t"\
00539         "psrad $" #shift ", %%mm4       \n\t"\
00540         "movq %%mm1, %%mm2              \n\t" /* A1             a1 */\
00541         "paddd %%mm5, %%mm1             \n\t" /* A1+B1          a1+b1 */\
00542         "psubd %%mm5, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00543         "psrad $" #shift ", %%mm1       \n\t"\
00544         "psrad $" #shift ", %%mm2       \n\t"\
00545         "packssdw %%mm1, %%mm7          \n\t" /* A1+B1  a1+b1   A0+B0   a0+b0 */\
00546         "packssdw %%mm4, %%mm2          \n\t" /* A0-B0  a0-b0   A1-B1   a1-b1 */\
00547         "movq %%mm7, " #dst "           \n\t"\
00548         "movq " #src1 ", %%mm1          \n\t" /* R3     R1      r3      r1 */\
00549         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00550         "movq %%mm2, 24+" #dst "        \n\t"\
00551         "pmaddwd %%mm1, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00552         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00553         "pmaddwd 96(%2), %%mm1          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00554         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00555         "movq %%mm0, %%mm2              \n\t" /* A2             a2 */\
00556         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00557         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00558         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00559         "psubd %%mm4, %%mm0             \n\t" /* a2-B2          a2-b2 */\
00560         "psrad $" #shift ", %%mm2       \n\t"\
00561         "psrad $" #shift ", %%mm0       \n\t"\
00562         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00563         "paddd %%mm1, %%mm3             \n\t" /* B3             b3 */\
00564         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00565         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00566         "psrad $" #shift ", %%mm6       \n\t"\
00567         "packssdw %%mm6, %%mm2          \n\t" /* A3+B3  a3+b3   A2+B2   a2+b2 */\
00568         "movq %%mm2, 8+" #dst "         \n\t"\
00569         "psrad $" #shift ", %%mm4       \n\t"\
00570         "packssdw %%mm0, %%mm4          \n\t" /* A2-B2  a2-b2   A3-B3   a3-b3 */\
00571         "movq %%mm4, 16+" #dst "        \n\t"\
00572 
00573 //IDCT(         src0,   src4,   src1,   src5,    dst,   rounder, shift)
00574 DC_COND_IDCT(  0(%0),  8(%0), 16(%0), 24(%0),  0(%1),paddd 8(%2), 11)
00575 Z_COND_IDCT(  32(%0), 40(%0), 48(%0), 56(%0), 32(%1),paddd (%2), 11, 4f)
00576 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 2f)
00577 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 1f)
00578 
00579 #undef IDCT
00580 #define IDCT(src0, src4, src1, src5, dst, shift) \
00581         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00582         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00583         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00584         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00585         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00586         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00587         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00588         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00589         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00590         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00591         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00592         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00593         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00594         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00595         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00596         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00597         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00598         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00599         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
00600         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
00601         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
00602         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00603         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00604         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
00605         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
00606         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00607         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00608         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00609         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00610         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
00611         "psrad $" #shift ", %%mm7       \n\t"\
00612         "psrad $" #shift ", %%mm4       \n\t"\
00613         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
00614         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00615         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00616         "psrad $" #shift ", %%mm0       \n\t"\
00617         "psrad $" #shift ", %%mm2       \n\t"\
00618         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
00619         "movd %%mm7, " #dst "           \n\t"\
00620         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00621         "movd %%mm0, 16+" #dst "        \n\t"\
00622         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
00623         "movd %%mm2, 96+" #dst "        \n\t"\
00624         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00625         "movd %%mm4, 112+" #dst "       \n\t"\
00626         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
00627         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00628         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00629         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00630         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00631         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00632         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
00633         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00634         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00635         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00636         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00637         "psrad $" #shift ", %%mm2       \n\t"\
00638         "psrad $" #shift ", %%mm5       \n\t"\
00639         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00640         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
00641         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00642         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00643         "psrad $" #shift ", %%mm6       \n\t"\
00644         "psrad $" #shift ", %%mm4       \n\t"\
00645         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
00646         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00647         "movd %%mm2, 32+" #dst "        \n\t"\
00648         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
00649         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00650         "movd %%mm6, 48+" #dst "        \n\t"\
00651         "movd %%mm4, 64+" #dst "        \n\t"\
00652         "movd %%mm5, 80+" #dst "        \n\t"
00653 
00654 
00655 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
00656 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00657 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00658 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00659 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00660         "jmp 9f                         \n\t"
00661 
00662         "# .p2align 4                   \n\t"\
00663         "4:                             \n\t"
00664 Z_COND_IDCT(  64(%0), 72(%0), 80(%0), 88(%0), 64(%1),paddd (%2), 11, 6f)
00665 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 5f)
00666 
00667 #undef IDCT
00668 #define IDCT(src0, src4, src1, src5, dst, shift) \
00669         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00670         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00671         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00672         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00673         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00674         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00675         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00676         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00677         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00678         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00679         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00680         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00681         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00682         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00683         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00684         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
00685         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
00686         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
00687         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00688         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
00689         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00690         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
00691         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00692         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00693         "psrad $" #shift ", %%mm1       \n\t"\
00694         "psrad $" #shift ", %%mm4       \n\t"\
00695         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
00696         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00697         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00698         "psrad $" #shift ", %%mm0       \n\t"\
00699         "psrad $" #shift ", %%mm2       \n\t"\
00700         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
00701         "movd %%mm1, " #dst "           \n\t"\
00702         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00703         "movd %%mm0, 16+" #dst "        \n\t"\
00704         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
00705         "movd %%mm2, 96+" #dst "        \n\t"\
00706         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00707         "movd %%mm4, 112+" #dst "       \n\t"\
00708         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
00709         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00710         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
00711         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00712         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00713         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00714         "psrad $" #shift ", %%mm2       \n\t"\
00715         "psrad $" #shift ", %%mm5       \n\t"\
00716         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
00717         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00718         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
00719         "psrad $" #shift ", %%mm6       \n\t"\
00720         "psrad $" #shift ", %%mm1       \n\t"\
00721         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
00722         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00723         "movd %%mm2, 32+" #dst "        \n\t"\
00724         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
00725         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00726         "movd %%mm6, 48+" #dst "        \n\t"\
00727         "movd %%mm1, 64+" #dst "        \n\t"\
00728         "movd %%mm5, 80+" #dst "        \n\t"
00729 
00730 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
00731 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00732 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00733 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00734 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00735         "jmp 9f                         \n\t"
00736 
00737         "# .p2align 4                   \n\t"\
00738         "6:                             \n\t"
00739 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 7f)
00740 
00741 #undef IDCT
00742 #define IDCT(src0, src4, src1, src5, dst, shift) \
00743         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00744         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00745         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00746         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00747         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00748         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00749         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00750         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00751         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
00752         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00753         "movq 72(%2), %%mm7             \n\t" /* -C5    -C1     -C5     -C1 */\
00754         "pmaddwd %%mm3, %%mm7           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00755         "paddd %%mm4, %%mm1             \n\t" /* A0+B0          a0+b0 */\
00756         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00757         "psubd %%mm1, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00758         "psrad $" #shift ", %%mm1       \n\t"\
00759         "psrad $" #shift ", %%mm4       \n\t"\
00760         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
00761         "paddd %%mm7, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00762         "psubd %%mm7, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00763         "psrad $" #shift ", %%mm0       \n\t"\
00764         "psrad $" #shift ", %%mm2       \n\t"\
00765         "packssdw %%mm1, %%mm1          \n\t" /* A0+B0  a0+b0 */\
00766         "movd %%mm1, " #dst "           \n\t"\
00767         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00768         "movd %%mm0, 16+" #dst "        \n\t"\
00769         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
00770         "movd %%mm2, 96+" #dst "        \n\t"\
00771         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00772         "movd %%mm4, 112+" #dst "       \n\t"\
00773         "movq 88(%2), %%mm1             \n\t" /* C3     C7      C3      C7 */\
00774         "pmaddwd %%mm3, %%mm1           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00775         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
00776         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00777         "paddd %%mm1, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00778         "psubd %%mm1, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00779         "psrad $" #shift ", %%mm2       \n\t"\
00780         "psrad $" #shift ", %%mm5       \n\t"\
00781         "movq %%mm6, %%mm1              \n\t" /* A3             a3 */\
00782         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00783         "psubd %%mm3, %%mm1             \n\t" /* a3-B3          a3-b3 */\
00784         "psrad $" #shift ", %%mm6       \n\t"\
00785         "psrad $" #shift ", %%mm1       \n\t"\
00786         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
00787         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00788         "movd %%mm2, 32+" #dst "        \n\t"\
00789         "packssdw %%mm1, %%mm1          \n\t" /* A3-B3  a3-b3 */\
00790         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00791         "movd %%mm6, 48+" #dst "        \n\t"\
00792         "movd %%mm1, 64+" #dst "        \n\t"\
00793         "movd %%mm5, 80+" #dst "        \n\t"
00794 
00795 
00796 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
00797 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00798 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00799 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00800 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00801         "jmp 9f                         \n\t"
00802 
00803         "# .p2align 4                   \n\t"\
00804         "2:                             \n\t"
00805 Z_COND_IDCT(  96(%0),104(%0),112(%0),120(%0), 96(%1),paddd (%2), 11, 3f)
00806 
00807 #undef IDCT
00808 #define IDCT(src0, src4, src1, src5, dst, shift) \
00809         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00810         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00811         "movq " #src5 ", %%mm3          \n\t" /* R7     R5      r7      r5 */\
00812         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00813         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00814         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00815         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00816         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00817         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00818         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00819         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00820         "movq 56(%2), %%mm1             \n\t" /* C7     C5      C7      C5 */\
00821         "pmaddwd %%mm3, %%mm1           \n\t" /* C7R7+C5R5      C7r7+C5r5 */\
00822         "pmaddwd 64(%2), %%mm2          \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00823         "paddd %%mm1, %%mm7             \n\t" /* B0             b0 */\
00824         "movq 72(%2), %%mm1             \n\t" /* -C5    -C1     -C5     -C1 */\
00825         "pmaddwd %%mm3, %%mm1           \n\t" /* -C5R7-C1R5     -C5r7-C1r5 */\
00826         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00827         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00828         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00829         "paddd %%mm2, %%mm1             \n\t" /* B1             b1 */\
00830         "psrad $" #shift ", %%mm7       \n\t"\
00831         "psrad $" #shift ", %%mm4       \n\t"\
00832         "movq %%mm0, %%mm2              \n\t" /* A1             a1 */\
00833         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00834         "psubd %%mm1, %%mm2             \n\t" /* A1-B1          a1-b1 */\
00835         "psrad $" #shift ", %%mm0       \n\t"\
00836         "psrad $" #shift ", %%mm2       \n\t"\
00837         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
00838         "movd %%mm7, " #dst "           \n\t"\
00839         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00840         "movd %%mm0, 16+" #dst "        \n\t"\
00841         "packssdw %%mm2, %%mm2          \n\t" /* A1-B1  a1-b1 */\
00842         "movd %%mm2, 96+" #dst "        \n\t"\
00843         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00844         "movd %%mm4, 112+" #dst "       \n\t"\
00845         "movq " #src1 ", %%mm0          \n\t" /* R3     R1      r3      r1 */\
00846         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00847         "pmaddwd %%mm0, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00848         "movq 88(%2), %%mm7             \n\t" /* C3     C7      C3      C7 */\
00849         "pmaddwd 96(%2), %%mm0          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00850         "pmaddwd %%mm3, %%mm7           \n\t" /* C3R7+C7R5      C3r7+C7r5 */\
00851         "movq %%mm5, %%mm2              \n\t" /* A2             a2 */\
00852         "pmaddwd 104(%2), %%mm3         \n\t" /* -C1R7+C3R5     -C1r7+C3r5 */\
00853         "paddd %%mm7, %%mm4             \n\t" /* B2             b2 */\
00854         "paddd %%mm4, %%mm2             \n\t" /* A2+B2          a2+b2 */\
00855         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00856         "psrad $" #shift ", %%mm2       \n\t"\
00857         "psrad $" #shift ", %%mm5       \n\t"\
00858         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00859         "paddd %%mm0, %%mm3             \n\t" /* B3             b3 */\
00860         "paddd %%mm3, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00861         "psubd %%mm3, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00862         "psrad $" #shift ", %%mm6       \n\t"\
00863         "psrad $" #shift ", %%mm4       \n\t"\
00864         "packssdw %%mm2, %%mm2          \n\t" /* A2+B2  a2+b2 */\
00865         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00866         "movd %%mm2, 32+" #dst "        \n\t"\
00867         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
00868         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00869         "movd %%mm6, 48+" #dst "        \n\t"\
00870         "movd %%mm4, 64+" #dst "        \n\t"\
00871         "movd %%mm5, 80+" #dst "        \n\t"
00872 
00873 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
00874 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00875 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00876 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00877 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00878         "jmp 9f                         \n\t"
00879 
00880         "# .p2align 4                   \n\t"\
00881         "3:                             \n\t"
00882 #undef IDCT
00883 #define IDCT(src0, src4, src1, src5, dst, shift) \
00884         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00885         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
00886         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00887         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00888         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00889         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00890         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00891         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
00892         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
00893         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00894         "movq 64(%2), %%mm3             \n\t"\
00895         "pmaddwd %%mm2, %%mm3           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
00896         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
00897         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
00898         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
00899         "psrad $" #shift ", %%mm7       \n\t"\
00900         "psrad $" #shift ", %%mm4       \n\t"\
00901         "movq %%mm0, %%mm1              \n\t" /* A1             a1 */\
00902         "paddd %%mm3, %%mm0             \n\t" /* A1+B1          a1+b1 */\
00903         "psubd %%mm3, %%mm1             \n\t" /* A1-B1          a1-b1 */\
00904         "psrad $" #shift ", %%mm0       \n\t"\
00905         "psrad $" #shift ", %%mm1       \n\t"\
00906         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
00907         "movd %%mm7, " #dst "           \n\t"\
00908         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
00909         "movd %%mm0, 16+" #dst "        \n\t"\
00910         "packssdw %%mm1, %%mm1          \n\t" /* A1-B1  a1-b1 */\
00911         "movd %%mm1, 96+" #dst "        \n\t"\
00912         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
00913         "movd %%mm4, 112+" #dst "       \n\t"\
00914         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
00915         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
00916         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
00917         "movq %%mm5, %%mm1              \n\t" /* A2             a2 */\
00918         "paddd %%mm4, %%mm1             \n\t" /* A2+B2          a2+b2 */\
00919         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
00920         "psrad $" #shift ", %%mm1       \n\t"\
00921         "psrad $" #shift ", %%mm5       \n\t"\
00922         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
00923         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
00924         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
00925         "psrad $" #shift ", %%mm6       \n\t"\
00926         "psrad $" #shift ", %%mm4       \n\t"\
00927         "packssdw %%mm1, %%mm1          \n\t" /* A2+B2  a2+b2 */\
00928         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00929         "movd %%mm1, 32+" #dst "        \n\t"\
00930         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
00931         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00932         "movd %%mm6, 48+" #dst "        \n\t"\
00933         "movd %%mm4, 64+" #dst "        \n\t"\
00934         "movd %%mm5, 80+" #dst "        \n\t"
00935 
00936 
00937 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
00938 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
00939 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
00940 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
00941 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
00942         "jmp 9f                         \n\t"
00943 
00944         "# .p2align 4                   \n\t"\
00945         "5:                             \n\t"
00946 #undef IDCT
00947 #define IDCT(src0, src4, src1, src5, dst, shift) \
00948         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
00949         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
00950         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
00951         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00952         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
00953         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00954         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
00955         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00956         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
00957         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00958         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00959         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
00960         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
00961         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00962         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
00963         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
00964         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
00965         "movq 8+" #src4 ", %%mm3        \n\t" /* R6     R2      r6      r2 */\
00966         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
00967         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
00968         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
00969         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
00970         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
00971         "pmaddwd %%mm3, %%mm7           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
00972         "pmaddwd 40(%2), %%mm3          \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
00973         "paddd %%mm1, %%mm7             \n\t" /* A0             a0 */\
00974         "paddd %%mm1, %%mm1             \n\t" /* 2C0            2c0 */\
00975         "psubd %%mm7, %%mm1             \n\t" /* A3             a3 */\
00976         "paddd %%mm2, %%mm3             \n\t" /* A1             a1 */\
00977         "paddd %%mm2, %%mm2             \n\t" /* 2C1            2c1 */\
00978         "psubd %%mm3, %%mm2             \n\t" /* A2             a2 */\
00979         "psrad $" #shift ", %%mm4       \n\t"\
00980         "psrad $" #shift ", %%mm7       \n\t"\
00981         "psrad $" #shift ", %%mm3       \n\t"\
00982         "packssdw %%mm7, %%mm4          \n\t" /* A0     a0 */\
00983         "movq %%mm4, " #dst "           \n\t"\
00984         "psrad $" #shift ", %%mm0       \n\t"\
00985         "packssdw %%mm3, %%mm0          \n\t" /* A1     a1 */\
00986         "movq %%mm0, 16+" #dst "        \n\t"\
00987         "movq %%mm0, 96+" #dst "        \n\t"\
00988         "movq %%mm4, 112+" #dst "       \n\t"\
00989         "psrad $" #shift ", %%mm5       \n\t"\
00990         "psrad $" #shift ", %%mm6       \n\t"\
00991         "psrad $" #shift ", %%mm2       \n\t"\
00992         "packssdw %%mm2, %%mm5          \n\t" /* A2-B2  a2-b2 */\
00993         "movq %%mm5, 32+" #dst "        \n\t"\
00994         "psrad $" #shift ", %%mm1       \n\t"\
00995         "packssdw %%mm1, %%mm6          \n\t" /* A3+B3  a3+b3 */\
00996         "movq %%mm6, 48+" #dst "        \n\t"\
00997         "movq %%mm6, 64+" #dst "        \n\t"\
00998         "movq %%mm5, 80+" #dst "        \n\t"
00999 
01000 
01001 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
01002 IDCT(    0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
01003 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
01004 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
01005 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
01006         "jmp 9f                         \n\t"
01007 
01008 
01009         "# .p2align 4                   \n\t"\
01010         "1:                             \n\t"
01011 #undef IDCT
01012 #define IDCT(src0, src4, src1, src5, dst, shift) \
01013         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
01014         "movq " #src4 ", %%mm1          \n\t" /* R6     R2      r6      r2 */\
01015         "movq " #src1 ", %%mm2          \n\t" /* R3     R1      r3      r1 */\
01016         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
01017         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
01018         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
01019         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
01020         "movq 32(%2), %%mm5             \n\t" /* C6     C2      C6      C2 */\
01021         "pmaddwd %%mm1, %%mm5           \n\t" /* C6R6+C2R2      C6r6+C2r2 */\
01022         "movq 40(%2), %%mm6             \n\t" /* -C2    C6      -C2     C6 */\
01023         "pmaddwd %%mm6, %%mm1           \n\t" /* -C2R6+C6R2     -C2r6+C6r2 */\
01024         "movq %%mm4, %%mm6              \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
01025         "movq 48(%2), %%mm7             \n\t" /* C3     C1      C3      C1 */\
01026         "pmaddwd %%mm2, %%mm7           \n\t" /* C3R3+C1R1      C3r3+C1r1 */\
01027         "paddd %%mm5, %%mm4             \n\t" /* A0             a0 */\
01028         "psubd %%mm5, %%mm6             \n\t" /* A3             a3 */\
01029         "movq %%mm0, %%mm5              \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
01030         "paddd %%mm1, %%mm0             \n\t" /* A1             a1 */\
01031         "psubd %%mm1, %%mm5             \n\t" /* A2             a2 */\
01032         "movq 64(%2), %%mm1             \n\t"\
01033         "pmaddwd %%mm2, %%mm1           \n\t" /* -C7R3+C3R1     -C7r3+C3r1 */\
01034         "paddd %%mm4, %%mm7             \n\t" /* A0+B0          a0+b0 */\
01035         "paddd %%mm4, %%mm4             \n\t" /* 2A0            2a0 */\
01036         "psubd %%mm7, %%mm4             \n\t" /* A0-B0          a0-b0 */\
01037         "psrad $" #shift ", %%mm7       \n\t"\
01038         "psrad $" #shift ", %%mm4       \n\t"\
01039         "movq %%mm0, %%mm3              \n\t" /* A1             a1 */\
01040         "paddd %%mm1, %%mm0             \n\t" /* A1+B1          a1+b1 */\
01041         "psubd %%mm1, %%mm3             \n\t" /* A1-B1          a1-b1 */\
01042         "psrad $" #shift ", %%mm0       \n\t"\
01043         "psrad $" #shift ", %%mm3       \n\t"\
01044         "packssdw %%mm7, %%mm7          \n\t" /* A0+B0  a0+b0 */\
01045         "movd %%mm7, " #dst "           \n\t"\
01046         "packssdw %%mm0, %%mm0          \n\t" /* A1+B1  a1+b1 */\
01047         "movd %%mm0, 16+" #dst "        \n\t"\
01048         "packssdw %%mm3, %%mm3          \n\t" /* A1-B1  a1-b1 */\
01049         "movd %%mm3, 96+" #dst "        \n\t"\
01050         "packssdw %%mm4, %%mm4          \n\t" /* A0-B0  a0-b0 */\
01051         "movd %%mm4, 112+" #dst "       \n\t"\
01052         "movq 80(%2), %%mm4             \n\t" /* -C1    C5      -C1     C5 */\
01053         "pmaddwd %%mm2, %%mm4           \n\t" /* -C1R3+C5R1     -C1r3+C5r1 */\
01054         "pmaddwd 96(%2), %%mm2          \n\t" /* -C5R3+C7R1     -C5r3+C7r1 */\
01055         "movq %%mm5, %%mm3              \n\t" /* A2             a2 */\
01056         "paddd %%mm4, %%mm3             \n\t" /* A2+B2          a2+b2 */\
01057         "psubd %%mm4, %%mm5             \n\t" /* a2-B2          a2-b2 */\
01058         "psrad $" #shift ", %%mm3       \n\t"\
01059         "psrad $" #shift ", %%mm5       \n\t"\
01060         "movq %%mm6, %%mm4              \n\t" /* A3             a3 */\
01061         "paddd %%mm2, %%mm6             \n\t" /* A3+B3          a3+b3 */\
01062         "psubd %%mm2, %%mm4             \n\t" /* a3-B3          a3-b3 */\
01063         "psrad $" #shift ", %%mm6       \n\t"\
01064         "packssdw %%mm3, %%mm3          \n\t" /* A2+B2  a2+b2 */\
01065         "movd %%mm3, 32+" #dst "        \n\t"\
01066         "psrad $" #shift ", %%mm4       \n\t"\
01067         "packssdw %%mm6, %%mm6          \n\t" /* A3+B3  a3+b3 */\
01068         "movd %%mm6, 48+" #dst "        \n\t"\
01069         "packssdw %%mm4, %%mm4          \n\t" /* A3-B3  a3-b3 */\
01070         "packssdw %%mm5, %%mm5          \n\t" /* A2-B2  a2-b2 */\
01071         "movd %%mm4, 64+" #dst "        \n\t"\
01072         "movd %%mm5, 80+" #dst "        \n\t"
01073 
01074 
01075 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
01076 IDCT(    (%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
01077 IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
01078 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
01079 IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
01080         "jmp 9f                         \n\t"
01081 
01082 
01083         "# .p2align 4                   \n\t"
01084         "7:                             \n\t"
01085 #undef IDCT
01086 #define IDCT(src0, src4, src1, src5, dst, shift) \
01087         "movq " #src0 ", %%mm0          \n\t" /* R4     R0      r4      r0 */\
01088         "movq 16(%2), %%mm4             \n\t" /* C4     C4      C4      C4 */\
01089         "pmaddwd %%mm0, %%mm4           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
01090         "movq 24(%2), %%mm5             \n\t" /* -C4    C4      -C4     C4 */\
01091         "pmaddwd %%mm5, %%mm0           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
01092         "psrad $" #shift ", %%mm4       \n\t"\
01093         "psrad $" #shift ", %%mm0       \n\t"\
01094         "movq 8+" #src0 ", %%mm2        \n\t" /* R4     R0      r4      r0 */\
01095         "movq 16(%2), %%mm1             \n\t" /* C4     C4      C4      C4 */\
01096         "pmaddwd %%mm2, %%mm1           \n\t" /* C4R4+C4R0      C4r4+C4r0 */\
01097         "movq 24(%2), %%mm7             \n\t" /* -C4    C4      -C4     C4 */\
01098         "pmaddwd %%mm7, %%mm2           \n\t" /* -C4R4+C4R0     -C4r4+C4r0 */\
01099         "movq 32(%2), %%mm7             \n\t" /* C6     C2      C6      C2 */\
01100         "psrad $" #shift ", %%mm1       \n\t"\
01101         "packssdw %%mm1, %%mm4          \n\t" /* A0     a0 */\
01102         "movq %%mm4, " #dst "           \n\t"\
01103         "psrad $" #shift ", %%mm2       \n\t"\
01104         "packssdw %%mm2, %%mm0          \n\t" /* A1     a1 */\
01105         "movq %%mm0, 16+" #dst "        \n\t"\
01106         "movq %%mm0, 96+" #dst "        \n\t"\
01107         "movq %%mm4, 112+" #dst "       \n\t"\
01108         "movq %%mm0, 32+" #dst "        \n\t"\
01109         "movq %%mm4, 48+" #dst "        \n\t"\
01110         "movq %%mm4, 64+" #dst "        \n\t"\
01111         "movq %%mm0, 80+" #dst "        \n\t"
01112 
01113 //IDCT(  src0,   src4,   src1,    src5,    dst, shift)
01114 IDCT(   0(%1), 64(%1), 32(%1),  96(%1),  0(%0), 20)
01115 //IDCT(   8(%1), 72(%1), 40(%1), 104(%1),  4(%0), 20)
01116 IDCT(  16(%1), 80(%1), 48(%1), 112(%1),  8(%0), 20)
01117 //IDCT(  24(%1), 88(%1), 56(%1), 120(%1), 12(%0), 20)
01118 
01119 
01120 #endif
01121 
01122 /*
01123 Input
01124  00 40 04 44 20 60 24 64
01125  10 30 14 34 50 70 54 74
01126  01 41 03 43 21 61 23 63
01127  11 31 13 33 51 71 53 73
01128  02 42 06 46 22 62 26 66
01129  12 32 16 36 52 72 56 76
01130  05 45 07 47 25 65 27 67
01131  15 35 17 37 55 75 57 77
01132 
01133 Temp
01134  00 04 10 14 20 24 30 34
01135  40 44 50 54 60 64 70 74
01136  01 03 11 13 21 23 31 33
01137  41 43 51 53 61 63 71 73
01138  02 06 12 16 22 26 32 36
01139  42 46 52 56 62 66 72 76
01140  05 07 15 17 25 27 35 37
01141  45 47 55 57 65 67 75 77
01142 */
01143 
01144 "9: \n\t"
01145                 :: "r" (block), "r" (temp), "r" (coeffs)
01146                 : "%eax"
01147         );
01148 }
01149 
01150 void ff_simple_idct_mmx(int16_t *block)
01151 {
01152     idct(block);
01153 }
01154 
01155 //FIXME merge add/put into the idct
01156 
01157 void ff_simple_idct_put_mmx(uint8_t *dest, int line_size, DCTELEM *block)
01158 {
01159     idct(block);
01160     ff_put_pixels_clamped_mmx(block, dest, line_size);
01161 }
01162 void ff_simple_idct_add_mmx(uint8_t *dest, int line_size, DCTELEM *block)
01163 {
01164     idct(block);
01165     ff_add_pixels_clamped_mmx(block, dest, line_size);
01166 }
01167 
01168 #endif /* HAVE_INLINE_ASM */
libavcodec/x86/simple_idct.c
