FFmpeg: libavcodec/x86/h264dsp

00001 /*
00002  * Copyright (c) 2004-2005 Michael Niedermayer, Loren Merritt
00003  *
00004  * This file is part of FFmpeg.
00005  *
00006  * FFmpeg is free software; you can redistribute it and/or
00007  * modify it under the terms of the GNU Lesser General Public
00008  * License as published by the Free Software Foundation; either
00009  * version 2.1 of the License, or (at your option) any later version.
00010  *
00011  * FFmpeg is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00014  * Lesser General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU Lesser General Public
00017  * License along with FFmpeg; if not, write to the Free Software
00018  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
00019  */
00020 
00021 #include "dsputil_mmx.h"
00022 
00023 DECLARE_ALIGNED(8, static const uint64_t, ff_pb_3_1  ) = 0x0103010301030103ULL;
00024 DECLARE_ALIGNED(8, static const uint64_t, ff_pb_7_3  ) = 0x0307030703070307ULL;
00025 
00026 /***********************************/
00027 /* IDCT */
00028 
00029 #define SUMSUB_BADC( a, b, c, d ) \
00030     "paddw "#b", "#a" \n\t"\
00031     "paddw "#d", "#c" \n\t"\
00032     "paddw "#b", "#b" \n\t"\
00033     "paddw "#d", "#d" \n\t"\
00034     "psubw "#a", "#b" \n\t"\
00035     "psubw "#c", "#d" \n\t"
00036 
00037 #define SUMSUBD2_AB( a, b, t ) \
00038     "movq  "#b", "#t" \n\t"\
00039     "psraw  $1 , "#b" \n\t"\
00040     "paddw "#a", "#b" \n\t"\
00041     "psraw  $1 , "#a" \n\t"\
00042     "psubw "#t", "#a" \n\t"
00043 
00044 #define IDCT4_1D( s02, s13, d02, d13, t ) \
00045     SUMSUB_BA  ( s02, d02 )\
00046     SUMSUBD2_AB( s13, d13, t )\
00047     SUMSUB_BADC( d13, s02, s13, d02 )
00048 
00049 #define STORE_DIFF_4P( p, t, z ) \
00050     "psraw      $6,     "#p" \n\t"\
00051     "movd       (%0),   "#t" \n\t"\
00052     "punpcklbw "#z",    "#t" \n\t"\
00053     "paddsw    "#t",    "#p" \n\t"\
00054     "packuswb  "#z",    "#p" \n\t"\
00055     "movd      "#p",    (%0) \n\t"
00056 
00057 static void ff_h264_idct_add_mmx(uint8_t *dst, int16_t *block, int stride)
00058 {
00059     /* Load dct coeffs */
00060     __asm__ volatile(
00061         "movq   (%0), %%mm0 \n\t"
00062         "movq  8(%0), %%mm1 \n\t"
00063         "movq 16(%0), %%mm2 \n\t"
00064         "movq 24(%0), %%mm3 \n\t"
00065     :: "r"(block) );
00066 
00067     __asm__ volatile(
00068         /* mm1=s02+s13  mm2=s02-s13  mm4=d02+d13  mm0=d02-d13 */
00069         IDCT4_1D( %%mm2, %%mm1, %%mm0, %%mm3, %%mm4 )
00070 
00071         "movq      %0,    %%mm6 \n\t"
00072         /* in: 1,4,0,2  out: 1,2,3,0 */
00073         TRANSPOSE4( %%mm3, %%mm1, %%mm0, %%mm2, %%mm4 )
00074 
00075         "paddw     %%mm6, %%mm3 \n\t"
00076 
00077         /* mm2=s02+s13  mm3=s02-s13  mm4=d02+d13  mm1=d02-d13 */
00078         IDCT4_1D( %%mm4, %%mm2, %%mm3, %%mm0, %%mm1 )
00079 
00080         "pxor %%mm7, %%mm7    \n\t"
00081     :: "m"(ff_pw_32));
00082 
00083     __asm__ volatile(
00084     STORE_DIFF_4P( %%mm0, %%mm1, %%mm7)
00085         "add %1, %0             \n\t"
00086     STORE_DIFF_4P( %%mm2, %%mm1, %%mm7)
00087         "add %1, %0             \n\t"
00088     STORE_DIFF_4P( %%mm3, %%mm1, %%mm7)
00089         "add %1, %0             \n\t"
00090     STORE_DIFF_4P( %%mm4, %%mm1, %%mm7)
00091         : "+r"(dst)
00092         : "r" ((x86_reg)stride)
00093     );
00094 }
00095 
00096 static inline void h264_idct8_1d(int16_t *block)
00097 {
00098     __asm__ volatile(
00099         "movq 112(%0), %%mm7  \n\t"
00100         "movq  80(%0), %%mm0  \n\t"
00101         "movq  48(%0), %%mm3  \n\t"
00102         "movq  16(%0), %%mm5  \n\t"
00103 
00104         "movq   %%mm0, %%mm4  \n\t"
00105         "movq   %%mm5, %%mm1  \n\t"
00106         "psraw  $1,    %%mm4  \n\t"
00107         "psraw  $1,    %%mm1  \n\t"
00108         "paddw  %%mm0, %%mm4  \n\t"
00109         "paddw  %%mm5, %%mm1  \n\t"
00110         "paddw  %%mm7, %%mm4  \n\t"
00111         "paddw  %%mm0, %%mm1  \n\t"
00112         "psubw  %%mm5, %%mm4  \n\t"
00113         "paddw  %%mm3, %%mm1  \n\t"
00114 
00115         "psubw  %%mm3, %%mm5  \n\t"
00116         "psubw  %%mm3, %%mm0  \n\t"
00117         "paddw  %%mm7, %%mm5  \n\t"
00118         "psubw  %%mm7, %%mm0  \n\t"
00119         "psraw  $1,    %%mm3  \n\t"
00120         "psraw  $1,    %%mm7  \n\t"
00121         "psubw  %%mm3, %%mm5  \n\t"
00122         "psubw  %%mm7, %%mm0  \n\t"
00123 
00124         "movq   %%mm4, %%mm3  \n\t"
00125         "movq   %%mm1, %%mm7  \n\t"
00126         "psraw  $2,    %%mm1  \n\t"
00127         "psraw  $2,    %%mm3  \n\t"
00128         "paddw  %%mm5, %%mm3  \n\t"
00129         "psraw  $2,    %%mm5  \n\t"
00130         "paddw  %%mm0, %%mm1  \n\t"
00131         "psraw  $2,    %%mm0  \n\t"
00132         "psubw  %%mm4, %%mm5  \n\t"
00133         "psubw  %%mm0, %%mm7  \n\t"
00134 
00135         "movq  32(%0), %%mm2  \n\t"
00136         "movq  96(%0), %%mm6  \n\t"
00137         "movq   %%mm2, %%mm4  \n\t"
00138         "movq   %%mm6, %%mm0  \n\t"
00139         "psraw  $1,    %%mm4  \n\t"
00140         "psraw  $1,    %%mm6  \n\t"
00141         "psubw  %%mm0, %%mm4  \n\t"
00142         "paddw  %%mm2, %%mm6  \n\t"
00143 
00144         "movq    (%0), %%mm2  \n\t"
00145         "movq  64(%0), %%mm0  \n\t"
00146         SUMSUB_BA( %%mm0, %%mm2 )
00147         SUMSUB_BA( %%mm6, %%mm0 )
00148         SUMSUB_BA( %%mm4, %%mm2 )
00149         SUMSUB_BA( %%mm7, %%mm6 )
00150         SUMSUB_BA( %%mm5, %%mm4 )
00151         SUMSUB_BA( %%mm3, %%mm2 )
00152         SUMSUB_BA( %%mm1, %%mm0 )
00153         :: "r"(block)
00154     );
00155 }
00156 
00157 static void ff_h264_idct8_add_mmx(uint8_t *dst, int16_t *block, int stride)
00158 {
00159     int i;
00160     DECLARE_ALIGNED(8, int16_t, b2)[64];
00161 
00162     block[0] += 32;
00163 
00164     for(i=0; i<2; i++){
00165         DECLARE_ALIGNED(8, uint64_t, tmp);
00166 
00167         h264_idct8_1d(block+4*i);
00168 
00169         __asm__ volatile(
00170             "movq   %%mm7,    %0   \n\t"
00171             TRANSPOSE4( %%mm0, %%mm2, %%mm4, %%mm6, %%mm7 )
00172             "movq   %%mm0,  8(%1)  \n\t"
00173             "movq   %%mm6, 24(%1)  \n\t"
00174             "movq   %%mm7, 40(%1)  \n\t"
00175             "movq   %%mm4, 56(%1)  \n\t"
00176             "movq    %0,    %%mm7  \n\t"
00177             TRANSPOSE4( %%mm7, %%mm5, %%mm3, %%mm1, %%mm0 )
00178             "movq   %%mm7,   (%1)  \n\t"
00179             "movq   %%mm1, 16(%1)  \n\t"
00180             "movq   %%mm0, 32(%1)  \n\t"
00181             "movq   %%mm3, 48(%1)  \n\t"
00182             : "=m"(tmp)
00183             : "r"(b2+32*i)
00184             : "memory"
00185         );
00186     }
00187 
00188     for(i=0; i<2; i++){
00189         h264_idct8_1d(b2+4*i);
00190 
00191         __asm__ volatile(
00192             "psraw     $6, %%mm7  \n\t"
00193             "psraw     $6, %%mm6  \n\t"
00194             "psraw     $6, %%mm5  \n\t"
00195             "psraw     $6, %%mm4  \n\t"
00196             "psraw     $6, %%mm3  \n\t"
00197             "psraw     $6, %%mm2  \n\t"
00198             "psraw     $6, %%mm1  \n\t"
00199             "psraw     $6, %%mm0  \n\t"
00200 
00201             "movq   %%mm7,    (%0)  \n\t"
00202             "movq   %%mm5,  16(%0)  \n\t"
00203             "movq   %%mm3,  32(%0)  \n\t"
00204             "movq   %%mm1,  48(%0)  \n\t"
00205             "movq   %%mm0,  64(%0)  \n\t"
00206             "movq   %%mm2,  80(%0)  \n\t"
00207             "movq   %%mm4,  96(%0)  \n\t"
00208             "movq   %%mm6, 112(%0)  \n\t"
00209             :: "r"(b2+4*i)
00210             : "memory"
00211         );
00212     }
00213 
00214     add_pixels_clamped_mmx(b2, dst, stride);
00215 }
00216 
00217 #define STORE_DIFF_8P( p, d, t, z )\
00218         "movq       "#d", "#t" \n"\
00219         "psraw       $6,  "#p" \n"\
00220         "punpcklbw  "#z", "#t" \n"\
00221         "paddsw     "#t", "#p" \n"\
00222         "packuswb   "#p", "#p" \n"\
00223         "movq       "#p", "#d" \n"
00224 
00225 #define H264_IDCT8_1D_SSE2(a,b,c,d,e,f,g,h)\
00226         "movdqa     "#c", "#a" \n"\
00227         "movdqa     "#g", "#e" \n"\
00228         "psraw       $1,  "#c" \n"\
00229         "psraw       $1,  "#g" \n"\
00230         "psubw      "#e", "#c" \n"\
00231         "paddw      "#a", "#g" \n"\
00232         "movdqa     "#b", "#e" \n"\
00233         "psraw       $1,  "#e" \n"\
00234         "paddw      "#b", "#e" \n"\
00235         "paddw      "#d", "#e" \n"\
00236         "paddw      "#f", "#e" \n"\
00237         "movdqa     "#f", "#a" \n"\
00238         "psraw       $1,  "#a" \n"\
00239         "paddw      "#f", "#a" \n"\
00240         "paddw      "#h", "#a" \n"\
00241         "psubw      "#b", "#a" \n"\
00242         "psubw      "#d", "#b" \n"\
00243         "psubw      "#d", "#f" \n"\
00244         "paddw      "#h", "#b" \n"\
00245         "psubw      "#h", "#f" \n"\
00246         "psraw       $1,  "#d" \n"\
00247         "psraw       $1,  "#h" \n"\
00248         "psubw      "#d", "#b" \n"\
00249         "psubw      "#h", "#f" \n"\
00250         "movdqa     "#e", "#d" \n"\
00251         "movdqa     "#a", "#h" \n"\
00252         "psraw       $2,  "#d" \n"\
00253         "psraw       $2,  "#h" \n"\
00254         "paddw      "#f", "#d" \n"\
00255         "paddw      "#b", "#h" \n"\
00256         "psraw       $2,  "#f" \n"\
00257         "psraw       $2,  "#b" \n"\
00258         "psubw      "#f", "#e" \n"\
00259         "psubw      "#a", "#b" \n"\
00260         "movdqa 0x00(%1), "#a" \n"\
00261         "movdqa 0x40(%1), "#f" \n"\
00262         SUMSUB_BA(f, a)\
00263         SUMSUB_BA(g, f)\
00264         SUMSUB_BA(c, a)\
00265         SUMSUB_BA(e, g)\
00266         SUMSUB_BA(b, c)\
00267         SUMSUB_BA(h, a)\
00268         SUMSUB_BA(d, f)
00269 
00270 static void ff_h264_idct8_add_sse2(uint8_t *dst, int16_t *block, int stride)
00271 {
00272     __asm__ volatile(
00273         "movdqa   0x10(%1), %%xmm1 \n"
00274         "movdqa   0x20(%1), %%xmm2 \n"
00275         "movdqa   0x30(%1), %%xmm3 \n"
00276         "movdqa   0x50(%1), %%xmm5 \n"
00277         "movdqa   0x60(%1), %%xmm6 \n"
00278         "movdqa   0x70(%1), %%xmm7 \n"
00279         H264_IDCT8_1D_SSE2(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm6, %%xmm7)
00280         TRANSPOSE8(%%xmm4, %%xmm1, %%xmm7, %%xmm3, %%xmm5, %%xmm0, %%xmm2, %%xmm6, (%1))
00281         "paddw          %4, %%xmm4 \n"
00282         "movdqa     %%xmm4, 0x00(%1) \n"
00283         "movdqa     %%xmm2, 0x40(%1) \n"
00284         H264_IDCT8_1D_SSE2(%%xmm4, %%xmm0, %%xmm6, %%xmm3, %%xmm2, %%xmm5, %%xmm7, %%xmm1)
00285         "movdqa     %%xmm6, 0x60(%1) \n"
00286         "movdqa     %%xmm7, 0x70(%1) \n"
00287         "pxor       %%xmm7, %%xmm7 \n"
00288         STORE_DIFF_8P(%%xmm2, (%0),      %%xmm6, %%xmm7)
00289         STORE_DIFF_8P(%%xmm0, (%0,%2),   %%xmm6, %%xmm7)
00290         STORE_DIFF_8P(%%xmm1, (%0,%2,2), %%xmm6, %%xmm7)
00291         STORE_DIFF_8P(%%xmm3, (%0,%3),   %%xmm6, %%xmm7)
00292         "lea     (%0,%2,4), %0 \n"
00293         STORE_DIFF_8P(%%xmm5, (%0),      %%xmm6, %%xmm7)
00294         STORE_DIFF_8P(%%xmm4, (%0,%2),   %%xmm6, %%xmm7)
00295         "movdqa   0x60(%1), %%xmm0 \n"
00296         "movdqa   0x70(%1), %%xmm1 \n"
00297         STORE_DIFF_8P(%%xmm0, (%0,%2,2), %%xmm6, %%xmm7)
00298         STORE_DIFF_8P(%%xmm1, (%0,%3),   %%xmm6, %%xmm7)
00299         :"+r"(dst)
00300         :"r"(block), "r"((x86_reg)stride), "r"((x86_reg)3L*stride), "m"(ff_pw_32)
00301     );
00302 }
00303 
00304 static void ff_h264_idct_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)
00305 {
00306     int dc = (block[0] + 32) >> 6;
00307     __asm__ volatile(
00308         "movd          %0, %%mm0 \n\t"
00309         "pshufw $0, %%mm0, %%mm0 \n\t"
00310         "pxor       %%mm1, %%mm1 \n\t"
00311         "psubw      %%mm0, %%mm1 \n\t"
00312         "packuswb   %%mm0, %%mm0 \n\t"
00313         "packuswb   %%mm1, %%mm1 \n\t"
00314         ::"r"(dc)
00315     );
00316     __asm__ volatile(
00317         "movd          %0, %%mm2 \n\t"
00318         "movd          %1, %%mm3 \n\t"
00319         "movd          %2, %%mm4 \n\t"
00320         "movd          %3, %%mm5 \n\t"
00321         "paddusb    %%mm0, %%mm2 \n\t"
00322         "paddusb    %%mm0, %%mm3 \n\t"
00323         "paddusb    %%mm0, %%mm4 \n\t"
00324         "paddusb    %%mm0, %%mm5 \n\t"
00325         "psubusb    %%mm1, %%mm2 \n\t"
00326         "psubusb    %%mm1, %%mm3 \n\t"
00327         "psubusb    %%mm1, %%mm4 \n\t"
00328         "psubusb    %%mm1, %%mm5 \n\t"
00329         "movd       %%mm2, %0    \n\t"
00330         "movd       %%mm3, %1    \n\t"
00331         "movd       %%mm4, %2    \n\t"
00332         "movd       %%mm5, %3    \n\t"
00333         :"+m"(*(uint32_t*)(dst+0*stride)),
00334          "+m"(*(uint32_t*)(dst+1*stride)),
00335          "+m"(*(uint32_t*)(dst+2*stride)),
00336          "+m"(*(uint32_t*)(dst+3*stride))
00337     );
00338 }
00339 
00340 static void ff_h264_idct8_dc_add_mmx2(uint8_t *dst, int16_t *block, int stride)
00341 {
00342     int dc = (block[0] + 32) >> 6;
00343     int y;
00344     __asm__ volatile(
00345         "movd          %0, %%mm0 \n\t"
00346         "pshufw $0, %%mm0, %%mm0 \n\t"
00347         "pxor       %%mm1, %%mm1 \n\t"
00348         "psubw      %%mm0, %%mm1 \n\t"
00349         "packuswb   %%mm0, %%mm0 \n\t"
00350         "packuswb   %%mm1, %%mm1 \n\t"
00351         ::"r"(dc)
00352     );
00353     for(y=2; y--; dst += 4*stride){
00354     __asm__ volatile(
00355         "movq          %0, %%mm2 \n\t"
00356         "movq          %1, %%mm3 \n\t"
00357         "movq          %2, %%mm4 \n\t"
00358         "movq          %3, %%mm5 \n\t"
00359         "paddusb    %%mm0, %%mm2 \n\t"
00360         "paddusb    %%mm0, %%mm3 \n\t"
00361         "paddusb    %%mm0, %%mm4 \n\t"
00362         "paddusb    %%mm0, %%mm5 \n\t"
00363         "psubusb    %%mm1, %%mm2 \n\t"
00364         "psubusb    %%mm1, %%mm3 \n\t"
00365         "psubusb    %%mm1, %%mm4 \n\t"
00366         "psubusb    %%mm1, %%mm5 \n\t"
00367         "movq       %%mm2, %0    \n\t"
00368         "movq       %%mm3, %1    \n\t"
00369         "movq       %%mm4, %2    \n\t"
00370         "movq       %%mm5, %3    \n\t"
00371         :"+m"(*(uint64_t*)(dst+0*stride)),
00372          "+m"(*(uint64_t*)(dst+1*stride)),
00373          "+m"(*(uint64_t*)(dst+2*stride)),
00374          "+m"(*(uint64_t*)(dst+3*stride))
00375     );
00376     }
00377 }
00378 
00379 //FIXME this table is a duplicate from h264data.h, and will be removed once the tables from, h264 have been split
00380 static const uint8_t scan8[16 + 2*4]={
00381  4+1*8, 5+1*8, 4+2*8, 5+2*8,
00382  6+1*8, 7+1*8, 6+2*8, 7+2*8,
00383  4+3*8, 5+3*8, 4+4*8, 5+4*8,
00384  6+3*8, 7+3*8, 6+4*8, 7+4*8,
00385  1+1*8, 2+1*8,
00386  1+2*8, 2+2*8,
00387  1+4*8, 2+4*8,
00388  1+5*8, 2+5*8,
00389 };
00390 
00391 static void ff_h264_idct_add16_mmx(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00392     int i;
00393     for(i=0; i<16; i++){
00394         if(nnzc[ scan8[i] ])
00395             ff_h264_idct_add_mmx(dst + block_offset[i], block + i*16, stride);
00396     }
00397 }
00398 
00399 static void ff_h264_idct8_add4_mmx(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00400     int i;
00401     for(i=0; i<16; i+=4){
00402         if(nnzc[ scan8[i] ])
00403             ff_h264_idct8_add_mmx(dst + block_offset[i], block + i*16, stride);
00404     }
00405 }
00406 
00407 
00408 static void ff_h264_idct_add16_mmx2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00409     int i;
00410     for(i=0; i<16; i++){
00411         int nnz = nnzc[ scan8[i] ];
00412         if(nnz){
00413             if(nnz==1 && block[i*16]) ff_h264_idct_dc_add_mmx2(dst + block_offset[i], block + i*16, stride);
00414             else                      ff_h264_idct_add_mmx    (dst + block_offset[i], block + i*16, stride);
00415         }
00416     }
00417 }
00418 
00419 static void ff_h264_idct_add16intra_mmx(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00420     int i;
00421     for(i=0; i<16; i++){
00422         if(nnzc[ scan8[i] ] || block[i*16])
00423             ff_h264_idct_add_mmx(dst + block_offset[i], block + i*16, stride);
00424     }
00425 }
00426 
00427 static void ff_h264_idct_add16intra_mmx2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00428     int i;
00429     for(i=0; i<16; i++){
00430         if(nnzc[ scan8[i] ]) ff_h264_idct_add_mmx    (dst + block_offset[i], block + i*16, stride);
00431         else if(block[i*16]) ff_h264_idct_dc_add_mmx2(dst + block_offset[i], block + i*16, stride);
00432     }
00433 }
00434 
00435 static void ff_h264_idct8_add4_mmx2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00436     int i;
00437     for(i=0; i<16; i+=4){
00438         int nnz = nnzc[ scan8[i] ];
00439         if(nnz){
00440             if(nnz==1 && block[i*16]) ff_h264_idct8_dc_add_mmx2(dst + block_offset[i], block + i*16, stride);
00441             else                      ff_h264_idct8_add_mmx    (dst + block_offset[i], block + i*16, stride);
00442         }
00443     }
00444 }
00445 
00446 static void ff_h264_idct8_add4_sse2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00447     int i;
00448     for(i=0; i<16; i+=4){
00449         int nnz = nnzc[ scan8[i] ];
00450         if(nnz){
00451             if(nnz==1 && block[i*16]) ff_h264_idct8_dc_add_mmx2(dst + block_offset[i], block + i*16, stride);
00452             else                      ff_h264_idct8_add_sse2   (dst + block_offset[i], block + i*16, stride);
00453         }
00454     }
00455 }
00456 
00457 static void ff_h264_idct_add8_mmx(uint8_t **dest, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00458     int i;
00459     for(i=16; i<16+8; i++){
00460         if(nnzc[ scan8[i] ] || block[i*16])
00461             ff_h264_idct_add_mmx    (dest[(i&4)>>2] + block_offset[i], block + i*16, stride);
00462     }
00463 }
00464 
00465 static void ff_h264_idct_add8_mmx2(uint8_t **dest, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00466     int i;
00467     for(i=16; i<16+8; i++){
00468         if(nnzc[ scan8[i] ])
00469             ff_h264_idct_add_mmx    (dest[(i&4)>>2] + block_offset[i], block + i*16, stride);
00470         else if(block[i*16])
00471             ff_h264_idct_dc_add_mmx2(dest[(i&4)>>2] + block_offset[i], block + i*16, stride);
00472     }
00473 }
00474 
00475 #if CONFIG_GPL && HAVE_YASM
00476 static void ff_h264_idct_dc_add8_mmx2(uint8_t *dst, int16_t *block, int stride)
00477 {
00478     __asm__ volatile(
00479         "movd             %0, %%mm0 \n\t"   //  0 0 X D
00480         "punpcklwd        %1, %%mm0 \n\t"   //  x X d D
00481         "paddsw           %2, %%mm0 \n\t"
00482         "psraw            $6, %%mm0 \n\t"
00483         "punpcklwd     %%mm0, %%mm0 \n\t"   //  d d D D
00484         "pxor          %%mm1, %%mm1 \n\t"   //  0 0 0 0
00485         "psubw         %%mm0, %%mm1 \n\t"   // -d-d-D-D
00486         "packuswb      %%mm1, %%mm0 \n\t"   // -d-d-D-D d d D D
00487         "pshufw $0xFA, %%mm0, %%mm1 \n\t"   // -d-d-d-d-D-D-D-D
00488         "punpcklwd     %%mm0, %%mm0 \n\t"   //  d d d d D D D D
00489         ::"m"(block[ 0]),
00490           "m"(block[16]),
00491           "m"(ff_pw_32)
00492     );
00493     __asm__ volatile(
00494         "movq          %0, %%mm2 \n\t"
00495         "movq          %1, %%mm3 \n\t"
00496         "movq          %2, %%mm4 \n\t"
00497         "movq          %3, %%mm5 \n\t"
00498         "paddusb    %%mm0, %%mm2 \n\t"
00499         "paddusb    %%mm0, %%mm3 \n\t"
00500         "paddusb    %%mm0, %%mm4 \n\t"
00501         "paddusb    %%mm0, %%mm5 \n\t"
00502         "psubusb    %%mm1, %%mm2 \n\t"
00503         "psubusb    %%mm1, %%mm3 \n\t"
00504         "psubusb    %%mm1, %%mm4 \n\t"
00505         "psubusb    %%mm1, %%mm5 \n\t"
00506         "movq       %%mm2, %0    \n\t"
00507         "movq       %%mm3, %1    \n\t"
00508         "movq       %%mm4, %2    \n\t"
00509         "movq       %%mm5, %3    \n\t"
00510         :"+m"(*(uint64_t*)(dst+0*stride)),
00511          "+m"(*(uint64_t*)(dst+1*stride)),
00512          "+m"(*(uint64_t*)(dst+2*stride)),
00513          "+m"(*(uint64_t*)(dst+3*stride))
00514     );
00515 }
00516 
00517 extern void ff_x264_add8x4_idct_sse2(uint8_t *dst, int16_t *block, int stride);
00518 
00519 static void ff_h264_idct_add16_sse2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00520     int i;
00521     for(i=0; i<16; i+=2)
00522         if(nnzc[ scan8[i+0] ]|nnzc[ scan8[i+1] ])
00523             ff_x264_add8x4_idct_sse2 (dst + block_offset[i], block + i*16, stride);
00524 }
00525 
00526 static void ff_h264_idct_add16intra_sse2(uint8_t *dst, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00527     int i;
00528     for(i=0; i<16; i+=2){
00529         if(nnzc[ scan8[i+0] ]|nnzc[ scan8[i+1] ])
00530             ff_x264_add8x4_idct_sse2 (dst + block_offset[i], block + i*16, stride);
00531         else if(block[i*16]|block[i*16+16])
00532             ff_h264_idct_dc_add8_mmx2(dst + block_offset[i], block + i*16, stride);
00533     }
00534 }
00535 
00536 static void ff_h264_idct_add8_sse2(uint8_t **dest, const int *block_offset, DCTELEM *block, int stride, const uint8_t nnzc[6*8]){
00537     int i;
00538     for(i=16; i<16+8; i+=2){
00539         if(nnzc[ scan8[i+0] ]|nnzc[ scan8[i+1] ])
00540             ff_x264_add8x4_idct_sse2 (dest[(i&4)>>2] + block_offset[i], block + i*16, stride);
00541         else if(block[i*16]|block[i*16+16])
00542             ff_h264_idct_dc_add8_mmx2(dest[(i&4)>>2] + block_offset[i], block + i*16, stride);
00543     }
00544 }
00545 #endif
00546 
00547 /***********************************/
00548 /* deblocking */
00549 
00550 // out: o = |x-y|>a
00551 // clobbers: t
00552 #define DIFF_GT_MMX(x,y,a,o,t)\
00553     "movq     "#y", "#t"  \n\t"\
00554     "movq     "#x", "#o"  \n\t"\
00555     "psubusb  "#x", "#t"  \n\t"\
00556     "psubusb  "#y", "#o"  \n\t"\
00557     "por      "#t", "#o"  \n\t"\
00558     "psubusb  "#a", "#o"  \n\t"
00559 
00560 // out: o = |x-y|>a
00561 // clobbers: t
00562 #define DIFF_GT2_MMX(x,y,a,o,t)\
00563     "movq     "#y", "#t"  \n\t"\
00564     "movq     "#x", "#o"  \n\t"\
00565     "psubusb  "#x", "#t"  \n\t"\
00566     "psubusb  "#y", "#o"  \n\t"\
00567     "psubusb  "#a", "#t"  \n\t"\
00568     "psubusb  "#a", "#o"  \n\t"\
00569     "pcmpeqb  "#t", "#o"  \n\t"\
00570 
00571 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1
00572 // out: mm5=beta-1, mm7=mask
00573 // clobbers: mm4,mm6
00574 #define H264_DEBLOCK_MASK(alpha1, beta1) \
00575     "pshufw $0, "#alpha1", %%mm4 \n\t"\
00576     "pshufw $0, "#beta1 ", %%mm5 \n\t"\
00577     "packuswb  %%mm4, %%mm4      \n\t"\
00578     "packuswb  %%mm5, %%mm5      \n\t"\
00579     DIFF_GT_MMX(%%mm1, %%mm2, %%mm4, %%mm7, %%mm6) /* |p0-q0| > alpha-1 */\
00580     DIFF_GT_MMX(%%mm0, %%mm1, %%mm5, %%mm4, %%mm6) /* |p1-p0| > beta-1 */\
00581     "por       %%mm4, %%mm7      \n\t"\
00582     DIFF_GT_MMX(%%mm3, %%mm2, %%mm5, %%mm4, %%mm6) /* |q1-q0| > beta-1 */\
00583     "por       %%mm4, %%mm7      \n\t"\
00584     "pxor      %%mm6, %%mm6      \n\t"\
00585     "pcmpeqb   %%mm6, %%mm7      \n\t"
00586 
00587 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask)
00588 // out: mm1=p0' mm2=q0'
00589 // clobbers: mm0,3-6
00590 #define H264_DEBLOCK_P0_Q0(pb_01, pb_3f)\
00591         "movq    %%mm1              , %%mm5 \n\t"\
00592         "pxor    %%mm2              , %%mm5 \n\t" /* p0^q0*/\
00593         "pand    "#pb_01"           , %%mm5 \n\t" /* (p0^q0)&1*/\
00594         "pcmpeqb %%mm4              , %%mm4 \n\t"\
00595         "pxor    %%mm4              , %%mm3 \n\t"\
00596         "pavgb   %%mm0              , %%mm3 \n\t" /* (p1 - q1 + 256)>>1*/\
00597         "pavgb   "MANGLE(ff_pb_3)"  , %%mm3 \n\t" /*(((p1 - q1 + 256)>>1)+4)>>1 = 64+2+(p1-q1)>>2*/\
00598         "pxor    %%mm1              , %%mm4 \n\t"\
00599         "pavgb   %%mm2              , %%mm4 \n\t" /* (q0 - p0 + 256)>>1*/\
00600         "pavgb   %%mm5              , %%mm3 \n\t"\
00601         "paddusb %%mm4              , %%mm3 \n\t" /* d+128+33*/\
00602         "movq    "MANGLE(ff_pb_A1)" , %%mm6 \n\t"\
00603         "psubusb %%mm3              , %%mm6 \n\t"\
00604         "psubusb "MANGLE(ff_pb_A1)" , %%mm3 \n\t"\
00605         "pminub  %%mm7              , %%mm6 \n\t"\
00606         "pminub  %%mm7              , %%mm3 \n\t"\
00607         "psubusb %%mm6              , %%mm1 \n\t"\
00608         "psubusb %%mm3              , %%mm2 \n\t"\
00609         "paddusb %%mm3              , %%mm1 \n\t"\
00610         "paddusb %%mm6              , %%mm2 \n\t"
00611 
00612 // in: mm0=p1 mm1=p0 mm2=q0 mm3=q1 mm7=(tc&mask) %8=ff_bone
00613 // out: (q1addr) = av_clip( (q2+((p0+q0+1)>>1))>>1, q1-tc0, q1+tc0 )
00614 // clobbers: q2, tmp, tc0
00615 #define H264_DEBLOCK_Q1(p1, q2, q2addr, q1addr, tc0, tmp)\
00616         "movq     %%mm1,  "#tmp"   \n\t"\
00617         "pavgb    %%mm2,  "#tmp"   \n\t"\
00618         "pavgb    "#tmp", "#q2"    \n\t" /* avg(p2,avg(p0,q0)) */\
00619         "pxor   "q2addr", "#tmp"   \n\t"\
00620         "pand     %9,     "#tmp"   \n\t" /* (p2^avg(p0,q0))&1 */\
00621         "psubusb  "#tmp", "#q2"    \n\t" /* (p2+((p0+q0+1)>>1))>>1 */\
00622         "movq     "#p1",  "#tmp"   \n\t"\
00623         "psubusb  "#tc0", "#tmp"   \n\t"\
00624         "paddusb  "#p1",  "#tc0"   \n\t"\
00625         "pmaxub   "#tmp", "#q2"    \n\t"\
00626         "pminub   "#tc0", "#q2"    \n\t"\
00627         "movq     "#q2",  "q1addr" \n\t"
00628 
00629 static inline void h264_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0)
00630 {
00631     DECLARE_ALIGNED(8, uint64_t, tmp0)[2];
00632 
00633     __asm__ volatile(
00634         "movq    (%2,%4), %%mm0    \n\t" //p1
00635         "movq    (%2,%4,2), %%mm1  \n\t" //p0
00636         "movq    (%3),    %%mm2    \n\t" //q0
00637         "movq    (%3,%4), %%mm3    \n\t" //q1
00638         H264_DEBLOCK_MASK(%7, %8)
00639 
00640         "movd      %6,    %%mm4    \n\t"
00641         "punpcklbw %%mm4, %%mm4    \n\t"
00642         "punpcklwd %%mm4, %%mm4    \n\t"
00643         "pcmpeqb   %%mm3, %%mm3    \n\t"
00644         "movq      %%mm4, %%mm6    \n\t"
00645         "pcmpgtb   %%mm3, %%mm4    \n\t"
00646         "movq      %%mm6, %1       \n\t"
00647         "pand      %%mm4, %%mm7    \n\t"
00648         "movq      %%mm7, %0       \n\t"
00649 
00650         /* filter p1 */
00651         "movq     (%2),   %%mm3    \n\t" //p2
00652         DIFF_GT2_MMX(%%mm1, %%mm3, %%mm5, %%mm6, %%mm4) // |p2-p0|>beta-1
00653         "pand     %%mm7,  %%mm6    \n\t" // mask & |p2-p0|<beta
00654         "pand     %1,     %%mm7    \n\t" // mask & tc0
00655         "movq     %%mm7,  %%mm4    \n\t"
00656         "psubb    %%mm6,  %%mm7    \n\t"
00657         "pand     %%mm4,  %%mm6    \n\t" // mask & |p2-p0|<beta & tc0
00658         H264_DEBLOCK_Q1(%%mm0, %%mm3, "(%2)", "(%2,%4)", %%mm6, %%mm4)
00659 
00660         /* filter q1 */
00661         "movq    (%3,%4,2), %%mm4  \n\t" //q2
00662         DIFF_GT2_MMX(%%mm2, %%mm4, %%mm5, %%mm6, %%mm3) // |q2-q0|>beta-1
00663         "pand     %0,     %%mm6    \n\t"
00664         "movq     %1,     %%mm5    \n\t" // can be merged with the and below but is slower then
00665         "pand     %%mm6,  %%mm5    \n\t"
00666         "psubb    %%mm6,  %%mm7    \n\t"
00667         "movq    (%3,%4), %%mm3    \n\t"
00668         H264_DEBLOCK_Q1(%%mm3, %%mm4, "(%3,%4,2)", "(%3,%4)", %%mm5, %%mm6)
00669 
00670         /* filter p0, q0 */
00671         H264_DEBLOCK_P0_Q0(%9, unused)
00672         "movq      %%mm1, (%2,%4,2) \n\t"
00673         "movq      %%mm2, (%3)      \n\t"
00674 
00675         : "=m"(tmp0[0]), "=m"(tmp0[1])
00676         : "r"(pix-3*stride), "r"(pix), "r"((x86_reg)stride),
00677           "m"(*tmp0/*unused*/), "m"(*(uint32_t*)tc0), "m"(alpha1), "m"(beta1),
00678           "m"(ff_bone)
00679     );
00680 }
00681 
00682 static void h264_v_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
00683 {
00684     if((tc0[0] & tc0[1]) >= 0)
00685         h264_loop_filter_luma_mmx2(pix, stride, alpha-1, beta-1, tc0);
00686     if((tc0[2] & tc0[3]) >= 0)
00687         h264_loop_filter_luma_mmx2(pix+8, stride, alpha-1, beta-1, tc0+2);
00688 }
00689 static void h264_h_loop_filter_luma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
00690 {
00691     //FIXME: could cut some load/stores by merging transpose with filter
00692     // also, it only needs to transpose 6x8
00693     DECLARE_ALIGNED(8, uint8_t, trans)[8*8];
00694     int i;
00695     for(i=0; i<2; i++, pix+=8*stride, tc0+=2) {
00696         if((tc0[0] & tc0[1]) < 0)
00697             continue;
00698         transpose4x4(trans,       pix-4,          8, stride);
00699         transpose4x4(trans  +4*8, pix,            8, stride);
00700         transpose4x4(trans+4,     pix-4+4*stride, 8, stride);
00701         transpose4x4(trans+4+4*8, pix  +4*stride, 8, stride);
00702         h264_loop_filter_luma_mmx2(trans+4*8, 8, alpha-1, beta-1, tc0);
00703         transpose4x4(pix-2,          trans  +2*8, stride, 8);
00704         transpose4x4(pix-2+4*stride, trans+4+2*8, stride, 8);
00705     }
00706 }
00707 
00708 static inline void h264_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha1, int beta1, int8_t *tc0)
00709 {
00710     __asm__ volatile(
00711         "movq    (%0),    %%mm0     \n\t" //p1
00712         "movq    (%0,%2), %%mm1     \n\t" //p0
00713         "movq    (%1),    %%mm2     \n\t" //q0
00714         "movq    (%1,%2), %%mm3     \n\t" //q1
00715         H264_DEBLOCK_MASK(%4, %5)
00716         "movd      %3,    %%mm6     \n\t"
00717         "punpcklbw %%mm6, %%mm6     \n\t"
00718         "pand      %%mm6, %%mm7     \n\t" // mm7 = tc&mask
00719         H264_DEBLOCK_P0_Q0(%6, %7)
00720         "movq      %%mm1, (%0,%2)   \n\t"
00721         "movq      %%mm2, (%1)      \n\t"
00722 
00723         :: "r"(pix-2*stride), "r"(pix), "r"((x86_reg)stride),
00724            "r"(*(uint32_t*)tc0),
00725            "m"(alpha1), "m"(beta1), "m"(ff_bone), "m"(ff_pb_3F)
00726     );
00727 }
00728 
00729 static void h264_v_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
00730 {
00731     h264_loop_filter_chroma_mmx2(pix, stride, alpha-1, beta-1, tc0);
00732 }
00733 
00734 static void h264_h_loop_filter_chroma_mmx2(uint8_t *pix, int stride, int alpha, int beta, int8_t *tc0)
00735 {
00736     //FIXME: could cut some load/stores by merging transpose with filter
00737     DECLARE_ALIGNED(8, uint8_t, trans)[8*4];
00738     transpose4x4(trans, pix-2, 8, stride);
00739     transpose4x4(trans+4, pix-2+4*stride, 8, stride);
00740     h264_loop_filter_chroma_mmx2(trans+2*8, 8, alpha-1, beta-1, tc0);
00741     transpose4x4(pix-2, trans, stride, 8);
00742     transpose4x4(pix-2+4*stride, trans+4, stride, 8);
00743 }
00744 
00745 // p0 = (p0 + q1 + 2*p1 + 2) >> 2
00746 #define H264_FILTER_CHROMA4(p0, p1, q1, one) \
00747     "movq    "#p0", %%mm4  \n\t"\
00748     "pxor    "#q1", %%mm4  \n\t"\
00749     "pand   "#one", %%mm4  \n\t" /* mm4 = (p0^q1)&1 */\
00750     "pavgb   "#q1", "#p0"  \n\t"\
00751     "psubusb %%mm4, "#p0"  \n\t"\
00752     "pavgb   "#p1", "#p0"  \n\t" /* dst = avg(p1, avg(p0,q1) - ((p0^q1)&1)) */\
00753 
00754 static inline void h264_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha1, int beta1)
00755 {
00756     __asm__ volatile(
00757         "movq    (%0),    %%mm0     \n\t"
00758         "movq    (%0,%2), %%mm1     \n\t"
00759         "movq    (%1),    %%mm2     \n\t"
00760         "movq    (%1,%2), %%mm3     \n\t"
00761         H264_DEBLOCK_MASK(%3, %4)
00762         "movq    %%mm1,   %%mm5     \n\t"
00763         "movq    %%mm2,   %%mm6     \n\t"
00764         H264_FILTER_CHROMA4(%%mm1, %%mm0, %%mm3, %5) //p0'
00765         H264_FILTER_CHROMA4(%%mm2, %%mm3, %%mm0, %5) //q0'
00766         "psubb   %%mm5,   %%mm1     \n\t"
00767         "psubb   %%mm6,   %%mm2     \n\t"
00768         "pand    %%mm7,   %%mm1     \n\t"
00769         "pand    %%mm7,   %%mm2     \n\t"
00770         "paddb   %%mm5,   %%mm1     \n\t"
00771         "paddb   %%mm6,   %%mm2     \n\t"
00772         "movq    %%mm1,   (%0,%2)   \n\t"
00773         "movq    %%mm2,   (%1)      \n\t"
00774         :: "r"(pix-2*stride), "r"(pix), "r"((x86_reg)stride),
00775            "m"(alpha1), "m"(beta1), "m"(ff_bone)
00776     );
00777 }
00778 
00779 static void h264_v_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta)
00780 {
00781     h264_loop_filter_chroma_intra_mmx2(pix, stride, alpha-1, beta-1);
00782 }
00783 
00784 static void h264_h_loop_filter_chroma_intra_mmx2(uint8_t *pix, int stride, int alpha, int beta)
00785 {
00786     //FIXME: could cut some load/stores by merging transpose with filter
00787     DECLARE_ALIGNED(8, uint8_t, trans)[8*4];
00788     transpose4x4(trans, pix-2, 8, stride);
00789     transpose4x4(trans+4, pix-2+4*stride, 8, stride);
00790     h264_loop_filter_chroma_intra_mmx2(trans+2*8, 8, alpha-1, beta-1);
00791     transpose4x4(pix-2, trans, stride, 8);
00792     transpose4x4(pix-2+4*stride, trans+4, stride, 8);
00793 }
00794 
00795 static void h264_loop_filter_strength_mmx2( int16_t bS[2][4][4], uint8_t nnz[40], int8_t ref[2][40], int16_t mv[2][40][2],
00796                                             int bidir, int edges, int step, int mask_mv0, int mask_mv1, int field ) {
00797     int dir;
00798     __asm__ volatile(
00799         "movq %0, %%mm7 \n"
00800         "movq %1, %%mm6 \n"
00801         ::"m"(ff_pb_1), "m"(ff_pb_3)
00802     );
00803     if(field)
00804         __asm__ volatile(
00805             "movq %0, %%mm6 \n"
00806             ::"m"(ff_pb_3_1)
00807         );
00808     __asm__ volatile(
00809         "movq  %%mm6, %%mm5 \n"
00810         "paddb %%mm5, %%mm5 \n"
00811     :);
00812 
00813     // could do a special case for dir==0 && edges==1, but it only reduces the
00814     // average filter time by 1.2%
00815     for( dir=1; dir>=0; dir-- ) {
00816         const x86_reg d_idx = dir ? -8 : -1;
00817         const int mask_mv = dir ? mask_mv1 : mask_mv0;
00818         DECLARE_ALIGNED(8, const uint64_t, mask_dir) = dir ? 0 : 0xffffffffffffffffULL;
00819         int b_idx, edge;
00820         for( b_idx=12, edge=0; edge<edges; edge+=step, b_idx+=8*step ) {
00821             __asm__ volatile(
00822                 "pand %0, %%mm0 \n\t"
00823                 ::"m"(mask_dir)
00824             );
00825             if(!(mask_mv & edge)) {
00826                 if(bidir) {
00827                     __asm__ volatile(
00828                         "movd         (%1,%0), %%mm2 \n"
00829                         "punpckldq  40(%1,%0), %%mm2 \n" // { ref0[bn], ref1[bn] }
00830                         "pshufw $0x44,   (%1), %%mm0 \n" // { ref0[b], ref0[b] }
00831                         "pshufw $0x44, 40(%1), %%mm1 \n" // { ref1[b], ref1[b] }
00832                         "pshufw $0x4E, %%mm2, %%mm3 \n"
00833                         "psubb         %%mm2, %%mm0 \n" // { ref0[b]!=ref0[bn], ref0[b]!=ref1[bn] }
00834                         "psubb         %%mm3, %%mm1 \n" // { ref1[b]!=ref1[bn], ref1[b]!=ref0[bn] }
00835                         "1: \n"
00836                         "por           %%mm1, %%mm0 \n"
00837                         "movq      (%2,%0,4), %%mm1 \n"
00838                         "movq     8(%2,%0,4), %%mm2 \n"
00839                         "movq          %%mm1, %%mm3 \n"
00840                         "movq          %%mm2, %%mm4 \n"
00841                         "psubw          (%2), %%mm1 \n"
00842                         "psubw         8(%2), %%mm2 \n"
00843                         "psubw       160(%2), %%mm3 \n"
00844                         "psubw       168(%2), %%mm4 \n"
00845                         "packsswb      %%mm2, %%mm1 \n"
00846                         "packsswb      %%mm4, %%mm3 \n"
00847                         "paddb         %%mm6, %%mm1 \n"
00848                         "paddb         %%mm6, %%mm3 \n"
00849                         "psubusb       %%mm5, %%mm1 \n" // abs(mv[b] - mv[bn]) >= limit
00850                         "psubusb       %%mm5, %%mm3 \n"
00851                         "packsswb      %%mm3, %%mm1 \n"
00852                         "add $40, %0 \n"
00853                         "cmp $40, %0 \n"
00854                         "jl 1b \n"
00855                         "sub $80, %0 \n"
00856                         "pshufw $0x4E, %%mm1, %%mm1 \n"
00857                         "por           %%mm1, %%mm0 \n"
00858                         "pshufw $0x4E, %%mm0, %%mm1 \n"
00859                         "pminub        %%mm1, %%mm0 \n"
00860                         ::"r"(d_idx),
00861                           "r"(ref[0]+b_idx),
00862                           "r"(mv[0]+b_idx)
00863                     );
00864                 } else {
00865                     __asm__ volatile(
00866                         "movd        (%1), %%mm0 \n"
00867                         "psubb    (%1,%0), %%mm0 \n" // ref[b] != ref[bn]
00868                         "movq        (%2), %%mm1 \n"
00869                         "movq       8(%2), %%mm2 \n"
00870                         "psubw  (%2,%0,4), %%mm1 \n"
00871                         "psubw 8(%2,%0,4), %%mm2 \n"
00872                         "packsswb   %%mm2, %%mm1 \n"
00873                         "paddb      %%mm6, %%mm1 \n"
00874                         "psubusb    %%mm5, %%mm1 \n" // abs(mv[b] - mv[bn]) >= limit
00875                         "packsswb   %%mm1, %%mm1 \n"
00876                         "por        %%mm1, %%mm0 \n"
00877                         ::"r"(d_idx),
00878                           "r"(ref[0]+b_idx),
00879                           "r"(mv[0]+b_idx)
00880                     );
00881                 }
00882             }
00883             __asm__ volatile(
00884                 "movd %0, %%mm1 \n"
00885                 "por  %1, %%mm1 \n" // nnz[b] || nnz[bn]
00886                 ::"m"(nnz[b_idx]),
00887                   "m"(nnz[b_idx+d_idx])
00888             );
00889             __asm__ volatile(
00890                 "pminub    %%mm7, %%mm1 \n"
00891                 "pminub    %%mm7, %%mm0 \n"
00892                 "psllw        $1, %%mm1 \n"
00893                 "pxor      %%mm2, %%mm2 \n"
00894                 "pmaxub    %%mm0, %%mm1 \n"
00895                 "punpcklbw %%mm2, %%mm1 \n"
00896                 "movq      %%mm1, %0    \n"
00897                 :"=m"(*bS[dir][edge])
00898                 ::"memory"
00899             );
00900         }
00901         edges = 4;
00902         step = 1;
00903     }
00904     __asm__ volatile(
00905         "movq   (%0), %%mm0 \n\t"
00906         "movq  8(%0), %%mm1 \n\t"
00907         "movq 16(%0), %%mm2 \n\t"
00908         "movq 24(%0), %%mm3 \n\t"
00909         TRANSPOSE4(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4)
00910         "movq %%mm0,   (%0) \n\t"
00911         "movq %%mm3,  8(%0) \n\t"
00912         "movq %%mm4, 16(%0) \n\t"
00913         "movq %%mm2, 24(%0) \n\t"
00914         ::"r"(bS[0])
00915         :"memory"
00916     );
00917 }
00918 
00919 /***********************************/
00920 /* motion compensation */
00921 
00922 #define QPEL_H264V_MM(A,B,C,D,E,F,OP,T,Z,d,q)\
00923         "mov"#q" "#C", "#T"         \n\t"\
00924         "mov"#d" (%0), "#F"         \n\t"\
00925         "paddw "#D", "#T"           \n\t"\
00926         "psllw $2, "#T"             \n\t"\
00927         "psubw "#B", "#T"           \n\t"\
00928         "psubw "#E", "#T"           \n\t"\
00929         "punpcklbw "#Z", "#F"       \n\t"\
00930         "pmullw %4, "#T"            \n\t"\
00931         "paddw %5, "#A"             \n\t"\
00932         "add %2, %0                 \n\t"\
00933         "paddw "#F", "#A"           \n\t"\
00934         "paddw "#A", "#T"           \n\t"\
00935         "psraw $5, "#T"             \n\t"\
00936         "packuswb "#T", "#T"        \n\t"\
00937         OP(T, (%1), A, d)\
00938         "add %3, %1                 \n\t"
00939 
00940 #define QPEL_H264HV_MM(A,B,C,D,E,F,OF,T,Z,d,q)\
00941         "mov"#q" "#C", "#T"         \n\t"\
00942         "mov"#d" (%0), "#F"         \n\t"\
00943         "paddw "#D", "#T"           \n\t"\
00944         "psllw $2, "#T"             \n\t"\
00945         "paddw %4, "#A"             \n\t"\
00946         "psubw "#B", "#T"           \n\t"\
00947         "psubw "#E", "#T"           \n\t"\
00948         "punpcklbw "#Z", "#F"       \n\t"\
00949         "pmullw %3, "#T"            \n\t"\
00950         "paddw "#F", "#A"           \n\t"\
00951         "add %2, %0                 \n\t"\
00952         "paddw "#A", "#T"           \n\t"\
00953         "mov"#q" "#T", "#OF"(%1)    \n\t"
00954 
00955 #define QPEL_H264V(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%mm6,%%mm7,d,q)
00956 #define QPEL_H264HV(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%mm6,%%mm7,d,q)
00957 #define QPEL_H264V_XMM(A,B,C,D,E,F,OP) QPEL_H264V_MM(A,B,C,D,E,F,OP,%%xmm6,%%xmm7,q,dqa)
00958 #define QPEL_H264HV_XMM(A,B,C,D,E,F,OF) QPEL_H264HV_MM(A,B,C,D,E,F,OF,%%xmm6,%%xmm7,q,dqa)
00959 
00960 
00961 #define QPEL_H264(OPNAME, OP, MMX)\
00962 static av_noinline void OPNAME ## h264_qpel4_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
00963     int h=4;\
00964 \
00965     __asm__ volatile(\
00966         "pxor %%mm7, %%mm7          \n\t"\
00967         "movq "MANGLE(ff_pw_5) ", %%mm4\n\t"\
00968         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\
00969         "1:                         \n\t"\
00970         "movd  -1(%0), %%mm1        \n\t"\
00971         "movd    (%0), %%mm2        \n\t"\
00972         "movd   1(%0), %%mm3        \n\t"\
00973         "movd   2(%0), %%mm0        \n\t"\
00974         "punpcklbw %%mm7, %%mm1     \n\t"\
00975         "punpcklbw %%mm7, %%mm2     \n\t"\
00976         "punpcklbw %%mm7, %%mm3     \n\t"\
00977         "punpcklbw %%mm7, %%mm0     \n\t"\
00978         "paddw %%mm0, %%mm1         \n\t"\
00979         "paddw %%mm3, %%mm2         \n\t"\
00980         "movd  -2(%0), %%mm0        \n\t"\
00981         "movd   3(%0), %%mm3        \n\t"\
00982         "punpcklbw %%mm7, %%mm0     \n\t"\
00983         "punpcklbw %%mm7, %%mm3     \n\t"\
00984         "paddw %%mm3, %%mm0         \n\t"\
00985         "psllw $2, %%mm2            \n\t"\
00986         "psubw %%mm1, %%mm2         \n\t"\
00987         "pmullw %%mm4, %%mm2        \n\t"\
00988         "paddw %%mm5, %%mm0         \n\t"\
00989         "paddw %%mm2, %%mm0         \n\t"\
00990         "psraw $5, %%mm0            \n\t"\
00991         "packuswb %%mm0, %%mm0      \n\t"\
00992         OP(%%mm0, (%1),%%mm6, d)\
00993         "add %3, %0                 \n\t"\
00994         "add %4, %1                 \n\t"\
00995         "decl %2                    \n\t"\
00996         " jnz 1b                    \n\t"\
00997         : "+a"(src), "+c"(dst), "+g"(h)\
00998         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\
00999         : "memory"\
01000     );\
01001 }\
01002 static av_noinline void OPNAME ## h264_qpel4_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01003     int h=4;\
01004     __asm__ volatile(\
01005         "pxor %%mm7, %%mm7          \n\t"\
01006         "movq %0, %%mm4             \n\t"\
01007         "movq %1, %%mm5             \n\t"\
01008         :: "m"(ff_pw_5), "m"(ff_pw_16)\
01009     );\
01010     do{\
01011     __asm__ volatile(\
01012         "movd  -1(%0), %%mm1        \n\t"\
01013         "movd    (%0), %%mm2        \n\t"\
01014         "movd   1(%0), %%mm3        \n\t"\
01015         "movd   2(%0), %%mm0        \n\t"\
01016         "punpcklbw %%mm7, %%mm1     \n\t"\
01017         "punpcklbw %%mm7, %%mm2     \n\t"\
01018         "punpcklbw %%mm7, %%mm3     \n\t"\
01019         "punpcklbw %%mm7, %%mm0     \n\t"\
01020         "paddw %%mm0, %%mm1         \n\t"\
01021         "paddw %%mm3, %%mm2         \n\t"\
01022         "movd  -2(%0), %%mm0        \n\t"\
01023         "movd   3(%0), %%mm3        \n\t"\
01024         "punpcklbw %%mm7, %%mm0     \n\t"\
01025         "punpcklbw %%mm7, %%mm3     \n\t"\
01026         "paddw %%mm3, %%mm0         \n\t"\
01027         "psllw $2, %%mm2            \n\t"\
01028         "psubw %%mm1, %%mm2         \n\t"\
01029         "pmullw %%mm4, %%mm2        \n\t"\
01030         "paddw %%mm5, %%mm0         \n\t"\
01031         "paddw %%mm2, %%mm0         \n\t"\
01032         "movd   (%2), %%mm3         \n\t"\
01033         "psraw $5, %%mm0            \n\t"\
01034         "packuswb %%mm0, %%mm0      \n\t"\
01035         PAVGB" %%mm3, %%mm0         \n\t"\
01036         OP(%%mm0, (%1),%%mm6, d)\
01037         "add %4, %0                 \n\t"\
01038         "add %4, %1                 \n\t"\
01039         "add %3, %2                 \n\t"\
01040         : "+a"(src), "+c"(dst), "+d"(src2)\
01041         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride)\
01042         : "memory"\
01043     );\
01044     }while(--h);\
01045 }\
01046 static av_noinline void OPNAME ## h264_qpel4_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01047     src -= 2*srcStride;\
01048     __asm__ volatile(\
01049         "pxor %%mm7, %%mm7          \n\t"\
01050         "movd (%0), %%mm0           \n\t"\
01051         "add %2, %0                 \n\t"\
01052         "movd (%0), %%mm1           \n\t"\
01053         "add %2, %0                 \n\t"\
01054         "movd (%0), %%mm2           \n\t"\
01055         "add %2, %0                 \n\t"\
01056         "movd (%0), %%mm3           \n\t"\
01057         "add %2, %0                 \n\t"\
01058         "movd (%0), %%mm4           \n\t"\
01059         "add %2, %0                 \n\t"\
01060         "punpcklbw %%mm7, %%mm0     \n\t"\
01061         "punpcklbw %%mm7, %%mm1     \n\t"\
01062         "punpcklbw %%mm7, %%mm2     \n\t"\
01063         "punpcklbw %%mm7, %%mm3     \n\t"\
01064         "punpcklbw %%mm7, %%mm4     \n\t"\
01065         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\
01066         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\
01067         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\
01068         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\
01069          \
01070         : "+a"(src), "+c"(dst)\
01071         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01072         : "memory"\
01073     );\
01074 }\
01075 static av_noinline void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\
01076     int h=4;\
01077     int w=3;\
01078     src -= 2*srcStride+2;\
01079     while(w--){\
01080         __asm__ volatile(\
01081             "pxor %%mm7, %%mm7      \n\t"\
01082             "movd (%0), %%mm0       \n\t"\
01083             "add %2, %0             \n\t"\
01084             "movd (%0), %%mm1       \n\t"\
01085             "add %2, %0             \n\t"\
01086             "movd (%0), %%mm2       \n\t"\
01087             "add %2, %0             \n\t"\
01088             "movd (%0), %%mm3       \n\t"\
01089             "add %2, %0             \n\t"\
01090             "movd (%0), %%mm4       \n\t"\
01091             "add %2, %0             \n\t"\
01092             "punpcklbw %%mm7, %%mm0 \n\t"\
01093             "punpcklbw %%mm7, %%mm1 \n\t"\
01094             "punpcklbw %%mm7, %%mm2 \n\t"\
01095             "punpcklbw %%mm7, %%mm3 \n\t"\
01096             "punpcklbw %%mm7, %%mm4 \n\t"\
01097             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*8*3)\
01098             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*8*3)\
01099             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*8*3)\
01100             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*8*3)\
01101              \
01102             : "+a"(src)\
01103             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01104             : "memory"\
01105         );\
01106         tmp += 4;\
01107         src += 4 - 9*srcStride;\
01108     }\
01109     tmp -= 3*4;\
01110     __asm__ volatile(\
01111         "1:                         \n\t"\
01112         "movq     (%0), %%mm0       \n\t"\
01113         "paddw  10(%0), %%mm0       \n\t"\
01114         "movq    2(%0), %%mm1       \n\t"\
01115         "paddw   8(%0), %%mm1       \n\t"\
01116         "movq    4(%0), %%mm2       \n\t"\
01117         "paddw   6(%0), %%mm2       \n\t"\
01118         "psubw %%mm1, %%mm0         \n\t"/*a-b   (abccba)*/\
01119         "psraw $2, %%mm0            \n\t"/*(a-b)/4 */\
01120         "psubw %%mm1, %%mm0         \n\t"/*(a-b)/4-b */\
01121         "paddsw %%mm2, %%mm0        \n\t"\
01122         "psraw $2, %%mm0            \n\t"/*((a-b)/4-b+c)/4 */\
01123         "paddw %%mm2, %%mm0         \n\t"/*(a-5*b+20*c)/16 */\
01124         "psraw $6, %%mm0            \n\t"\
01125         "packuswb %%mm0, %%mm0      \n\t"\
01126         OP(%%mm0, (%1),%%mm7, d)\
01127         "add $24, %0                \n\t"\
01128         "add %3, %1                 \n\t"\
01129         "decl %2                    \n\t"\
01130         " jnz 1b                    \n\t"\
01131         : "+a"(tmp), "+c"(dst), "+g"(h)\
01132         : "S"((x86_reg)dstStride)\
01133         : "memory"\
01134     );\
01135 }\
01136 \
01137 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01138     int h=8;\
01139     __asm__ volatile(\
01140         "pxor %%mm7, %%mm7          \n\t"\
01141         "movq "MANGLE(ff_pw_5)", %%mm6\n\t"\
01142         "1:                         \n\t"\
01143         "movq    (%0), %%mm0        \n\t"\
01144         "movq   1(%0), %%mm2        \n\t"\
01145         "movq %%mm0, %%mm1          \n\t"\
01146         "movq %%mm2, %%mm3          \n\t"\
01147         "punpcklbw %%mm7, %%mm0     \n\t"\
01148         "punpckhbw %%mm7, %%mm1     \n\t"\
01149         "punpcklbw %%mm7, %%mm2     \n\t"\
01150         "punpckhbw %%mm7, %%mm3     \n\t"\
01151         "paddw %%mm2, %%mm0         \n\t"\
01152         "paddw %%mm3, %%mm1         \n\t"\
01153         "psllw $2, %%mm0            \n\t"\
01154         "psllw $2, %%mm1            \n\t"\
01155         "movq   -1(%0), %%mm2       \n\t"\
01156         "movq    2(%0), %%mm4       \n\t"\
01157         "movq %%mm2, %%mm3          \n\t"\
01158         "movq %%mm4, %%mm5          \n\t"\
01159         "punpcklbw %%mm7, %%mm2     \n\t"\
01160         "punpckhbw %%mm7, %%mm3     \n\t"\
01161         "punpcklbw %%mm7, %%mm4     \n\t"\
01162         "punpckhbw %%mm7, %%mm5     \n\t"\
01163         "paddw %%mm4, %%mm2         \n\t"\
01164         "paddw %%mm3, %%mm5         \n\t"\
01165         "psubw %%mm2, %%mm0         \n\t"\
01166         "psubw %%mm5, %%mm1         \n\t"\
01167         "pmullw %%mm6, %%mm0        \n\t"\
01168         "pmullw %%mm6, %%mm1        \n\t"\
01169         "movd   -2(%0), %%mm2       \n\t"\
01170         "movd    7(%0), %%mm5       \n\t"\
01171         "punpcklbw %%mm7, %%mm2     \n\t"\
01172         "punpcklbw %%mm7, %%mm5     \n\t"\
01173         "paddw %%mm3, %%mm2         \n\t"\
01174         "paddw %%mm5, %%mm4         \n\t"\
01175         "movq "MANGLE(ff_pw_16)", %%mm5\n\t"\
01176         "paddw %%mm5, %%mm2         \n\t"\
01177         "paddw %%mm5, %%mm4         \n\t"\
01178         "paddw %%mm2, %%mm0         \n\t"\
01179         "paddw %%mm4, %%mm1         \n\t"\
01180         "psraw $5, %%mm0            \n\t"\
01181         "psraw $5, %%mm1            \n\t"\
01182         "packuswb %%mm1, %%mm0      \n\t"\
01183         OP(%%mm0, (%1),%%mm5, q)\
01184         "add %3, %0                 \n\t"\
01185         "add %4, %1                 \n\t"\
01186         "decl %2                    \n\t"\
01187         " jnz 1b                    \n\t"\
01188         : "+a"(src), "+c"(dst), "+g"(h)\
01189         : "d"((x86_reg)srcStride), "S"((x86_reg)dstStride)\
01190         : "memory"\
01191     );\
01192 }\
01193 \
01194 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01195     int h=8;\
01196     __asm__ volatile(\
01197         "pxor %%mm7, %%mm7          \n\t"\
01198         "movq %0, %%mm6             \n\t"\
01199         :: "m"(ff_pw_5)\
01200     );\
01201     do{\
01202     __asm__ volatile(\
01203         "movq    (%0), %%mm0        \n\t"\
01204         "movq   1(%0), %%mm2        \n\t"\
01205         "movq %%mm0, %%mm1          \n\t"\
01206         "movq %%mm2, %%mm3          \n\t"\
01207         "punpcklbw %%mm7, %%mm0     \n\t"\
01208         "punpckhbw %%mm7, %%mm1     \n\t"\
01209         "punpcklbw %%mm7, %%mm2     \n\t"\
01210         "punpckhbw %%mm7, %%mm3     \n\t"\
01211         "paddw %%mm2, %%mm0         \n\t"\
01212         "paddw %%mm3, %%mm1         \n\t"\
01213         "psllw $2, %%mm0            \n\t"\
01214         "psllw $2, %%mm1            \n\t"\
01215         "movq   -1(%0), %%mm2       \n\t"\
01216         "movq    2(%0), %%mm4       \n\t"\
01217         "movq %%mm2, %%mm3          \n\t"\
01218         "movq %%mm4, %%mm5          \n\t"\
01219         "punpcklbw %%mm7, %%mm2     \n\t"\
01220         "punpckhbw %%mm7, %%mm3     \n\t"\
01221         "punpcklbw %%mm7, %%mm4     \n\t"\
01222         "punpckhbw %%mm7, %%mm5     \n\t"\
01223         "paddw %%mm4, %%mm2         \n\t"\
01224         "paddw %%mm3, %%mm5         \n\t"\
01225         "psubw %%mm2, %%mm0         \n\t"\
01226         "psubw %%mm5, %%mm1         \n\t"\
01227         "pmullw %%mm6, %%mm0        \n\t"\
01228         "pmullw %%mm6, %%mm1        \n\t"\
01229         "movd   -2(%0), %%mm2       \n\t"\
01230         "movd    7(%0), %%mm5       \n\t"\
01231         "punpcklbw %%mm7, %%mm2     \n\t"\
01232         "punpcklbw %%mm7, %%mm5     \n\t"\
01233         "paddw %%mm3, %%mm2         \n\t"\
01234         "paddw %%mm5, %%mm4         \n\t"\
01235         "movq %5, %%mm5             \n\t"\
01236         "paddw %%mm5, %%mm2         \n\t"\
01237         "paddw %%mm5, %%mm4         \n\t"\
01238         "paddw %%mm2, %%mm0         \n\t"\
01239         "paddw %%mm4, %%mm1         \n\t"\
01240         "psraw $5, %%mm0            \n\t"\
01241         "psraw $5, %%mm1            \n\t"\
01242         "movq (%2), %%mm4           \n\t"\
01243         "packuswb %%mm1, %%mm0      \n\t"\
01244         PAVGB" %%mm4, %%mm0         \n\t"\
01245         OP(%%mm0, (%1),%%mm5, q)\
01246         "add %4, %0                 \n\t"\
01247         "add %4, %1                 \n\t"\
01248         "add %3, %2                 \n\t"\
01249         : "+a"(src), "+c"(dst), "+d"(src2)\
01250         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\
01251           "m"(ff_pw_16)\
01252         : "memory"\
01253     );\
01254     }while(--h);\
01255 }\
01256 \
01257 static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\
01258     int w= 2;\
01259     src -= 2*srcStride;\
01260     \
01261     while(w--){\
01262       __asm__ volatile(\
01263         "pxor %%mm7, %%mm7          \n\t"\
01264         "movd (%0), %%mm0           \n\t"\
01265         "add %2, %0                 \n\t"\
01266         "movd (%0), %%mm1           \n\t"\
01267         "add %2, %0                 \n\t"\
01268         "movd (%0), %%mm2           \n\t"\
01269         "add %2, %0                 \n\t"\
01270         "movd (%0), %%mm3           \n\t"\
01271         "add %2, %0                 \n\t"\
01272         "movd (%0), %%mm4           \n\t"\
01273         "add %2, %0                 \n\t"\
01274         "punpcklbw %%mm7, %%mm0     \n\t"\
01275         "punpcklbw %%mm7, %%mm1     \n\t"\
01276         "punpcklbw %%mm7, %%mm2     \n\t"\
01277         "punpcklbw %%mm7, %%mm3     \n\t"\
01278         "punpcklbw %%mm7, %%mm4     \n\t"\
01279         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\
01280         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\
01281         QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\
01282         QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\
01283         QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\
01284         QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\
01285         QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\
01286         QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\
01287          \
01288         : "+a"(src), "+c"(dst)\
01289         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01290         : "memory"\
01291      );\
01292      if(h==16){\
01293         __asm__ volatile(\
01294             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\
01295             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\
01296             QPEL_H264V(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, OP)\
01297             QPEL_H264V(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, OP)\
01298             QPEL_H264V(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, OP)\
01299             QPEL_H264V(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, OP)\
01300             QPEL_H264V(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, OP)\
01301             QPEL_H264V(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, OP)\
01302             \
01303            : "+a"(src), "+c"(dst)\
01304            : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01305            : "memory"\
01306         );\
01307      }\
01308      src += 4-(h+5)*srcStride;\
01309      dst += 4-h*dstStride;\
01310    }\
01311 }\
01312 static av_always_inline void OPNAME ## h264_qpel8or16_hv1_lowpass_ ## MMX(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){\
01313     int w = (size+8)>>2;\
01314     src -= 2*srcStride+2;\
01315     while(w--){\
01316         __asm__ volatile(\
01317             "pxor %%mm7, %%mm7      \n\t"\
01318             "movd (%0), %%mm0       \n\t"\
01319             "add %2, %0             \n\t"\
01320             "movd (%0), %%mm1       \n\t"\
01321             "add %2, %0             \n\t"\
01322             "movd (%0), %%mm2       \n\t"\
01323             "add %2, %0             \n\t"\
01324             "movd (%0), %%mm3       \n\t"\
01325             "add %2, %0             \n\t"\
01326             "movd (%0), %%mm4       \n\t"\
01327             "add %2, %0             \n\t"\
01328             "punpcklbw %%mm7, %%mm0 \n\t"\
01329             "punpcklbw %%mm7, %%mm1 \n\t"\
01330             "punpcklbw %%mm7, %%mm2 \n\t"\
01331             "punpcklbw %%mm7, %%mm3 \n\t"\
01332             "punpcklbw %%mm7, %%mm4 \n\t"\
01333             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 0*48)\
01334             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 1*48)\
01335             QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 2*48)\
01336             QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 3*48)\
01337             QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 4*48)\
01338             QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 5*48)\
01339             QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 6*48)\
01340             QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 7*48)\
01341             : "+a"(src)\
01342             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01343             : "memory"\
01344         );\
01345         if(size==16){\
01346             __asm__ volatile(\
01347                 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1,  8*48)\
01348                 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2,  9*48)\
01349                 QPEL_H264HV(%%mm4, %%mm5, %%mm0, %%mm1, %%mm2, %%mm3, 10*48)\
01350                 QPEL_H264HV(%%mm5, %%mm0, %%mm1, %%mm2, %%mm3, %%mm4, 11*48)\
01351                 QPEL_H264HV(%%mm0, %%mm1, %%mm2, %%mm3, %%mm4, %%mm5, 12*48)\
01352                 QPEL_H264HV(%%mm1, %%mm2, %%mm3, %%mm4, %%mm5, %%mm0, 13*48)\
01353                 QPEL_H264HV(%%mm2, %%mm3, %%mm4, %%mm5, %%mm0, %%mm1, 14*48)\
01354                 QPEL_H264HV(%%mm3, %%mm4, %%mm5, %%mm0, %%mm1, %%mm2, 15*48)\
01355                 : "+a"(src)\
01356                 : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01357                 : "memory"\
01358             );\
01359         }\
01360         tmp += 4;\
01361         src += 4 - (size+5)*srcStride;\
01362     }\
01363 }\
01364 static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\
01365     int w = size>>4;\
01366     do{\
01367     int h = size;\
01368     __asm__ volatile(\
01369         "1:                         \n\t"\
01370         "movq     (%0), %%mm0       \n\t"\
01371         "movq    8(%0), %%mm3       \n\t"\
01372         "movq    2(%0), %%mm1       \n\t"\
01373         "movq   10(%0), %%mm4       \n\t"\
01374         "paddw   %%mm4, %%mm0       \n\t"\
01375         "paddw   %%mm3, %%mm1       \n\t"\
01376         "paddw  18(%0), %%mm3       \n\t"\
01377         "paddw  16(%0), %%mm4       \n\t"\
01378         "movq    4(%0), %%mm2       \n\t"\
01379         "movq   12(%0), %%mm5       \n\t"\
01380         "paddw   6(%0), %%mm2       \n\t"\
01381         "paddw  14(%0), %%mm5       \n\t"\
01382         "psubw %%mm1, %%mm0         \n\t"\
01383         "psubw %%mm4, %%mm3         \n\t"\
01384         "psraw $2, %%mm0            \n\t"\
01385         "psraw $2, %%mm3            \n\t"\
01386         "psubw %%mm1, %%mm0         \n\t"\
01387         "psubw %%mm4, %%mm3         \n\t"\
01388         "paddsw %%mm2, %%mm0        \n\t"\
01389         "paddsw %%mm5, %%mm3        \n\t"\
01390         "psraw $2, %%mm0            \n\t"\
01391         "psraw $2, %%mm3            \n\t"\
01392         "paddw %%mm2, %%mm0         \n\t"\
01393         "paddw %%mm5, %%mm3         \n\t"\
01394         "psraw $6, %%mm0            \n\t"\
01395         "psraw $6, %%mm3            \n\t"\
01396         "packuswb %%mm3, %%mm0      \n\t"\
01397         OP(%%mm0, (%1),%%mm7, q)\
01398         "add $48, %0                \n\t"\
01399         "add %3, %1                 \n\t"\
01400         "decl %2                    \n\t"\
01401         " jnz 1b                    \n\t"\
01402         : "+a"(tmp), "+c"(dst), "+g"(h)\
01403         : "S"((x86_reg)dstStride)\
01404         : "memory"\
01405     );\
01406     tmp += 8 - size*24;\
01407     dst += 8 - size*dstStride;\
01408     }while(w--);\
01409 }\
01410 \
01411 static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01412     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\
01413 }\
01414 static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01415     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\
01416     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
01417 }\
01418 \
01419 static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01420     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
01421     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
01422     src += 8*srcStride;\
01423     dst += 8*dstStride;\
01424     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
01425     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
01426 }\
01427 \
01428 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01429     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
01430     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
01431     src += 8*dstStride;\
01432     dst += 8*dstStride;\
01433     src2 += 8*src2Stride;\
01434     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
01435     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
01436 }\
01437 \
01438 static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
01439           put_h264_qpel8or16_hv1_lowpass_ ## MMX(tmp, src, tmpStride, srcStride, size);\
01440     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
01441 }\
01442 static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\
01443     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 8);\
01444 }\
01445 \
01446 static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\
01447     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst  , tmp  , src  , dstStride, tmpStride, srcStride, 16);\
01448 }\
01449 \
01450 static av_noinline void OPNAME ## pixels4_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\
01451 {\
01452     __asm__ volatile(\
01453         "movq      (%1), %%mm0          \n\t"\
01454         "movq    24(%1), %%mm1          \n\t"\
01455         "psraw      $5,  %%mm0          \n\t"\
01456         "psraw      $5,  %%mm1          \n\t"\
01457         "packuswb %%mm0, %%mm0          \n\t"\
01458         "packuswb %%mm1, %%mm1          \n\t"\
01459         PAVGB"     (%0), %%mm0          \n\t"\
01460         PAVGB"  (%0,%3), %%mm1          \n\t"\
01461         OP(%%mm0, (%2),    %%mm4, d)\
01462         OP(%%mm1, (%2,%4), %%mm5, d)\
01463         "lea  (%0,%3,2), %0             \n\t"\
01464         "lea  (%2,%4,2), %2             \n\t"\
01465         "movq    48(%1), %%mm0          \n\t"\
01466         "movq    72(%1), %%mm1          \n\t"\
01467         "psraw      $5,  %%mm0          \n\t"\
01468         "psraw      $5,  %%mm1          \n\t"\
01469         "packuswb %%mm0, %%mm0          \n\t"\
01470         "packuswb %%mm1, %%mm1          \n\t"\
01471         PAVGB"     (%0), %%mm0          \n\t"\
01472         PAVGB"  (%0,%3), %%mm1          \n\t"\
01473         OP(%%mm0, (%2),    %%mm4, d)\
01474         OP(%%mm1, (%2,%4), %%mm5, d)\
01475         :"+a"(src8), "+c"(src16), "+d"(dst)\
01476         :"S"((x86_reg)src8Stride), "D"((x86_reg)dstStride)\
01477         :"memory");\
01478 }\
01479 static av_noinline void OPNAME ## pixels8_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\
01480 {\
01481     do{\
01482     __asm__ volatile(\
01483         "movq      (%1), %%mm0          \n\t"\
01484         "movq     8(%1), %%mm1          \n\t"\
01485         "movq    48(%1), %%mm2          \n\t"\
01486         "movq  8+48(%1), %%mm3          \n\t"\
01487         "psraw      $5,  %%mm0          \n\t"\
01488         "psraw      $5,  %%mm1          \n\t"\
01489         "psraw      $5,  %%mm2          \n\t"\
01490         "psraw      $5,  %%mm3          \n\t"\
01491         "packuswb %%mm1, %%mm0          \n\t"\
01492         "packuswb %%mm3, %%mm2          \n\t"\
01493         PAVGB"     (%0), %%mm0          \n\t"\
01494         PAVGB"  (%0,%3), %%mm2          \n\t"\
01495         OP(%%mm0, (%2), %%mm5, q)\
01496         OP(%%mm2, (%2,%4), %%mm5, q)\
01497         ::"a"(src8), "c"(src16), "d"(dst),\
01498           "r"((x86_reg)src8Stride), "r"((x86_reg)dstStride)\
01499         :"memory");\
01500         src8 += 2L*src8Stride;\
01501         src16 += 48;\
01502         dst += 2L*dstStride;\
01503     }while(h-=2);\
01504 }\
01505 static void OPNAME ## pixels16_l2_shift5_ ## MMX(uint8_t *dst, int16_t *src16, uint8_t *src8, int dstStride, int src8Stride, int h)\
01506 {\
01507     OPNAME ## pixels8_l2_shift5_ ## MMX(dst  , src16  , src8  , dstStride, src8Stride, h);\
01508     OPNAME ## pixels8_l2_shift5_ ## MMX(dst+8, src16+8, src8+8, dstStride, src8Stride, h);\
01509 }\
01510 
01511 
01512 #if ARCH_X86_64
01513 #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
01514 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01515     int h=16;\
01516     __asm__ volatile(\
01517         "pxor %%xmm15, %%xmm15      \n\t"\
01518         "movdqa %6, %%xmm14         \n\t"\
01519         "movdqa %7, %%xmm13         \n\t"\
01520         "1:                         \n\t"\
01521         "lddqu    6(%0), %%xmm1     \n\t"\
01522         "lddqu   -2(%0), %%xmm7     \n\t"\
01523         "movdqa  %%xmm1, %%xmm0     \n\t"\
01524         "punpckhbw %%xmm15, %%xmm1  \n\t"\
01525         "punpcklbw %%xmm15, %%xmm0  \n\t"\
01526         "punpcklbw %%xmm15, %%xmm7  \n\t"\
01527         "movdqa  %%xmm1, %%xmm2     \n\t"\
01528         "movdqa  %%xmm0, %%xmm6     \n\t"\
01529         "movdqa  %%xmm1, %%xmm3     \n\t"\
01530         "movdqa  %%xmm0, %%xmm8     \n\t"\
01531         "movdqa  %%xmm1, %%xmm4     \n\t"\
01532         "movdqa  %%xmm0, %%xmm9     \n\t"\
01533         "movdqa  %%xmm0, %%xmm12    \n\t"\
01534         "movdqa  %%xmm1, %%xmm11    \n\t"\
01535         "palignr $10,%%xmm0, %%xmm11\n\t"\
01536         "palignr $10,%%xmm7, %%xmm12\n\t"\
01537         "palignr $2, %%xmm0, %%xmm4 \n\t"\
01538         "palignr $2, %%xmm7, %%xmm9 \n\t"\
01539         "palignr $4, %%xmm0, %%xmm3 \n\t"\
01540         "palignr $4, %%xmm7, %%xmm8 \n\t"\
01541         "palignr $6, %%xmm0, %%xmm2 \n\t"\
01542         "palignr $6, %%xmm7, %%xmm6 \n\t"\
01543         "paddw   %%xmm0 ,%%xmm11    \n\t"\
01544         "palignr $8, %%xmm0, %%xmm1 \n\t"\
01545         "palignr $8, %%xmm7, %%xmm0 \n\t"\
01546         "paddw   %%xmm12,%%xmm7     \n\t"\
01547         "paddw   %%xmm3, %%xmm2     \n\t"\
01548         "paddw   %%xmm8, %%xmm6     \n\t"\
01549         "paddw   %%xmm4, %%xmm1     \n\t"\
01550         "paddw   %%xmm9, %%xmm0     \n\t"\
01551         "psllw   $2,     %%xmm2     \n\t"\
01552         "psllw   $2,     %%xmm6     \n\t"\
01553         "psubw   %%xmm1, %%xmm2     \n\t"\
01554         "psubw   %%xmm0, %%xmm6     \n\t"\
01555         "paddw   %%xmm13,%%xmm11    \n\t"\
01556         "paddw   %%xmm13,%%xmm7     \n\t"\
01557         "pmullw  %%xmm14,%%xmm2     \n\t"\
01558         "pmullw  %%xmm14,%%xmm6     \n\t"\
01559         "lddqu   (%2),   %%xmm3     \n\t"\
01560         "paddw   %%xmm11,%%xmm2     \n\t"\
01561         "paddw   %%xmm7, %%xmm6     \n\t"\
01562         "psraw   $5,     %%xmm2     \n\t"\
01563         "psraw   $5,     %%xmm6     \n\t"\
01564         "packuswb %%xmm2,%%xmm6     \n\t"\
01565         "pavgb   %%xmm3, %%xmm6     \n\t"\
01566         OP(%%xmm6, (%1), %%xmm4, dqa)\
01567         "add %5, %0                 \n\t"\
01568         "add %5, %1                 \n\t"\
01569         "add %4, %2                 \n\t"\
01570         "decl %3                    \n\t"\
01571         "jg 1b                      \n\t"\
01572         : "+a"(src), "+c"(dst), "+d"(src2), "+g"(h)\
01573         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\
01574           "m"(ff_pw_5), "m"(ff_pw_16)\
01575         : "memory"\
01576     );\
01577 }
01578 #else // ARCH_X86_64
01579 #define QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
01580 static av_noinline void OPNAME ## h264_qpel16_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01581     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
01582     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
01583     src += 8*dstStride;\
01584     dst += 8*dstStride;\
01585     src2 += 8*src2Stride;\
01586     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst  , src  , src2  , dstStride, src2Stride);\
01587     OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(dst+8, src+8, src2+8, dstStride, src2Stride);\
01588 }
01589 #endif // ARCH_X86_64
01590 
01591 #define QPEL_H264_H_XMM(OPNAME, OP, MMX)\
01592 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_l2_ ## MMX(uint8_t *dst, uint8_t *src, uint8_t *src2, int dstStride, int src2Stride){\
01593     int h=8;\
01594     __asm__ volatile(\
01595         "pxor %%xmm7, %%xmm7        \n\t"\
01596         "movdqa %0, %%xmm6          \n\t"\
01597         :: "m"(ff_pw_5)\
01598     );\
01599     do{\
01600     __asm__ volatile(\
01601         "lddqu   -2(%0), %%xmm1     \n\t"\
01602         "movdqa  %%xmm1, %%xmm0     \n\t"\
01603         "punpckhbw %%xmm7, %%xmm1   \n\t"\
01604         "punpcklbw %%xmm7, %%xmm0   \n\t"\
01605         "movdqa  %%xmm1, %%xmm2     \n\t"\
01606         "movdqa  %%xmm1, %%xmm3     \n\t"\
01607         "movdqa  %%xmm1, %%xmm4     \n\t"\
01608         "movdqa  %%xmm1, %%xmm5     \n\t"\
01609         "palignr $2, %%xmm0, %%xmm4 \n\t"\
01610         "palignr $4, %%xmm0, %%xmm3 \n\t"\
01611         "palignr $6, %%xmm0, %%xmm2 \n\t"\
01612         "palignr $8, %%xmm0, %%xmm1 \n\t"\
01613         "palignr $10,%%xmm0, %%xmm5 \n\t"\
01614         "paddw   %%xmm5, %%xmm0     \n\t"\
01615         "paddw   %%xmm3, %%xmm2     \n\t"\
01616         "paddw   %%xmm4, %%xmm1     \n\t"\
01617         "psllw   $2,     %%xmm2     \n\t"\
01618         "movq    (%2),   %%xmm3     \n\t"\
01619         "psubw   %%xmm1, %%xmm2     \n\t"\
01620         "paddw   %5,     %%xmm0     \n\t"\
01621         "pmullw  %%xmm6, %%xmm2     \n\t"\
01622         "paddw   %%xmm0, %%xmm2     \n\t"\
01623         "psraw   $5,     %%xmm2     \n\t"\
01624         "packuswb %%xmm2, %%xmm2    \n\t"\
01625         "pavgb   %%xmm3, %%xmm2     \n\t"\
01626         OP(%%xmm2, (%1), %%xmm4, q)\
01627         "add %4, %0                 \n\t"\
01628         "add %4, %1                 \n\t"\
01629         "add %3, %2                 \n\t"\
01630         : "+a"(src), "+c"(dst), "+d"(src2)\
01631         : "D"((x86_reg)src2Stride), "S"((x86_reg)dstStride),\
01632           "m"(ff_pw_16)\
01633         : "memory"\
01634     );\
01635     }while(--h);\
01636 }\
01637 QPEL_H264_H16_XMM(OPNAME, OP, MMX)\
01638 \
01639 static av_noinline void OPNAME ## h264_qpel8_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01640     int h=8;\
01641     __asm__ volatile(\
01642         "pxor %%xmm7, %%xmm7        \n\t"\
01643         "movdqa "MANGLE(ff_pw_5)", %%xmm6\n\t"\
01644         "1:                         \n\t"\
01645         "lddqu   -2(%0), %%xmm1     \n\t"\
01646         "movdqa  %%xmm1, %%xmm0     \n\t"\
01647         "punpckhbw %%xmm7, %%xmm1   \n\t"\
01648         "punpcklbw %%xmm7, %%xmm0   \n\t"\
01649         "movdqa  %%xmm1, %%xmm2     \n\t"\
01650         "movdqa  %%xmm1, %%xmm3     \n\t"\
01651         "movdqa  %%xmm1, %%xmm4     \n\t"\
01652         "movdqa  %%xmm1, %%xmm5     \n\t"\
01653         "palignr $2, %%xmm0, %%xmm4 \n\t"\
01654         "palignr $4, %%xmm0, %%xmm3 \n\t"\
01655         "palignr $6, %%xmm0, %%xmm2 \n\t"\
01656         "palignr $8, %%xmm0, %%xmm1 \n\t"\
01657         "palignr $10,%%xmm0, %%xmm5 \n\t"\
01658         "paddw   %%xmm5, %%xmm0     \n\t"\
01659         "paddw   %%xmm3, %%xmm2     \n\t"\
01660         "paddw   %%xmm4, %%xmm1     \n\t"\
01661         "psllw   $2,     %%xmm2     \n\t"\
01662         "psubw   %%xmm1, %%xmm2     \n\t"\
01663         "paddw   "MANGLE(ff_pw_16)", %%xmm0\n\t"\
01664         "pmullw  %%xmm6, %%xmm2     \n\t"\
01665         "paddw   %%xmm0, %%xmm2     \n\t"\
01666         "psraw   $5,     %%xmm2     \n\t"\
01667         "packuswb %%xmm2, %%xmm2    \n\t"\
01668         OP(%%xmm2, (%1), %%xmm4, q)\
01669         "add %3, %0                 \n\t"\
01670         "add %4, %1                 \n\t"\
01671         "decl %2                    \n\t"\
01672         " jnz 1b                    \n\t"\
01673         : "+a"(src), "+c"(dst), "+g"(h)\
01674         : "D"((x86_reg)srcStride), "S"((x86_reg)dstStride)\
01675         : "memory"\
01676     );\
01677 }\
01678 static void OPNAME ## h264_qpel16_h_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01679     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
01680     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
01681     src += 8*srcStride;\
01682     dst += 8*dstStride;\
01683     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst  , src  , dstStride, srcStride);\
01684     OPNAME ## h264_qpel8_h_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride);\
01685 }\
01686 
01687 #define QPEL_H264_V_XMM(OPNAME, OP, MMX)\
01688 static av_noinline void OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride, int h){\
01689     src -= 2*srcStride;\
01690     \
01691     __asm__ volatile(\
01692         "pxor %%xmm7, %%xmm7        \n\t"\
01693         "movq (%0), %%xmm0          \n\t"\
01694         "add %2, %0                 \n\t"\
01695         "movq (%0), %%xmm1          \n\t"\
01696         "add %2, %0                 \n\t"\
01697         "movq (%0), %%xmm2          \n\t"\
01698         "add %2, %0                 \n\t"\
01699         "movq (%0), %%xmm3          \n\t"\
01700         "add %2, %0                 \n\t"\
01701         "movq (%0), %%xmm4          \n\t"\
01702         "add %2, %0                 \n\t"\
01703         "punpcklbw %%xmm7, %%xmm0   \n\t"\
01704         "punpcklbw %%xmm7, %%xmm1   \n\t"\
01705         "punpcklbw %%xmm7, %%xmm2   \n\t"\
01706         "punpcklbw %%xmm7, %%xmm3   \n\t"\
01707         "punpcklbw %%xmm7, %%xmm4   \n\t"\
01708         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\
01709         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\
01710         QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\
01711         QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\
01712         QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\
01713         QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\
01714         QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\
01715         QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\
01716          \
01717         : "+a"(src), "+c"(dst)\
01718         : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01719         : "memory"\
01720     );\
01721     if(h==16){\
01722         __asm__ volatile(\
01723             QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\
01724             QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\
01725             QPEL_H264V_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, OP)\
01726             QPEL_H264V_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, OP)\
01727             QPEL_H264V_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, OP)\
01728             QPEL_H264V_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, OP)\
01729             QPEL_H264V_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, OP)\
01730             QPEL_H264V_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, OP)\
01731             \
01732             : "+a"(src), "+c"(dst)\
01733             : "S"((x86_reg)srcStride), "D"((x86_reg)dstStride), "m"(ff_pw_5), "m"(ff_pw_16)\
01734             : "memory"\
01735         );\
01736     }\
01737 }\
01738 static void OPNAME ## h264_qpel8_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01739     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 8);\
01740 }\
01741 static av_noinline void OPNAME ## h264_qpel16_v_lowpass_ ## MMX(uint8_t *dst, uint8_t *src, int dstStride, int srcStride){\
01742     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst  , src  , dstStride, srcStride, 16);\
01743     OPNAME ## h264_qpel8or16_v_lowpass_ ## MMX(dst+8, src+8, dstStride, srcStride, 16);\
01744 }
01745 
01746 static av_always_inline void put_h264_qpel8or16_hv1_lowpass_sse2(int16_t *tmp, uint8_t *src, int tmpStride, int srcStride, int size){
01747     int w = (size+8)>>3;
01748     src -= 2*srcStride+2;
01749     while(w--){
01750         __asm__ volatile(
01751             "pxor %%xmm7, %%xmm7        \n\t"
01752             "movq (%0), %%xmm0          \n\t"
01753             "add %2, %0                 \n\t"
01754             "movq (%0), %%xmm1          \n\t"
01755             "add %2, %0                 \n\t"
01756             "movq (%0), %%xmm2          \n\t"
01757             "add %2, %0                 \n\t"
01758             "movq (%0), %%xmm3          \n\t"
01759             "add %2, %0                 \n\t"
01760             "movq (%0), %%xmm4          \n\t"
01761             "add %2, %0                 \n\t"
01762             "punpcklbw %%xmm7, %%xmm0   \n\t"
01763             "punpcklbw %%xmm7, %%xmm1   \n\t"
01764             "punpcklbw %%xmm7, %%xmm2   \n\t"
01765             "punpcklbw %%xmm7, %%xmm3   \n\t"
01766             "punpcklbw %%xmm7, %%xmm4   \n\t"
01767             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 0*48)
01768             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 1*48)
01769             QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 2*48)
01770             QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 3*48)
01771             QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 4*48)
01772             QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 5*48)
01773             QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 6*48)
01774             QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 7*48)
01775             : "+a"(src)
01776             : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)
01777             : "memory"
01778         );
01779         if(size==16){
01780             __asm__ volatile(
01781                 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1,  8*48)
01782                 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2,  9*48)
01783                 QPEL_H264HV_XMM(%%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, 10*48)
01784                 QPEL_H264HV_XMM(%%xmm5, %%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, 11*48)
01785                 QPEL_H264HV_XMM(%%xmm0, %%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, 12*48)
01786                 QPEL_H264HV_XMM(%%xmm1, %%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, 13*48)
01787                 QPEL_H264HV_XMM(%%xmm2, %%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, 14*48)
01788                 QPEL_H264HV_XMM(%%xmm3, %%xmm4, %%xmm5, %%xmm0, %%xmm1, %%xmm2, 15*48)
01789                 : "+a"(src)
01790                 : "c"(tmp), "S"((x86_reg)srcStride), "m"(ff_pw_5), "m"(ff_pw_16)
01791                 : "memory"
01792             );
01793         }
01794         tmp += 8;
01795         src += 8 - (size+5)*srcStride;
01796     }
01797 }
01798 
01799 #define QPEL_H264_HV2_XMM(OPNAME, OP, MMX)\
01800 static av_always_inline void OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, int dstStride, int tmpStride, int size){\
01801     int h = size;\
01802     if(size == 16){\
01803         __asm__ volatile(\
01804             "1:                         \n\t"\
01805             "movdqa 32(%0), %%xmm4      \n\t"\
01806             "movdqa 16(%0), %%xmm5      \n\t"\
01807             "movdqa   (%0), %%xmm7      \n\t"\
01808             "movdqa %%xmm4, %%xmm3      \n\t"\
01809             "movdqa %%xmm4, %%xmm2      \n\t"\
01810             "movdqa %%xmm4, %%xmm1      \n\t"\
01811             "movdqa %%xmm4, %%xmm0      \n\t"\
01812             "palignr $10, %%xmm5, %%xmm0 \n\t"\
01813             "palignr  $8, %%xmm5, %%xmm1 \n\t"\
01814             "palignr  $6, %%xmm5, %%xmm2 \n\t"\
01815             "palignr  $4, %%xmm5, %%xmm3 \n\t"\
01816             "palignr  $2, %%xmm5, %%xmm4 \n\t"\
01817             "paddw  %%xmm5, %%xmm0      \n\t"\
01818             "paddw  %%xmm4, %%xmm1      \n\t"\
01819             "paddw  %%xmm3, %%xmm2      \n\t"\
01820             "movdqa %%xmm5, %%xmm6      \n\t"\
01821             "movdqa %%xmm5, %%xmm4      \n\t"\
01822             "movdqa %%xmm5, %%xmm3      \n\t"\
01823             "palignr  $8, %%xmm7, %%xmm4 \n\t"\
01824             "palignr  $2, %%xmm7, %%xmm6 \n\t"\
01825             "palignr $10, %%xmm7, %%xmm3 \n\t"\
01826             "paddw  %%xmm6, %%xmm4      \n\t"\
01827             "movdqa %%xmm5, %%xmm6      \n\t"\
01828             "palignr  $6, %%xmm7, %%xmm5 \n\t"\
01829             "palignr  $4, %%xmm7, %%xmm6 \n\t"\
01830             "paddw  %%xmm7, %%xmm3      \n\t"\
01831             "paddw  %%xmm6, %%xmm5      \n\t"\
01832             \
01833             "psubw  %%xmm1, %%xmm0      \n\t"\
01834             "psubw  %%xmm4, %%xmm3      \n\t"\
01835             "psraw      $2, %%xmm0      \n\t"\
01836             "psraw      $2, %%xmm3      \n\t"\
01837             "psubw  %%xmm1, %%xmm0      \n\t"\
01838             "psubw  %%xmm4, %%xmm3      \n\t"\
01839             "paddw  %%xmm2, %%xmm0      \n\t"\
01840             "paddw  %%xmm5, %%xmm3      \n\t"\
01841             "psraw      $2, %%xmm0      \n\t"\
01842             "psraw      $2, %%xmm3      \n\t"\
01843             "paddw  %%xmm2, %%xmm0      \n\t"\
01844             "paddw  %%xmm5, %%xmm3      \n\t"\
01845             "psraw      $6, %%xmm0      \n\t"\
01846             "psraw      $6, %%xmm3      \n\t"\
01847             "packuswb %%xmm0, %%xmm3    \n\t"\
01848             OP(%%xmm3, (%1), %%xmm7, dqa)\
01849             "add $48, %0                \n\t"\
01850             "add %3, %1                 \n\t"\
01851             "decl %2                    \n\t"\
01852             " jnz 1b                    \n\t"\
01853             : "+a"(tmp), "+c"(dst), "+g"(h)\
01854             : "S"((x86_reg)dstStride)\
01855             : "memory"\
01856         );\
01857     }else{\
01858         __asm__ volatile(\
01859             "1:                         \n\t"\
01860             "movdqa 16(%0), %%xmm1      \n\t"\
01861             "movdqa   (%0), %%xmm0      \n\t"\
01862             "movdqa %%xmm1, %%xmm2      \n\t"\
01863             "movdqa %%xmm1, %%xmm3      \n\t"\
01864             "movdqa %%xmm1, %%xmm4      \n\t"\
01865             "movdqa %%xmm1, %%xmm5      \n\t"\
01866             "palignr $10, %%xmm0, %%xmm5 \n\t"\
01867             "palignr  $8, %%xmm0, %%xmm4 \n\t"\
01868             "palignr  $6, %%xmm0, %%xmm3 \n\t"\
01869             "palignr  $4, %%xmm0, %%xmm2 \n\t"\
01870             "palignr  $2, %%xmm0, %%xmm1 \n\t"\
01871             "paddw  %%xmm5, %%xmm0      \n\t"\
01872             "paddw  %%xmm4, %%xmm1      \n\t"\
01873             "paddw  %%xmm3, %%xmm2      \n\t"\
01874             "psubw  %%xmm1, %%xmm0      \n\t"\
01875             "psraw      $2, %%xmm0      \n\t"\
01876             "psubw  %%xmm1, %%xmm0      \n\t"\
01877             "paddw  %%xmm2, %%xmm0      \n\t"\
01878             "psraw      $2, %%xmm0      \n\t"\
01879             "paddw  %%xmm2, %%xmm0      \n\t"\
01880             "psraw      $6, %%xmm0      \n\t"\
01881             "packuswb %%xmm0, %%xmm0    \n\t"\
01882             OP(%%xmm0, (%1), %%xmm7, q)\
01883             "add $48, %0                \n\t"\
01884             "add %3, %1                 \n\t"\
01885             "decl %2                    \n\t"\
01886             " jnz 1b                    \n\t"\
01887             : "+a"(tmp), "+c"(dst), "+g"(h)\
01888             : "S"((x86_reg)dstStride)\
01889             : "memory"\
01890         );\
01891     }\
01892 }
01893 
01894 #define QPEL_H264_HV_XMM(OPNAME, OP, MMX)\
01895 static av_noinline void OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride, int size){\
01896           put_h264_qpel8or16_hv1_lowpass_sse2(tmp, src, tmpStride, srcStride, size);\
01897     OPNAME ## h264_qpel8or16_hv2_lowpass_ ## MMX(dst, tmp, dstStride, tmpStride, size);\
01898 }\
01899 static void OPNAME ## h264_qpel8_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\
01900     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 8);\
01901 }\
01902 static void OPNAME ## h264_qpel16_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp, uint8_t *src, int dstStride, int tmpStride, int srcStride){\
01903     OPNAME ## h264_qpel8or16_hv_lowpass_ ## MMX(dst, tmp, src, dstStride, tmpStride, srcStride, 16);\
01904 }\
01905 
01906 #define put_pixels8_l2_sse2 put_pixels8_l2_mmx2
01907 #define avg_pixels8_l2_sse2 avg_pixels8_l2_mmx2
01908 #define put_pixels16_l2_sse2 put_pixels16_l2_mmx2
01909 #define avg_pixels16_l2_sse2 avg_pixels16_l2_mmx2
01910 #define put_pixels8_l2_ssse3 put_pixels8_l2_mmx2
01911 #define avg_pixels8_l2_ssse3 avg_pixels8_l2_mmx2
01912 #define put_pixels16_l2_ssse3 put_pixels16_l2_mmx2
01913 #define avg_pixels16_l2_ssse3 avg_pixels16_l2_mmx2
01914 
01915 #define put_pixels8_l2_shift5_sse2 put_pixels8_l2_shift5_mmx2
01916 #define avg_pixels8_l2_shift5_sse2 avg_pixels8_l2_shift5_mmx2
01917 #define put_pixels16_l2_shift5_sse2 put_pixels16_l2_shift5_mmx2
01918 #define avg_pixels16_l2_shift5_sse2 avg_pixels16_l2_shift5_mmx2
01919 #define put_pixels8_l2_shift5_ssse3 put_pixels8_l2_shift5_mmx2
01920 #define avg_pixels8_l2_shift5_ssse3 avg_pixels8_l2_shift5_mmx2
01921 #define put_pixels16_l2_shift5_ssse3 put_pixels16_l2_shift5_mmx2
01922 #define avg_pixels16_l2_shift5_ssse3 avg_pixels16_l2_shift5_mmx2
01923 
01924 #define put_h264_qpel8_h_lowpass_l2_sse2 put_h264_qpel8_h_lowpass_l2_mmx2
01925 #define avg_h264_qpel8_h_lowpass_l2_sse2 avg_h264_qpel8_h_lowpass_l2_mmx2
01926 #define put_h264_qpel16_h_lowpass_l2_sse2 put_h264_qpel16_h_lowpass_l2_mmx2
01927 #define avg_h264_qpel16_h_lowpass_l2_sse2 avg_h264_qpel16_h_lowpass_l2_mmx2
01928 
01929 #define put_h264_qpel8_v_lowpass_ssse3 put_h264_qpel8_v_lowpass_sse2
01930 #define avg_h264_qpel8_v_lowpass_ssse3 avg_h264_qpel8_v_lowpass_sse2
01931 #define put_h264_qpel16_v_lowpass_ssse3 put_h264_qpel16_v_lowpass_sse2
01932 #define avg_h264_qpel16_v_lowpass_ssse3 avg_h264_qpel16_v_lowpass_sse2
01933 
01934 #define put_h264_qpel8or16_hv2_lowpass_sse2 put_h264_qpel8or16_hv2_lowpass_mmx2
01935 #define avg_h264_qpel8or16_hv2_lowpass_sse2 avg_h264_qpel8or16_hv2_lowpass_mmx2
01936 
01937 #define H264_MC(OPNAME, SIZE, MMX, ALIGN) \
01938 H264_MC_C(OPNAME, SIZE, MMX, ALIGN)\
01939 H264_MC_V(OPNAME, SIZE, MMX, ALIGN)\
01940 H264_MC_H(OPNAME, SIZE, MMX, ALIGN)\
01941 H264_MC_HV(OPNAME, SIZE, MMX, ALIGN)\
01942 
01943 static void put_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){
01944     put_pixels16_sse2(dst, src, stride, 16);
01945 }
01946 static void avg_h264_qpel16_mc00_sse2 (uint8_t *dst, uint8_t *src, int stride){
01947     avg_pixels16_sse2(dst, src, stride, 16);
01948 }
01949 #define put_h264_qpel8_mc00_sse2 put_h264_qpel8_mc00_mmx2
01950 #define avg_h264_qpel8_mc00_sse2 avg_h264_qpel8_mc00_mmx2
01951 
01952 #define H264_MC_C(OPNAME, SIZE, MMX, ALIGN) \
01953 static void OPNAME ## h264_qpel ## SIZE ## _mc00_ ## MMX (uint8_t *dst, uint8_t *src, int stride){\
01954     OPNAME ## pixels ## SIZE ## _ ## MMX(dst, src, stride, SIZE);\
01955 }\
01956 
01957 #define H264_MC_H(OPNAME, SIZE, MMX, ALIGN) \
01958 static void OPNAME ## h264_qpel ## SIZE ## _mc10_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01959     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src, stride, stride);\
01960 }\
01961 \
01962 static void OPNAME ## h264_qpel ## SIZE ## _mc20_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01963     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_ ## MMX(dst, src, stride, stride);\
01964 }\
01965 \
01966 static void OPNAME ## h264_qpel ## SIZE ## _mc30_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01967     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, src+1, stride, stride);\
01968 }\
01969 
01970 #define H264_MC_V(OPNAME, SIZE, MMX, ALIGN) \
01971 static void OPNAME ## h264_qpel ## SIZE ## _mc01_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01972     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
01973     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
01974     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src, temp, stride, stride, SIZE);\
01975 }\
01976 \
01977 static void OPNAME ## h264_qpel ## SIZE ## _mc02_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01978     OPNAME ## h264_qpel ## SIZE ## _v_lowpass_ ## MMX(dst, src, stride, stride);\
01979 }\
01980 \
01981 static void OPNAME ## h264_qpel ## SIZE ## _mc03_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01982     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
01983     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
01984     OPNAME ## pixels ## SIZE ## _l2_ ## MMX(dst, src+stride, temp, stride, stride, SIZE);\
01985 }\
01986 
01987 #define H264_MC_HV(OPNAME, SIZE, MMX, ALIGN) \
01988 static void OPNAME ## h264_qpel ## SIZE ## _mc11_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01989     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
01990     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
01991     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
01992 }\
01993 \
01994 static void OPNAME ## h264_qpel ## SIZE ## _mc31_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
01995     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
01996     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
01997     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, temp, stride, SIZE);\
01998 }\
01999 \
02000 static void OPNAME ## h264_qpel ## SIZE ## _mc13_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02001     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
02002     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src, SIZE, stride);\
02003     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
02004 }\
02005 \
02006 static void OPNAME ## h264_qpel ## SIZE ## _mc33_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02007     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*SIZE];\
02008     put_h264_qpel ## SIZE ## _v_lowpass_ ## MMX(temp, src+1, SIZE, stride);\
02009     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, temp, stride, SIZE);\
02010 }\
02011 \
02012 static void OPNAME ## h264_qpel ## SIZE ## _mc22_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02013     DECLARE_ALIGNED(ALIGN, uint16_t, temp)[SIZE*(SIZE<8?12:24)];\
02014     OPNAME ## h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(dst, temp, src, stride, SIZE, stride);\
02015 }\
02016 \
02017 static void OPNAME ## h264_qpel ## SIZE ## _mc21_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02018     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\
02019     uint8_t * const halfHV= temp;\
02020     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
02021     assert(((int)temp & 7) == 0);\
02022     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
02023     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src, halfHV, stride, SIZE);\
02024 }\
02025 \
02026 static void OPNAME ## h264_qpel ## SIZE ## _mc23_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02027     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\
02028     uint8_t * const halfHV= temp;\
02029     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
02030     assert(((int)temp & 7) == 0);\
02031     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
02032     OPNAME ## h264_qpel ## SIZE ## _h_lowpass_l2_ ## MMX(dst, src+stride, halfHV, stride, SIZE);\
02033 }\
02034 \
02035 static void OPNAME ## h264_qpel ## SIZE ## _mc12_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02036     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\
02037     uint8_t * const halfHV= temp;\
02038     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
02039     assert(((int)temp & 7) == 0);\
02040     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
02041     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+2, halfHV, stride, SIZE, SIZE);\
02042 }\
02043 \
02044 static void OPNAME ## h264_qpel ## SIZE ## _mc32_ ## MMX(uint8_t *dst, uint8_t *src, int stride){\
02045     DECLARE_ALIGNED(ALIGN, uint8_t, temp)[SIZE*(SIZE<8?12:24)*2 + SIZE*SIZE];\
02046     uint8_t * const halfHV= temp;\
02047     int16_t * const halfV= (int16_t*)(temp + SIZE*SIZE);\
02048     assert(((int)temp & 7) == 0);\
02049     put_h264_qpel ## SIZE ## _hv_lowpass_ ## MMX(halfHV, halfV, src, SIZE, SIZE, stride);\
02050     OPNAME ## pixels ## SIZE ## _l2_shift5_ ## MMX(dst, halfV+3, halfHV, stride, SIZE, SIZE);\
02051 }\
02052 
02053 #define H264_MC_4816(MMX)\
02054 H264_MC(put_, 4, MMX, 8)\
02055 H264_MC(put_, 8, MMX, 8)\
02056 H264_MC(put_, 16,MMX, 8)\
02057 H264_MC(avg_, 4, MMX, 8)\
02058 H264_MC(avg_, 8, MMX, 8)\
02059 H264_MC(avg_, 16,MMX, 8)\
02060 
02061 #define H264_MC_816(QPEL, XMM)\
02062 QPEL(put_, 8, XMM, 16)\
02063 QPEL(put_, 16,XMM, 16)\
02064 QPEL(avg_, 8, XMM, 16)\
02065 QPEL(avg_, 16,XMM, 16)\
02066 
02067 
02068 #define AVG_3DNOW_OP(a,b,temp, size) \
02069 "mov" #size " " #b ", " #temp "   \n\t"\
02070 "pavgusb " #temp ", " #a "        \n\t"\
02071 "mov" #size " " #a ", " #b "      \n\t"
02072 #define AVG_MMX2_OP(a,b,temp, size) \
02073 "mov" #size " " #b ", " #temp "   \n\t"\
02074 "pavgb " #temp ", " #a "          \n\t"\
02075 "mov" #size " " #a ", " #b "      \n\t"
02076 
02077 #define PAVGB "pavgusb"
02078 QPEL_H264(put_,       PUT_OP, 3dnow)
02079 QPEL_H264(avg_, AVG_3DNOW_OP, 3dnow)
02080 #undef PAVGB
02081 #define PAVGB "pavgb"
02082 QPEL_H264(put_,       PUT_OP, mmx2)
02083 QPEL_H264(avg_,  AVG_MMX2_OP, mmx2)
02084 QPEL_H264_V_XMM(put_,       PUT_OP, sse2)
02085 QPEL_H264_V_XMM(avg_,  AVG_MMX2_OP, sse2)
02086 QPEL_H264_HV_XMM(put_,       PUT_OP, sse2)
02087 QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, sse2)
02088 #if HAVE_SSSE3
02089 QPEL_H264_H_XMM(put_,       PUT_OP, ssse3)
02090 QPEL_H264_H_XMM(avg_,  AVG_MMX2_OP, ssse3)
02091 QPEL_H264_HV2_XMM(put_,       PUT_OP, ssse3)
02092 QPEL_H264_HV2_XMM(avg_,  AVG_MMX2_OP, ssse3)
02093 QPEL_H264_HV_XMM(put_,       PUT_OP, ssse3)
02094 QPEL_H264_HV_XMM(avg_,  AVG_MMX2_OP, ssse3)
02095 #endif
02096 #undef PAVGB
02097 
02098 H264_MC_4816(3dnow)
02099 H264_MC_4816(mmx2)
02100 H264_MC_816(H264_MC_V, sse2)
02101 H264_MC_816(H264_MC_HV, sse2)
02102 #if HAVE_SSSE3
02103 H264_MC_816(H264_MC_H, ssse3)
02104 H264_MC_816(H264_MC_HV, ssse3)
02105 #endif
02106 
02107 /* rnd interleaved with rnd div 8, use p+1 to access rnd div 8 */
02108 DECLARE_ALIGNED(8, static const uint64_t, h264_rnd_reg)[4] = {
02109     0x0020002000200020ULL, 0x0004000400040004ULL, 0x001C001C001C001CULL, 0x0003000300030003ULL
02110 };
02111 
02112 #define H264_CHROMA_OP(S,D)
02113 #define H264_CHROMA_OP4(S,D,T)
02114 #define H264_CHROMA_MC8_TMPL put_h264_chroma_generic_mc8_mmx
02115 #define H264_CHROMA_MC4_TMPL put_h264_chroma_generic_mc4_mmx
02116 #define H264_CHROMA_MC2_TMPL put_h264_chroma_mc2_mmx2
02117 #define H264_CHROMA_MC8_MV0 put_pixels8_mmx
02118 #include "dsputil_h264_template_mmx.c"
02119 
02120 static void put_h264_chroma_mc8_mmx_rnd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02121 {
02122     put_h264_chroma_generic_mc8_mmx(dst, src, stride, h, x, y, h264_rnd_reg);
02123 }
02124 static void put_vc1_chroma_mc8_mmx_nornd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02125 {
02126     put_h264_chroma_generic_mc8_mmx(dst, src, stride, h, x, y, h264_rnd_reg+2);
02127 }
02128 static void put_h264_chroma_mc4_mmx(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02129 {
02130     put_h264_chroma_generic_mc4_mmx(dst, src, stride, h, x, y, h264_rnd_reg);
02131 }
02132 
02133 #undef H264_CHROMA_OP
02134 #undef H264_CHROMA_OP4
02135 #undef H264_CHROMA_MC8_TMPL
02136 #undef H264_CHROMA_MC4_TMPL
02137 #undef H264_CHROMA_MC2_TMPL
02138 #undef H264_CHROMA_MC8_MV0
02139 
02140 #define H264_CHROMA_OP(S,D) "pavgb " #S ", " #D " \n\t"
02141 #define H264_CHROMA_OP4(S,D,T) "movd  " #S ", " #T " \n\t"\
02142                                "pavgb " #T ", " #D " \n\t"
02143 #define H264_CHROMA_MC8_TMPL avg_h264_chroma_generic_mc8_mmx2
02144 #define H264_CHROMA_MC4_TMPL avg_h264_chroma_generic_mc4_mmx2
02145 #define H264_CHROMA_MC2_TMPL avg_h264_chroma_mc2_mmx2
02146 #define H264_CHROMA_MC8_MV0 avg_pixels8_mmx2
02147 #include "dsputil_h264_template_mmx.c"
02148 static void avg_h264_chroma_mc8_mmx2_rnd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02149 {
02150     avg_h264_chroma_generic_mc8_mmx2(dst, src, stride, h, x, y, h264_rnd_reg);
02151 }
02152 static void avg_vc1_chroma_mc8_mmx2_nornd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02153 {
02154     avg_h264_chroma_generic_mc8_mmx2(dst, src, stride, h, x, y, h264_rnd_reg+2);
02155 }
02156 static void avg_h264_chroma_mc4_mmx2(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02157 {
02158     avg_h264_chroma_generic_mc4_mmx2(dst, src, stride, h, x, y, h264_rnd_reg);
02159 }
02160 #undef H264_CHROMA_OP
02161 #undef H264_CHROMA_OP4
02162 #undef H264_CHROMA_MC8_TMPL
02163 #undef H264_CHROMA_MC4_TMPL
02164 #undef H264_CHROMA_MC2_TMPL
02165 #undef H264_CHROMA_MC8_MV0
02166 
02167 #define H264_CHROMA_OP(S,D) "pavgusb " #S ", " #D " \n\t"
02168 #define H264_CHROMA_OP4(S,D,T) "movd " #S ", " #T " \n\t"\
02169                                "pavgusb " #T ", " #D " \n\t"
02170 #define H264_CHROMA_MC8_TMPL avg_h264_chroma_generic_mc8_3dnow
02171 #define H264_CHROMA_MC4_TMPL avg_h264_chroma_generic_mc4_3dnow
02172 #define H264_CHROMA_MC8_MV0 avg_pixels8_3dnow
02173 #include "dsputil_h264_template_mmx.c"
02174 static void avg_h264_chroma_mc8_3dnow_rnd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02175 {
02176     avg_h264_chroma_generic_mc8_3dnow(dst, src, stride, h, x, y, h264_rnd_reg);
02177 }
02178 static void avg_h264_chroma_mc4_3dnow(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02179 {
02180     avg_h264_chroma_generic_mc4_3dnow(dst, src, stride, h, x, y, h264_rnd_reg);
02181 }
02182 #undef H264_CHROMA_OP
02183 #undef H264_CHROMA_OP4
02184 #undef H264_CHROMA_MC8_TMPL
02185 #undef H264_CHROMA_MC4_TMPL
02186 #undef H264_CHROMA_MC8_MV0
02187 
02188 #if HAVE_SSSE3
02189 #define AVG_OP(X)
02190 #undef H264_CHROMA_MC8_TMPL
02191 #undef H264_CHROMA_MC4_TMPL
02192 #define H264_CHROMA_MC8_TMPL put_h264_chroma_mc8_ssse3
02193 #define H264_CHROMA_MC4_TMPL put_h264_chroma_mc4_ssse3
02194 #define H264_CHROMA_MC8_MV0 put_pixels8_mmx
02195 #include "dsputil_h264_template_ssse3.c"
02196 static void put_h264_chroma_mc8_ssse3_rnd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02197 {
02198     put_h264_chroma_mc8_ssse3(dst, src, stride, h, x, y, 1);
02199 }
02200 static void put_vc1_chroma_mc8_ssse3_nornd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02201 {
02202     put_h264_chroma_mc8_ssse3(dst, src, stride, h, x, y, 0);
02203 }
02204 
02205 #undef AVG_OP
02206 #undef H264_CHROMA_MC8_TMPL
02207 #undef H264_CHROMA_MC4_TMPL
02208 #undef H264_CHROMA_MC8_MV0
02209 #define AVG_OP(X) X
02210 #define H264_CHROMA_MC8_TMPL avg_h264_chroma_mc8_ssse3
02211 #define H264_CHROMA_MC4_TMPL avg_h264_chroma_mc4_ssse3
02212 #define H264_CHROMA_MC8_MV0 avg_pixels8_mmx2
02213 #include "dsputil_h264_template_ssse3.c"
02214 static void avg_h264_chroma_mc8_ssse3_rnd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02215 {
02216     avg_h264_chroma_mc8_ssse3(dst, src, stride, h, x, y, 1);
02217 }
02218 static void avg_vc1_chroma_mc8_ssse3_nornd(uint8_t *dst/*align 8*/, uint8_t *src/*align 1*/, int stride, int h, int x, int y)
02219 {
02220     avg_h264_chroma_mc8_ssse3(dst, src, stride, h, x, y, 0);
02221 }
02222 #undef AVG_OP
02223 #undef H264_CHROMA_MC8_TMPL
02224 #undef H264_CHROMA_MC4_TMPL
02225 #undef H264_CHROMA_MC8_MV0
02226 #endif
02227 
02228 /***********************************/
02229 /* weighted prediction */
02230 
02231 static inline void ff_h264_weight_WxH_mmx2(uint8_t *dst, int stride, int log2_denom, int weight, int offset, int w, int h)
02232 {
02233     int x, y;
02234     offset <<= log2_denom;
02235     offset += (1 << log2_denom) >> 1;
02236     __asm__ volatile(
02237         "movd    %0, %%mm4        \n\t"
02238         "movd    %1, %%mm5        \n\t"
02239         "movd    %2, %%mm6        \n\t"
02240         "pshufw  $0, %%mm4, %%mm4 \n\t"
02241         "pshufw  $0, %%mm5, %%mm5 \n\t"
02242         "pxor    %%mm7, %%mm7     \n\t"
02243         :: "g"(weight), "g"(offset), "g"(log2_denom)
02244     );
02245     for(y=0; y<h; y+=2){
02246         for(x=0; x<w; x+=4){
02247             __asm__ volatile(
02248                 "movd      %0,    %%mm0 \n\t"
02249                 "movd      %1,    %%mm1 \n\t"
02250                 "punpcklbw %%mm7, %%mm0 \n\t"
02251                 "punpcklbw %%mm7, %%mm1 \n\t"
02252                 "pmullw    %%mm4, %%mm0 \n\t"
02253                 "pmullw    %%mm4, %%mm1 \n\t"
02254                 "paddsw    %%mm5, %%mm0 \n\t"
02255                 "paddsw    %%mm5, %%mm1 \n\t"
02256                 "psraw     %%mm6, %%mm0 \n\t"
02257                 "psraw     %%mm6, %%mm1 \n\t"
02258                 "packuswb  %%mm7, %%mm0 \n\t"
02259                 "packuswb  %%mm7, %%mm1 \n\t"
02260                 "movd      %%mm0, %0    \n\t"
02261                 "movd      %%mm1, %1    \n\t"
02262                 : "+m"(*(uint32_t*)(dst+x)),
02263                   "+m"(*(uint32_t*)(dst+x+stride))
02264             );
02265         }
02266         dst += 2*stride;
02267     }
02268 }
02269 
02270 static inline void ff_h264_biweight_WxH_mmx2(uint8_t *dst, uint8_t *src, int stride, int log2_denom, int weightd, int weights, int offset, int w, int h)
02271 {
02272     int x, y;
02273     offset = ((offset + 1) | 1) << log2_denom;
02274     __asm__ volatile(
02275         "movd    %0, %%mm3        \n\t"
02276         "movd    %1, %%mm4        \n\t"
02277         "movd    %2, %%mm5        \n\t"
02278         "movd    %3, %%mm6        \n\t"
02279         "pshufw  $0, %%mm3, %%mm3 \n\t"
02280         "pshufw  $0, %%mm4, %%mm4 \n\t"
02281         "pshufw  $0, %%mm5, %%mm5 \n\t"
02282         "pxor    %%mm7, %%mm7     \n\t"
02283         :: "g"(weightd), "g"(weights), "g"(offset), "g"(log2_denom+1)
02284     );
02285     for(y=0; y<h; y++){
02286         for(x=0; x<w; x+=4){
02287             __asm__ volatile(
02288                 "movd      %0,    %%mm0 \n\t"
02289                 "movd      %1,    %%mm1 \n\t"
02290                 "punpcklbw %%mm7, %%mm0 \n\t"
02291                 "punpcklbw %%mm7, %%mm1 \n\t"
02292                 "pmullw    %%mm3, %%mm0 \n\t"
02293                 "pmullw    %%mm4, %%mm1 \n\t"
02294                 "paddsw    %%mm1, %%mm0 \n\t"
02295                 "paddsw    %%mm5, %%mm0 \n\t"
02296                 "psraw     %%mm6, %%mm0 \n\t"
02297                 "packuswb  %%mm0, %%mm0 \n\t"
02298                 "movd      %%mm0, %0    \n\t"
02299                 : "+m"(*(uint32_t*)(dst+x))
02300                 :  "m"(*(uint32_t*)(src+x))
02301             );
02302         }
02303         src += stride;
02304         dst += stride;
02305     }
02306 }
02307 
02308 #define H264_WEIGHT(W,H) \
02309 static void ff_h264_biweight_ ## W ## x ## H ## _mmx2(uint8_t *dst, uint8_t *src, int stride, int log2_denom, int weightd, int weights, int offset){ \
02310     ff_h264_biweight_WxH_mmx2(dst, src, stride, log2_denom, weightd, weights, offset, W, H); \
02311 } \
02312 static void ff_h264_weight_ ## W ## x ## H ## _mmx2(uint8_t *dst, int stride, int log2_denom, int weight, int offset){ \
02313     ff_h264_weight_WxH_mmx2(dst, stride, log2_denom, weight, offset, W, H); \
02314 }
02315 
02316 H264_WEIGHT(16,16)
02317 H264_WEIGHT(16, 8)
02318 H264_WEIGHT( 8,16)
02319 H264_WEIGHT( 8, 8)
02320 H264_WEIGHT( 8, 4)
02321 H264_WEIGHT( 4, 8)
02322 H264_WEIGHT( 4, 4)
02323 H264_WEIGHT( 4, 2)
02324
libavcodec/x86/h264dsp_mmx.c
