in modules/imgproc/src/imgwarp.cpp [5709:6031]
virtual void operator() (const Range& range) const
{
const int BLOCK_SZ = 32;
short XY[BLOCK_SZ*BLOCK_SZ*2], A[BLOCK_SZ*BLOCK_SZ];
int x, y, x1, y1, width = dst.cols, height = dst.rows;
int bh0 = std::min(BLOCK_SZ/2, height);
int bw0 = std::min(BLOCK_SZ*BLOCK_SZ/bh0, width);
bh0 = std::min(BLOCK_SZ*BLOCK_SZ/bw0, height);
#if CV_SSE4_1
bool haveSSE4_1 = checkHardwareSupport(CV_CPU_SSE4_1);
__m128d v_M0 = _mm_set1_pd(M[0]);
__m128d v_M3 = _mm_set1_pd(M[3]);
__m128d v_M6 = _mm_set1_pd(M[6]);
__m128d v_intmax = _mm_set1_pd((double)INT_MAX);
__m128d v_intmin = _mm_set1_pd((double)INT_MIN);
__m128d v_2 = _mm_set1_pd(2),
v_zero = _mm_setzero_pd(),
v_1 = _mm_set1_pd(1),
v_its = _mm_set1_pd(INTER_TAB_SIZE);
__m128i v_itsi1 = _mm_set1_epi32(INTER_TAB_SIZE - 1);
#endif
for( y = range.start; y < range.end; y += bh0 )
{
for( x = 0; x < width; x += bw0 )
{
int bw = std::min( bw0, width - x);
int bh = std::min( bh0, range.end - y); // height
Mat _XY(bh, bw, CV_16SC2, XY), matA;
Mat dpart(dst, Rect(x, y, bw, bh));
for( y1 = 0; y1 < bh; y1++ )
{
short* xy = XY + y1*bw*2;
double X0 = M[0]*x + M[1]*(y + y1) + M[2];
double Y0 = M[3]*x + M[4]*(y + y1) + M[5];
double W0 = M[6]*x + M[7]*(y + y1) + M[8];
if( interpolation == INTER_NEAREST )
{
x1 = 0;
#if CV_SSE4_1
if (haveSSE4_1)
{
__m128d v_X0d = _mm_set1_pd(X0);
__m128d v_Y0d = _mm_set1_pd(Y0);
__m128d v_W0 = _mm_set1_pd(W0);
__m128d v_x1 = _mm_set_pd(1, 0);
for( ; x1 <= bw - 16; x1 += 16 )
{
// 0-3
__m128i v_X0, v_Y0;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 4-8
__m128i v_X1, v_Y1;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 8-11
__m128i v_X2, v_Y2;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 12-15
__m128i v_X3, v_Y3;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_1, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// convert to 16s
v_X0 = _mm_packs_epi32(v_X0, v_X1);
v_X1 = _mm_packs_epi32(v_X2, v_X3);
v_Y0 = _mm_packs_epi32(v_Y0, v_Y1);
v_Y1 = _mm_packs_epi32(v_Y2, v_Y3);
_mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1);
_mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1);
}
}
#endif
for( ; x1 < bw; x1++ )
{
double W = W0 + M[6]*x1;
W = W ? 1./W : 0;
double fX = std::max((double)INT_MIN, std::min((double)INT_MAX, (X0 + M[0]*x1)*W));
double fY = std::max((double)INT_MIN, std::min((double)INT_MAX, (Y0 + M[3]*x1)*W));
int X = saturate_cast<int>(fX);
int Y = saturate_cast<int>(fY);
xy[x1*2] = saturate_cast<short>(X);
xy[x1*2+1] = saturate_cast<short>(Y);
}
}
else
{
short* alpha = A + y1*bw;
x1 = 0;
#if CV_SSE4_1
if (haveSSE4_1)
{
__m128d v_X0d = _mm_set1_pd(X0);
__m128d v_Y0d = _mm_set1_pd(Y0);
__m128d v_W0 = _mm_set1_pd(W0);
__m128d v_x1 = _mm_set_pd(1, 0);
for( ; x1 <= bw - 16; x1 += 16 )
{
// 0-3
__m128i v_X0, v_Y0;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y0 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 4-8
__m128i v_X1, v_Y1;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y1 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 8-11
__m128i v_X2, v_Y2;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y2 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// 12-15
__m128i v_X3, v_Y3;
{
__m128d v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY0 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_W = _mm_add_pd(_mm_mul_pd(v_M6, v_x1), v_W0);
v_W = _mm_andnot_pd(_mm_cmpeq_pd(v_W, v_zero), _mm_div_pd(v_its, v_W));
__m128d v_fX1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_X0d, _mm_mul_pd(v_M0, v_x1)), v_W)));
__m128d v_fY1 = _mm_max_pd(v_intmin, _mm_min_pd(v_intmax, _mm_mul_pd(_mm_add_pd(v_Y0d, _mm_mul_pd(v_M3, v_x1)), v_W)));
v_x1 = _mm_add_pd(v_x1, v_2);
v_X3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fX1))));
v_Y3 = _mm_castps_si128(_mm_movelh_ps(_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY0)),
_mm_castsi128_ps(_mm_cvtpd_epi32(v_fY1))));
}
// store alpha
__m128i v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y0, v_itsi1), INTER_BITS),
_mm_and_si128(v_X0, v_itsi1));
__m128i v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y1, v_itsi1), INTER_BITS),
_mm_and_si128(v_X1, v_itsi1));
_mm_storeu_si128((__m128i *)(alpha + x1), _mm_packs_epi32(v_alpha0, v_alpha1));
v_alpha0 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y2, v_itsi1), INTER_BITS),
_mm_and_si128(v_X2, v_itsi1));
v_alpha1 = _mm_add_epi32(_mm_slli_epi32(_mm_and_si128(v_Y3, v_itsi1), INTER_BITS),
_mm_and_si128(v_X3, v_itsi1));
_mm_storeu_si128((__m128i *)(alpha + x1 + 8), _mm_packs_epi32(v_alpha0, v_alpha1));
// convert to 16s
v_X0 = _mm_packs_epi32(_mm_srai_epi32(v_X0, INTER_BITS), _mm_srai_epi32(v_X1, INTER_BITS));
v_X1 = _mm_packs_epi32(_mm_srai_epi32(v_X2, INTER_BITS), _mm_srai_epi32(v_X3, INTER_BITS));
v_Y0 = _mm_packs_epi32(_mm_srai_epi32(v_Y0, INTER_BITS), _mm_srai_epi32(v_Y1, INTER_BITS));
v_Y1 = _mm_packs_epi32(_mm_srai_epi32(v_Y2, INTER_BITS), _mm_srai_epi32(v_Y3, INTER_BITS));
_mm_interleave_epi16(v_X0, v_X1, v_Y0, v_Y1);
_mm_storeu_si128((__m128i *)(xy + x1 * 2), v_X0);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 8), v_X1);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 16), v_Y0);
_mm_storeu_si128((__m128i *)(xy + x1 * 2 + 24), v_Y1);
}
}
#endif
for( ; x1 < bw; x1++ )
{
double W = W0 + M[6]*x1;
W = W ? INTER_TAB_SIZE/W : 0;
double fX = std::max((double)INT_MIN, std::min((double)INT_MAX, (X0 + M[0]*x1)*W));
double fY = std::max((double)INT_MIN, std::min((double)INT_MAX, (Y0 + M[3]*x1)*W));
int X = saturate_cast<int>(fX);
int Y = saturate_cast<int>(fY);
xy[x1*2] = saturate_cast<short>(X >> INTER_BITS);
xy[x1*2+1] = saturate_cast<short>(Y >> INTER_BITS);
alpha[x1] = (short)((Y & (INTER_TAB_SIZE-1))*INTER_TAB_SIZE +
(X & (INTER_TAB_SIZE-1)));
}
}
}
if( interpolation == INTER_NEAREST )
remap( src, dpart, _XY, Mat(), interpolation, borderType, borderValue );
else
{
Mat _matA(bh, bw, CV_16U, A);
remap( src, dpart, _XY, _matA, interpolation, borderType, borderValue );
}
}
}
}