blob: 508532b9d86f7296edfd75652cd4a3986182317a [file] [log] [blame]
/*
* Copyright (c) 2022 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "./vpx_dsp_rtcd.h"
#include "vpx_dsp/loongarch/fwd_txfm_lsx.h"
#define LSX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \
do { \
__m128i _s0, _s1, _s2, _s3, _t0, _t1, _t2, _t3; \
\
DUP2_ARG2(__lsx_vilvl_h, _in2, _in0, _in3, _in1, _s0, _s1); \
DUP2_ARG2(__lsx_vilvh_h, _in2, _in0, _in3, _in1, _s2, _s3); \
_t0 = __lsx_vilvl_h(_s1, _s0); \
_t1 = __lsx_vilvh_h(_s1, _s0); \
_t2 = __lsx_vilvl_h(_s3, _s2); \
_t3 = __lsx_vilvh_h(_s3, _s2); \
DUP2_ARG2(__lsx_vpickev_d, _t2, _t0, _t3, _t1, _out0, _out2); \
DUP2_ARG2(__lsx_vpickod_d, _t2, _t0, _t3, _t1, _out1, _out3); \
} while (0)
#if !CONFIG_VP9_HIGHBITDEPTH
void fdct8x16_1d_column(const int16_t *input, int16_t *tmp_ptr,
int32_t src_stride) {
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
__m128i in8, in9, in10, in11, in12, in13, in14, in15;
__m128i stp21, stp22, stp23, stp24, stp25, stp26, stp30;
__m128i stp31, stp32, stp33, stp34, stp35, stp36, stp37;
__m128i vec0, vec1, vec2, vec3, vec4, vec5, cnst0, cnst1, cnst4, cnst5;
__m128i coeff = { 0x187e3b21d2bf2d41, 0x238e3537e782c4df };
__m128i coeff1 = { 0x289a317906463fb1, 0x12943d3f1e2b3871 };
__m128i coeff2 = { 0xed6cd766c78fc04f, 0x0 };
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
int32_t src_stride8 = src_stride4 << 1;
int16_t *input_tmp = (int16_t *)input;
in0 = __lsx_vld(input_tmp, 0);
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in1, in2, in3, in4);
input_tmp += src_stride4;
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in5, in6, in7, in8);
input_tmp += src_stride4;
DUP4_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4,
input_tmp, src_stride6, input_tmp, src_stride8, in9, in10, in11,
in12);
input_tmp += src_stride4;
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in13,
in14);
input_tmp += src_stride2;
in15 = __lsx_vldx(input_tmp, src_stride2);
DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vslli_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vslli_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14,
in15);
DUP4_ARG2(__lsx_vadd_h, in0, in15, in1, in14, in2, in13, in3, in12, tmp0,
tmp1, tmp2, tmp3);
DUP4_ARG2(__lsx_vadd_h, in4, in11, in5, in10, in6, in9, in7, in8, tmp4, tmp5,
tmp6, tmp7);
FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1,
tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
__lsx_vst(tmp0, tmp_ptr, 0);
__lsx_vst(tmp1, tmp_ptr, 64);
__lsx_vst(tmp2, tmp_ptr, 128);
__lsx_vst(tmp3, tmp_ptr, 192);
__lsx_vst(tmp4, tmp_ptr, 256);
__lsx_vst(tmp5, tmp_ptr, 320);
__lsx_vst(tmp6, tmp_ptr, 384);
__lsx_vst(tmp7, tmp_ptr, 448);
DUP4_ARG2(__lsx_vsub_h, in0, in15, in1, in14, in2, in13, in3, in12, in15,
in14, in13, in12);
DUP4_ARG2(__lsx_vsub_h, in4, in11, in5, in10, in6, in9, in7, in8, in11, in10,
in9, in8);
tmp_ptr += 16;
/* stp 1 */
DUP2_ARG2(__lsx_vilvh_h, in10, in13, in11, in12, vec2, vec4);
DUP2_ARG2(__lsx_vilvl_h, in10, in13, in11, in12, vec3, vec5);
cnst4 = __lsx_vreplvei_h(coeff, 0);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst4, stp25);
cnst5 = __lsx_vreplvei_h(coeff, 1);
cnst5 = __lsx_vpackev_h(cnst5, cnst4);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst5, stp22);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst4, stp24);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst5, stp23);
/* stp2 */
LSX_BUTTERFLY_4_H(in8, in9, stp22, stp23, stp30, stp31, stp32, stp33);
LSX_BUTTERFLY_4_H(in15, in14, stp25, stp24, stp37, stp36, stp35, stp34);
DUP2_ARG2(__lsx_vilvh_h, stp36, stp31, stp35, stp32, vec2, vec4);
DUP2_ARG2(__lsx_vilvl_h, stp36, stp31, stp35, stp32, vec3, vec5);
DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 3, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst0, stp26);
cnst0 = __lsx_vreplvei_h(coeff, 4);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec2, vec3, cnst1, stp21);
LSX_BUTTERFLY_4_H(stp30, stp37, stp26, stp21, in8, in15, in14, in9);
vec1 = __lsx_vilvl_h(in15, in8);
vec0 = __lsx_vilvh_h(in15, in8);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 0, coeff1, 1, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 0);
cnst0 = __lsx_vreplvei_h(coeff2, 0);
cnst0 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 448);
vec1 = __lsx_vilvl_h(in14, in9);
vec0 = __lsx_vilvh_h(in14, in9);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 2, coeff1, 3, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8);
__lsx_vst(in8, tmp_ptr, 256);
cnst1 = __lsx_vreplvei_h(coeff2, 2);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 192);
DUP2_ARG2(__lsx_vreplvei_h, coeff, 2, coeff, 5, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp25);
cnst1 = __lsx_vreplvei_h(coeff, 3);
cnst1 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec4, vec5, cnst1, stp22);
/* stp4 */
DUP2_ARG2(__lsx_vadd_h, stp34, stp25, stp33, stp22, in13, in10);
vec1 = __lsx_vilvl_h(in13, in10);
vec0 = __lsx_vilvh_h(in13, in10);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 4, coeff1, 5, cnst0, cnst1);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 128);
cnst0 = __lsx_vreplvei_h(coeff2, 1);
cnst0 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 320);
DUP2_ARG2(__lsx_vsub_h, stp34, stp25, stp33, stp22, in12, in11);
vec1 = __lsx_vilvl_h(in12, in11);
vec0 = __lsx_vilvh_h(in12, in11);
DUP2_ARG2(__lsx_vreplvei_h, coeff1, 6, coeff1, 7, cnst0, cnst1);
cnst1 = __lsx_vpackev_h(cnst1, cnst0);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst1, in8);
__lsx_vst(in8, tmp_ptr, 384);
cnst1 = __lsx_vreplvei_h(coeff2, 3);
cnst0 = __lsx_vpackev_h(cnst0, cnst1);
DOT_SHIFT_RIGHT_PCK_H(vec0, vec1, cnst0, in8);
__lsx_vst(in8, tmp_ptr, 64);
}
void fdct16x8_1d_row(int16_t *input, int16_t *output) {
__m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
__m128i in8, in9, in10, in11, in12, in13, in14, in15;
int16_t *input_tmp = input;
DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in0, in1, in2,
in3);
DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in4, in5,
in6, in7);
DUP4_ARG2(__lsx_vld, input_tmp, 16, input_tmp, 48, input_tmp, 80, input_tmp,
112, in8, in9, in10, in11);
DUP4_ARG2(__lsx_vld, input_tmp, 144, input_tmp, 176, input_tmp, 208,
input_tmp, 240, in12, in13, in14, in15);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
LSX_TRANSPOSE8x8_H(in8, in9, in10, in11, in12, in13, in14, in15, in8, in9,
in10, in11, in12, in13, in14, in15);
DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vaddi_hu, in4, 1, in5, 1, in6, 1, in7, 1, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vaddi_hu, in8, 1, in9, 1, in10, 1, in11, 1, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vaddi_hu, in12, 1, in13, 1, in14, 1, in15, 1, in12, in13,
in14, in15);
DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vsrai_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
DUP4_ARG2(__lsx_vsrai_h, in8, 2, in9, 2, in10, 2, in11, 2, in8, in9, in10,
in11);
DUP4_ARG2(__lsx_vsrai_h, in12, 2, in13, 2, in14, 2, in15, 2, in12, in13, in14,
in15);
LSX_BUTTERFLY_16_H(in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10,
in11, in12, in13, in14, in15, tmp0, tmp1, tmp2, tmp3, tmp4,
tmp5, tmp6, tmp7, in8, in9, in10, in11, in12, in13, in14,
in15);
__lsx_vst(in8, input, 0);
__lsx_vst(in9, input, 32);
__lsx_vst(in10, input, 64);
__lsx_vst(in11, input, 96);
__lsx_vst(in12, input, 128);
__lsx_vst(in13, input, 160);
__lsx_vst(in14, input, 192);
__lsx_vst(in15, input, 224);
FDCT8x16_EVEN(tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp0, tmp1,
tmp2, tmp3, tmp4, tmp5, tmp6, tmp7);
DUP4_ARG2(__lsx_vld, input, 0, input, 32, input, 64, input, 96, in8, in9,
in10, in11);
DUP4_ARG2(__lsx_vld, input, 128, input, 160, input, 192, input, 224, in12,
in13, in14, in15);
FDCT8x16_ODD(in8, in9, in10, in11, in12, in13, in14, in15, in0, in1, in2, in3,
in4, in5, in6, in7);
LSX_TRANSPOSE8x8_H(tmp0, in0, tmp1, in1, tmp2, in2, tmp3, in3, tmp0, in0,
tmp1, in1, tmp2, in2, tmp3, in3);
__lsx_vst(tmp0, output, 0);
__lsx_vst(in0, output, 32);
__lsx_vst(tmp1, output, 64);
__lsx_vst(in1, output, 96);
__lsx_vst(tmp2, output, 128);
__lsx_vst(in2, output, 160);
__lsx_vst(tmp3, output, 192);
__lsx_vst(in3, output, 224);
LSX_TRANSPOSE8x8_H(tmp4, in4, tmp5, in5, tmp6, in6, tmp7, in7, tmp4, in4,
tmp5, in5, tmp6, in6, tmp7, in7);
__lsx_vst(tmp4, output, 16);
__lsx_vst(in4, output, 48);
__lsx_vst(tmp5, output, 80);
__lsx_vst(in5, output, 112);
__lsx_vst(tmp6, output, 144);
__lsx_vst(in6, output, 176);
__lsx_vst(tmp7, output, 208);
__lsx_vst(in7, output, 240);
}
void vpx_fdct4x4_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
__m128i in0, in1, in2, in3;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
in0 = __lsx_vld(input, 0);
DUP2_ARG2(__lsx_vldx, input, src_stride2, input, src_stride4, in1, in2);
in3 = __lsx_vldx(input, src_stride6);
/* fdct4 pre-process */
{
__m128i vec, mask;
__m128i zero = __lsx_vldi(0);
mask = __lsx_vinsgr2vr_b(zero, 1, 0);
DUP4_ARG2(__lsx_vslli_h, in0, 4, in1, 4, in2, 4, in3, 4, in0, in1, in2,
in3);
vec = __lsx_vseqi_h(in0, 0);
vec = __lsx_vxori_b(vec, 255);
vec = __lsx_vand_v(mask, vec);
in0 = __lsx_vadd_h(in0, vec);
}
VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3);
LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3);
VP9_FDCT4(in0, in1, in2, in3, in0, in1, in2, in3);
LSX_TRANSPOSE4x4_H(in0, in1, in2, in3, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vaddi_hu, in0, 1, in1, 1, in2, 1, in3, 1, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vsrai_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP2_ARG2(__lsx_vpickev_d, in1, in0, in3, in2, in0, in2);
__lsx_vst(in0, output, 0);
__lsx_vst(in2, output, 16);
}
void vpx_fdct8x8_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
__m128i in0, in1, in2, in3, in4, in5, in6, in7;
int32_t src_stride2 = src_stride << 1;
int32_t src_stride4 = src_stride2 << 1;
int32_t src_stride6 = src_stride4 + src_stride2;
int16_t *input_tmp = (int16_t *)input;
in0 = __lsx_vld(input_tmp, 0);
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in1,
in2);
in3 = __lsx_vldx(input_tmp, src_stride6);
input_tmp += src_stride4;
in4 = __lsx_vld(input_tmp, 0);
DUP2_ARG2(__lsx_vldx, input_tmp, src_stride2, input_tmp, src_stride4, in5,
in6);
in7 = __lsx_vldx(input_tmp, src_stride6);
DUP4_ARG2(__lsx_vslli_h, in0, 2, in1, 2, in2, 2, in3, 2, in0, in1, in2, in3);
DUP4_ARG2(__lsx_vslli_h, in4, 2, in5, 2, in6, 2, in7, 2, in4, in5, in6, in7);
VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4,
in5, in6, in7);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
VP9_FDCT8(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3, in4,
in5, in6, in7);
LSX_TRANSPOSE8x8_H(in0, in1, in2, in3, in4, in5, in6, in7, in0, in1, in2, in3,
in4, in5, in6, in7);
SRLI_AVE_S_4V_H(in0, in1, in2, in3, in4, in5, in6, in7);
__lsx_vst(in0, output, 0);
__lsx_vst(in1, output, 16);
__lsx_vst(in2, output, 32);
__lsx_vst(in3, output, 48);
__lsx_vst(in4, output, 64);
__lsx_vst(in5, output, 80);
__lsx_vst(in6, output, 96);
__lsx_vst(in7, output, 112);
}
void vpx_fdct16x16_lsx(const int16_t *input, int16_t *output,
int32_t src_stride) {
int32_t i;
DECLARE_ALIGNED(32, int16_t, tmp_buf[16 * 16]);
/* column transform */
for (i = 0; i < 2; ++i) {
fdct8x16_1d_column((input + 8 * i), (&tmp_buf[0] + 8 * i), src_stride);
}
/* row transform */
for (i = 0; i < 2; ++i) {
fdct16x8_1d_row((&tmp_buf[0] + (128 * i)), (output + (128 * i)));
}
}
#endif // !CONFIG_VP9_HIGHBITDEPTH