| // Copyright 2020 Google LLC |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // https://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| // ----------------------------------------------------------------------------- |
| // |
| // Custom-CSP critical functions |
| // |
| // Author: Skal (pascal.massimino@gmail.com) |
| |
| #include <cassert> |
| #include <cstdint> |
| |
| #include "src/dsp/dsp.h" |
| #include "src/dsp/math.h" |
| #include "src/wp2/format_constants.h" |
| |
| namespace { |
| |
| constexpr uint32_t kMtxShift = 12u; |
| |
| //------------------------------------------------------------------------------ |
| |
| // Converts a row of 'width' pixels from the 'y,u,v' planes into the |
| // 'dst0,dst1,dst2' planes using an 'offset' then a 'mtx' multiplication. |
| void YuvToCustom_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t offset[3], const int16_t mtx[9], int16_t* dst0, |
| int16_t* dst1, int16_t* dst2, uint32_t width) { |
| for (uint32_t x = 0; x < width; ++x) { |
| // TODO(yguyon): No-overflow guaranteed only if custom ccsp is kYuvMaxPrec |
| int32_t ccsp[3]; |
| WP2::Multiply(y[x] + offset[0], u[x] + offset[1], v[x] + offset[2], mtx, |
| &ccsp[0], &ccsp[1], &ccsp[2]); |
| dst0[x] = (int16_t)WP2::RightShiftRound(ccsp[0], kMtxShift); |
| dst1[x] = (int16_t)WP2::RightShiftRound(ccsp[1], kMtxShift); |
| dst2[x] = (int16_t)WP2::RightShiftRound(ccsp[2], kMtxShift); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| |
| // Converts the 0-centered 'y,u,v,a' values into the alpha-premultiplied 'argb' |
| // channels using the matrix 'im' and the offset 'avg'. |
| // 'kNumRgbBits' can be either 8 or 10. Alpha is always 8-bit. |
| template <int kNumRgbBits, typename TypeOut> |
| void AyuvToArgb(int16_t y, int16_t u, int16_t v, int32_t a, |
| const int16_t avg[3], const int16_t im[9], |
| TypeOut* const argb) { |
| static_assert(sizeof(TypeOut) * 8 >= kNumRgbBits, "TypeOut is too narrow"); |
| a = WP2::Clamp<int32_t>(a, 0, WP2::kAlphaMax); |
| argb[0] = a; |
| if (a == 0) { |
| argb[1] = argb[2] = argb[3] = 0; // Alpha-premultiplied so 0 everywhere. |
| } else { |
| // For 10b, the maximum value is 1023 for RGB but 255 for A. Shifting alpha |
| // by 2 would limit to 1020, hence the dithering to keep the full range. |
| if (kNumRgbBits == 10) a = (a << 2) | (a >> 6); |
| int32_t rgb[3]; |
| WP2::Multiply<int32_t>(y, u, v, im, &rgb[0], &rgb[1], &rgb[2]); |
| for (uint32_t i : {0, 1, 2}) { |
| rgb[i] = WP2::RightShiftRound(rgb[i], kMtxShift - (kNumRgbBits - 8)); |
| argb[1 + i] = WP2::Clamp<int32_t>(rgb[i] + avg[i], 0, a); |
| } |
| } |
| } |
| |
| // {Y, U, V, alpha} planes to {alpha, premultiplied R,G,B} channels |
| void AyuvToArgb32_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| AyuvToArgb</*kNumRgbBits=*/8>(y[x], u[x], v[x], a[x], avg, mtx, |
| argb32 + 4 * x); |
| } |
| } |
| void AyuvToArgb38_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint16_t* const argb38 = (uint16_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| AyuvToArgb</*kNumRgbBits=*/10>(y[x], u[x], v[x], a[x], avg, mtx, |
| argb38 + 4 * x); |
| } |
| } |
| |
| // {Y, U, V, alpha} planes to {alpha, unmultiplied R,G,B} channels |
| void AyuvToARGB32_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| // Convert. |
| AyuvToArgb</*kNumRgbBits=*/8>(y[x], u[x], v[x], a[x], avg, mtx, |
| argb32 + 4 * x); |
| |
| // Now demultiply. |
| if (argb32[4 * x + 0] > 0 && argb32[4 * x + 0] < WP2::kAlphaMax) { |
| const uint32_t m = WP2::kAlphaDiv[argb32[4 * x + 0]]; |
| for (int i : {1, 2, 3}) { |
| argb32[4 * x + i] = WP2::DivByAlphaDiv(argb32[4 * x + i], m); |
| } |
| } |
| } |
| } |
| |
| // {Y, U, V, alpha} planes to {alpha set to 255, unmultiplied R,G,B} channels |
| void AyuvToXRGB32_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| AyuvToARGB32_C(y, u, v, a, avg, mtx, argb32, width); |
| for (uint32_t x = 0; x < width; ++x) { |
| argb32[4 * x] = WP2::kAlphaMax; |
| } |
| } |
| |
| // {Y, U, V} planes to {alpha set to 255, R,G,B} channels |
| void YuvToArgb32_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t avg[3], const int16_t mtx[9], void* argb, |
| uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| AyuvToArgb</*kNumRgbBits=*/8>(y[x], u[x], v[x], WP2::kAlphaMax, avg, mtx, |
| argb32 + 4 * x); |
| } |
| } |
| void YuvToArgb38_C(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t avg[3], const int16_t mtx[9], void* argb, |
| uint32_t width) { |
| uint16_t* const argb38 = (uint16_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| AyuvToArgb</*kNumRgbBits=*/10>(y[x], u[x], v[x], WP2::kAlphaMax, avg, mtx, |
| argb38 + 4 * x); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| // SSE4.1 implementation |
| |
| #if defined(WP2_USE_SSE) |
| |
| //------------------------------------------------------------------------------ |
| // some useful macros used extensively below |
| |
| // load the matrix elements mtx[] into variables mtx00, ... mtx21 |
| #define MAKE32b(A, B) ((uint32_t)((A) & 0xffff) | ((uint32_t)(B) << 16)) |
| |
| #define LOAD_MTX_ELEMENTS(MTX) \ |
| constexpr uint32_t kRnd = 1u << kMtxShift >> 1; \ |
| const __m128i mtx00 = _mm_set1_epi32(MAKE32b(MTX[0], MTX[1])); \ |
| const __m128i mtx01 = _mm_set1_epi32(MAKE32b(MTX[2], kRnd)); \ |
| const __m128i mtx10 = _mm_set1_epi32(MAKE32b(MTX[3], MTX[4])); \ |
| const __m128i mtx11 = _mm_set1_epi32(MAKE32b(MTX[5], kRnd)); \ |
| const __m128i mtx20 = _mm_set1_epi32(MAKE32b(MTX[6], MTX[7])); \ |
| const __m128i mtx21 = _mm_set1_epi32(MAKE32b(MTX[8], kRnd)); \ |
| const __m128i kCst = _mm_set1_epi16(1) |
| |
| // load the y[x],u[x],v[x] samples in variables Y0, U0, V0 |
| #define LOAD_YUV(Y0, U0, V0) \ |
| const __m128i Y0 = _mm_loadu_si128((const __m128i*)(y + x)); \ |
| const __m128i U0 = _mm_loadu_si128((const __m128i*)(u + x)); \ |
| const __m128i V0 = _mm_loadu_si128((const __m128i*)(v + x)) |
| |
| // unpack Y/U/V to variables 16b-pairs Y|U and V|K |
| #define UNPACK_YUV(Y, U, V) \ |
| const __m128i YU_lo = _mm_unpacklo_epi16(Y, U); \ |
| const __m128i YU_hi = _mm_unpackhi_epi16(Y, U); \ |
| const __m128i VK_lo = _mm_unpacklo_epi16(V, kCst); \ |
| const __m128i VK_hi = _mm_unpackhi_epi16(V, kCst) |
| |
| #define MULT_ROW(MTX0, MTX1, OUT) \ |
| __m128i OUT; \ |
| do { \ |
| const __m128i a0 = _mm_madd_epi16(YU_lo, MTX0); \ |
| const __m128i a1 = _mm_madd_epi16(YU_hi, MTX0); \ |
| const __m128i b0 = _mm_madd_epi16(VK_lo, MTX1); \ |
| const __m128i b1 = _mm_madd_epi16(VK_hi, MTX1); \ |
| const __m128i c0 = _mm_add_epi32(a0, b0); \ |
| const __m128i c1 = _mm_add_epi32(a1, b1); \ |
| const __m128i d0 = _mm_srai_epi32(c0, kMtxShift); \ |
| const __m128i d1 = _mm_srai_epi32(c1, kMtxShift); \ |
| OUT = _mm_packs_epi32(d0, d1); \ |
| } while (false) |
| |
| #define PACK_RGB(A, R, G, B, DST) \ |
| do { \ |
| const __m128i out_rb = _mm_packus_epi16(R, B); \ |
| const __m128i out_ag = _mm_packus_epi16(A, G); \ |
| const __m128i out_ar = _mm_unpacklo_epi8(out_ag, out_rb); \ |
| const __m128i out_gb = _mm_unpackhi_epi8(out_ag, out_rb); \ |
| const __m128i argb0 = _mm_unpacklo_epi16(out_ar, out_gb); \ |
| const __m128i argb1 = _mm_unpackhi_epi16(out_ar, out_gb); \ |
| _mm_storeu_si128((__m128i*)(DST + 0), argb0); \ |
| _mm_storeu_si128((__m128i*)(DST + 16), argb1); \ |
| } while (false) |
| |
| //------------------------------------------------------------------------------ |
| |
| void YuvToCustom_SSE(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t offset[3], const int16_t mtx[9], |
| int16_t* dst0, int16_t* dst1, int16_t* dst2, |
| uint32_t width) { |
| const __m128i Off0 = _mm_set1_epi16(offset[0]); |
| const __m128i Off1 = _mm_set1_epi16(offset[1]); |
| const __m128i Off2 = _mm_set1_epi16(offset[2]); |
| LOAD_MTX_ELEMENTS(mtx); |
| |
| uint32_t x = 0; |
| for (; x + 8 <= width; x += 8) { |
| // load source |
| LOAD_YUV(Y0, U0, V0); |
| // add offset |
| const __m128i Y1 = _mm_adds_epi16(Y0, Off0); |
| const __m128i U1 = _mm_adds_epi16(U0, Off1); |
| const __m128i V1 = _mm_adds_epi16(V0, Off2); |
| // unpack to YU + VK |
| UNPACK_YUV(Y1, U1, V1); |
| // multiply with mtx |
| MULT_ROW(mtx00, mtx01, out0); |
| MULT_ROW(mtx10, mtx11, out1); |
| MULT_ROW(mtx20, mtx21, out2); |
| // and store |
| _mm_storeu_si128((__m128i*)(dst0 + x), out0); |
| _mm_storeu_si128((__m128i*)(dst1 + x), out1); |
| _mm_storeu_si128((__m128i*)(dst2 + x), out2); |
| } |
| |
| if (x < width) { // left-over |
| YuvToCustom_C(y + x, u + x, v + x, offset, mtx, dst0 + x, dst1 + x, |
| dst2 + x, width - x); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| |
| // {YUV, alpha} to {alpha, premultiplied RGB} |
| void AyuvToArgb32_SSE(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| const __m128i Avg0 = _mm_set1_epi16(avg[0]); |
| const __m128i Avg1 = _mm_set1_epi16(avg[1]); |
| const __m128i Avg2 = _mm_set1_epi16(avg[2]); |
| LOAD_MTX_ELEMENTS(mtx); |
| uint32_t x = 0; |
| for (; x + 8 <= width; x += 8) { |
| // load source |
| LOAD_YUV(Y0, U0, V0); |
| UNPACK_YUV(Y0, U0, V0); |
| |
| // multiply with mtx and add average |
| MULT_ROW(mtx00, mtx01, R0); |
| MULT_ROW(mtx10, mtx11, G0); |
| MULT_ROW(mtx20, mtx21, B0); |
| |
| const __m128i A1 = _mm_loadu_si128((const __m128i*)(a + x)); |
| const __m128i R1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg0, R0)); |
| const __m128i G1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg1, G0)); |
| const __m128i B1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg2, B0)); |
| |
| // repack to 8b, with alpha |
| PACK_RGB(A1, R1, G1, B1, argb32 + 4 * x); |
| } |
| |
| if (x < width) { // left-over |
| AyuvToArgb32_C(y + x, u + x, v + x, a + x, avg, mtx, argb32 + 4 * x, |
| width - x); |
| } |
| } |
| |
| // {YUV, alpha} to {alpha, unmultiplied RGB} |
| void AyuvToARGB32_SSE(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| const __m128i Avg0 = _mm_set1_epi16(avg[0]); |
| const __m128i Avg1 = _mm_set1_epi16(avg[1]); |
| const __m128i Avg2 = _mm_set1_epi16(avg[2]); |
| LOAD_MTX_ELEMENTS(mtx); |
| uint32_t x = 0; |
| for (; x + 8 <= width; x += 8) { |
| LOAD_YUV(Y0, U0, V0); |
| UNPACK_YUV(Y0, U0, V0); |
| MULT_ROW(mtx00, mtx01, R0); |
| MULT_ROW(mtx10, mtx11, G0); |
| MULT_ROW(mtx20, mtx21, B0); |
| const __m128i A1 = _mm_loadu_si128((const __m128i*)(a + x)); |
| const __m128i R1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg0, R0)); |
| const __m128i G1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg1, G0)); |
| const __m128i B1 = _mm_min_epi16(A1, _mm_adds_epi16(Avg2, B0)); |
| uint8_t tmp[4 * 8]; |
| PACK_RGB(A1, R1, G1, B1, tmp); |
| for (uint32_t i = 0; i < 8; ++i) { |
| const uint8_t A = tmp[4 * i + 0]; |
| if (A == 255) { |
| argb32[4 * (x + i) + 0] = tmp[4 * i + 0]; |
| argb32[4 * (x + i) + 1] = tmp[4 * i + 1]; |
| argb32[4 * (x + i) + 2] = tmp[4 * i + 2]; |
| argb32[4 * (x + i) + 3] = tmp[4 * i + 3]; |
| } else { |
| const uint32_t M = WP2::kAlphaDiv[A]; |
| argb32[4 * (x + i) + 0] = A; |
| argb32[4 * (x + i) + 1] = WP2::DivByAlphaDiv(tmp[4 * i + 1], M); |
| argb32[4 * (x + i) + 2] = WP2::DivByAlphaDiv(tmp[4 * i + 2], M); |
| argb32[4 * (x + i) + 3] = WP2::DivByAlphaDiv(tmp[4 * i + 3], M); |
| } |
| } |
| } |
| |
| if (x < width) { // left-over |
| AyuvToARGB32_C(y + x, u + x, v + x, a + x, avg, mtx, argb32 + 4 * x, |
| width - x); |
| } |
| } |
| |
| // {YUV, alpha} to {alpha set to 255, unmultiplied RGB} |
| void AyuvToXRGB32_SSE(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t* a, const int16_t avg[3], |
| const int16_t mtx[9], void* argb, uint32_t width) { |
| AyuvToARGB32_SSE(y, u, v, a, avg, mtx, argb, width); |
| uint8_t* const argb32 = (uint8_t*)argb; |
| for (uint32_t x = 0; x < width; ++x) { |
| argb32[4 * x] = WP2::kAlphaMax; |
| } |
| } |
| |
| // {YUV} to {alpha set to 255, RGB} |
| void YuvToArgb32_SSE(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t avg[3], const int16_t mtx[9], void* argb, |
| uint32_t width) { |
| uint8_t* const argb32 = (uint8_t*)argb; |
| const __m128i Avg0 = _mm_set1_epi16(avg[0]); |
| const __m128i Avg1 = _mm_set1_epi16(avg[1]); |
| const __m128i Avg2 = _mm_set1_epi16(avg[2]); |
| LOAD_MTX_ELEMENTS(mtx); |
| const __m128i alpha = _mm_set1_epi16(255u); |
| uint32_t x = 0; |
| for (; x + 8 <= width; x += 8) { |
| // load source |
| LOAD_YUV(Y0, U0, V0); |
| UNPACK_YUV(Y0, U0, V0); |
| |
| // multiply with mtx and add average |
| MULT_ROW(mtx00, mtx01, R0); |
| MULT_ROW(mtx10, mtx11, G0); |
| MULT_ROW(mtx20, mtx21, B0); |
| |
| const __m128i R1 = _mm_adds_epi16(Avg0, R0); |
| const __m128i G1 = _mm_adds_epi16(Avg1, G0); |
| const __m128i B1 = _mm_adds_epi16(Avg2, B0); |
| |
| // repack to 8b, with alpha |
| PACK_RGB(alpha, R1, G1, B1, argb32 + 4 * x); |
| } |
| |
| if (x < width) { // left-over |
| YuvToArgb32_C(y + x, u + x, v + x, avg, mtx, argb32 + 4 * x, width - x); |
| } |
| } |
| |
| //------------------------------------------------------------------------------ |
| |
| #undef LOAD_MTX_ELEMENTS |
| #undef LOAD_YUV |
| #undef UNPACK_YUV |
| #undef MULT_ROW |
| #undef PACK_RGB |
| |
| WP2_TSAN_IGNORE_FUNCTION void WP2CSPConverterInitSSE() { |
| WP2YuvToCustom = YuvToCustom_SSE; |
| WP2AyuvToArgb32 = AyuvToArgb32_SSE; |
| WP2AyuvToARGB32 = AyuvToARGB32_SSE; |
| WP2AyuvToXRGB32 = AyuvToXRGB32_SSE; |
| WP2YuvToArgb32 = YuvToArgb32_SSE; |
| } |
| |
| #endif // WP2_USE_SSE |
| |
| //------------------------------------------------------------------------------ |
| // NEON implementation (TODO) |
| |
| //------------------------------------------------------------------------------ |
| |
| } // namespace |
| |
| void (*WP2YuvToCustom)(const int16_t* y, const int16_t* u, const int16_t* v, |
| const int16_t offset[3], const int16_t mtx[9], |
| int16_t* dst0, int16_t* dst1, int16_t* dst2, |
| uint32_t width) = nullptr; |
| |
| WP2AyuvToArgbFunc WP2AyuvToArgb32; |
| WP2AyuvToArgbFunc WP2AyuvToARGB32; |
| WP2AyuvToArgbFunc WP2AyuvToXRGB32; |
| WP2YuvToArgbFunc WP2YuvToArgb32; |
| WP2AyuvToArgbFunc WP2AyuvToArgb38; |
| WP2YuvToArgbFunc WP2YuvToArgb38; |
| |
| static volatile WP2CPUInfo csp_converter_last_cpuinfo_used = |
| (WP2CPUInfo)&csp_converter_last_cpuinfo_used; |
| |
| WP2_TSAN_IGNORE_FUNCTION void WP2CSPConverterInit() { |
| if (csp_converter_last_cpuinfo_used == WP2GetCPUInfo) return; |
| |
| WP2YuvToCustom = YuvToCustom_C; |
| WP2AyuvToArgb32 = AyuvToArgb32_C; |
| WP2AyuvToARGB32 = AyuvToARGB32_C; |
| WP2AyuvToXRGB32 = AyuvToXRGB32_C; |
| WP2YuvToArgb32 = YuvToArgb32_C; |
| WP2AyuvToArgb38 = AyuvToArgb38_C; |
| WP2YuvToArgb38 = YuvToArgb38_C; |
| |
| if (WP2GetCPUInfo != nullptr) { |
| #if defined(WP2_USE_SSE) |
| if (WP2GetCPUInfo(kSSE)) WP2CSPConverterInitSSE(); |
| #endif |
| } |
| csp_converter_last_cpuinfo_used = WP2GetCPUInfo; |
| } |
| |
| //------------------------------------------------------------------------------ |
