| /* |
| * Copyright 2016 Google Inc. |
| * |
| * Use of this source code is governed by a BSD-style license that can be |
| * found in the LICENSE file. |
| */ |
| |
| #include "skia/ext/convolver_neon.h" |
| |
| #include <arm_neon.h> |
| |
| namespace skia { |
| |
| static SK_ALWAYS_INLINE int32x4_t |
| AccumRemainder(const unsigned char* pixels_left, |
| const ConvolutionFilter1D::Fixed* filter_values, |
| int r) { |
| int remainder[4] = {0, 0, 0, 0}; |
| for (int i = 0; i < r; i++) { |
| ConvolutionFilter1D::Fixed coeff = filter_values[i]; |
| remainder[0] += coeff * pixels_left[i * 4 + 0]; |
| remainder[1] += coeff * pixels_left[i * 4 + 1]; |
| remainder[2] += coeff * pixels_left[i * 4 + 2]; |
| remainder[3] += coeff * pixels_left[i * 4 + 3]; |
| } |
| return vld1q_s32(remainder); |
| } |
| |
| // Convolves horizontally along a single row. The row data is given in |
| // |src_data| and continues for the num_values() of the filter. |
| void ConvolveHorizontally_Neon(const unsigned char* src_data, |
| const ConvolutionFilter1D& filter, |
| unsigned char* out_row, |
| bool /*has_alpha*/) { |
| // Loop over each pixel on this row in the output image. |
| int num_values = filter.num_values(); |
| for (int out_x = 0; out_x < num_values; out_x++) { |
| // Get the filter that determines the current output pixel. |
| int filter_offset, filter_length; |
| const ConvolutionFilter1D::Fixed* filter_values = |
| filter.FilterForValue(out_x, &filter_offset, &filter_length); |
| |
| // Compute the first pixel in this row that the filter affects. It will |
| // touch |filter_length| pixels (4 bytes each) after this. |
| const unsigned char* row_to_filter = &src_data[filter_offset * 4]; |
| |
| // Apply the filter to the row to get the destination pixel in |accum|. |
| int32x4_t accum = vdupq_n_s32(0); |
| for (int filter_x = 0; filter_x < (filter_length / 4); filter_x++) { |
| // Load 4 coefficients. |
| int16x4_t coeffs = vld1_s16(filter_values); |
| // Load 4 pixels into a q-register. |
| uint8x16_t pixels = vld1q_u8(row_to_filter); |
| |
| // Expand to 16-bit channels split across two q-registers. |
| int16x8_t p01_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pixels))); |
| int16x8_t p23_16 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pixels))); |
| |
| // Scale each pixel (each d-register) by its filter coefficients, |
| // accumulating into 32-bit. |
| accum = vmlal_lane_s16(accum, vget_low_s16(p01_16), coeffs, 0); |
| accum = vmlal_lane_s16(accum, vget_high_s16(p01_16), coeffs, 1); |
| accum = vmlal_lane_s16(accum, vget_low_s16(p23_16), coeffs, 2); |
| accum = vmlal_lane_s16(accum, vget_high_s16(p23_16), coeffs, 3); |
| |
| // Advance to next elements. |
| row_to_filter += 16; |
| filter_values += 4; |
| } |
| |
| int remainder = filter_length & 3; |
| if (remainder) { |
| int remainder_offset = (filter_offset + filter_length - remainder) * 4; |
| accum += |
| AccumRemainder(src_data + remainder_offset, filter_values, remainder); |
| } |
| |
| // Bring this value back in range. All of the filter scaling factors |
| // are in fixed point with kShiftBits bits of fractional part. |
| int16x4_t accum16 = vqshrn_n_s32(accum, ConvolutionFilter1D::kShiftBits); |
| |
| // Pack and store the new pixel. |
| uint8x8_t accum8 = vqmovun_s16(vcombine_s16(accum16, accum16)); |
| vst1_lane_u32(reinterpret_cast<uint32_t*>(out_row), |
| vreinterpret_u32_u8(accum8), 0); |
| out_row += 4; |
| } |
| } |
| |
| // Convolves horizontally along four rows. The row data is given in |
| // |src_data| and continues for the num_values() of the filter. |
| // The algorithm is almost same as |convolve_horizontally|. Please |
| // refer to that function for detailed comments. |
| void Convolve4RowsHorizontally_Neon(const unsigned char* src_data[4], |
| const ConvolutionFilter1D& filter, |
| unsigned char* out_row[4]) { |
| // Output one pixel each iteration, calculating all channels (RGBA) together. |
| int num_values = filter.num_values(); |
| for (int out_x = 0; out_x < num_values; out_x++) { |
| int filter_offset, filter_length; |
| const ConvolutionFilter1D::Fixed* filter_values = |
| filter.FilterForValue(out_x, &filter_offset, &filter_length); |
| |
| // Four pixels in a column per iteration. |
| int32x4_t accum0 = vdupq_n_s32(0); |
| int32x4_t accum1 = vdupq_n_s32(0); |
| int32x4_t accum2 = vdupq_n_s32(0); |
| int32x4_t accum3 = vdupq_n_s32(0); |
| |
| int start = filter_offset * 4; |
| |
| // Load and accumulate with four coefficients per iteration. |
| for (int filter_x = 0; filter_x < (filter_length / 4); filter_x++) { |
| // Load 4 coefficients. |
| int16x4_t coeffs = vld1_s16(filter_values); |
| |
| auto iteration = [=](const uint8_t* src) { |
| // c.f. ConvolveHorizontally_Neon() above. |
| uint8x16_t pixels = vld1q_u8(src); |
| int16x8_t p01_16 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(pixels))); |
| int16x8_t p23_16 = |
| vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(pixels))); |
| int32x4_t accum = vdupq_n_s32(0); |
| accum = vmlal_lane_s16(accum, vget_low_s16(p01_16), coeffs, 0); |
| accum = vmlal_lane_s16(accum, vget_high_s16(p01_16), coeffs, 1); |
| accum = vmlal_lane_s16(accum, vget_low_s16(p23_16), coeffs, 2); |
| accum = vmlal_lane_s16(accum, vget_high_s16(p23_16), coeffs, 3); |
| return accum; |
| }; |
| |
| accum0 += iteration(src_data[0] + start); |
| accum1 += iteration(src_data[1] + start); |
| accum2 += iteration(src_data[2] + start); |
| accum3 += iteration(src_data[3] + start); |
| |
| start += 16; |
| filter_values += 4; |
| } |
| |
| int remainder = filter_length & 3; |
| if (remainder) { |
| int remainder_offset = (filter_offset + filter_length - remainder) * 4; |
| accum0 += AccumRemainder(src_data[0] + remainder_offset, filter_values, |
| remainder); |
| accum1 += AccumRemainder(src_data[1] + remainder_offset, filter_values, |
| remainder); |
| accum2 += AccumRemainder(src_data[2] + remainder_offset, filter_values, |
| remainder); |
| accum3 += AccumRemainder(src_data[3] + remainder_offset, filter_values, |
| remainder); |
| } |
| |
| auto pack_result = [](int32x4_t accum) { |
| int16x4_t accum16 = vqshrn_n_s32(accum, ConvolutionFilter1D::kShiftBits); |
| return vqmovun_s16(vcombine_s16(accum16, accum16)); |
| }; |
| |
| uint8x8_t res0 = pack_result(accum0); |
| uint8x8_t res1 = pack_result(accum1); |
| uint8x8_t res2 = pack_result(accum2); |
| uint8x8_t res3 = pack_result(accum3); |
| |
| vst1_lane_u32(reinterpret_cast<uint32_t*>(out_row[0]), |
| vreinterpret_u32_u8(res0), 0); |
| vst1_lane_u32(reinterpret_cast<uint32_t*>(out_row[1]), |
| vreinterpret_u32_u8(res1), 0); |
| vst1_lane_u32(reinterpret_cast<uint32_t*>(out_row[2]), |
| vreinterpret_u32_u8(res2), 0); |
| vst1_lane_u32(reinterpret_cast<uint32_t*>(out_row[3]), |
| vreinterpret_u32_u8(res3), 0); |
| out_row[0] += 4; |
| out_row[1] += 4; |
| out_row[2] += 4; |
| out_row[3] += 4; |
| } |
| } |
| |
| // Does vertical convolution to produce one output row. The filter values and |
| // length are given in the first two parameters. These are applied to each |
| // of the rows pointed to in the |source_data_rows| array, with each row |
| // being |pixel_width| wide. |
| // |
| // The output must have room for |pixel_width * 4| bytes. |
| void ConvolveVertically_Neon(const ConvolutionFilter1D::Fixed* filter_values, |
| int filter_length, |
| unsigned char* const* source_data_rows, |
| int pixel_width, |
| unsigned char* out_row, |
| bool has_alpha) { |
| int width = pixel_width & ~3; |
| |
| // Output four pixels per iteration (16 bytes). |
| for (int out_x = 0; out_x < width; out_x += 4) { |
| // Accumulated result for each pixel. 32 bits per RGBA channel. |
| int32x4_t accum0 = vdupq_n_s32(0); |
| int32x4_t accum1 = vdupq_n_s32(0); |
| int32x4_t accum2 = vdupq_n_s32(0); |
| int32x4_t accum3 = vdupq_n_s32(0); |
| |
| // Convolve with one filter coefficient per iteration. |
| for (int filter_y = 0; filter_y < filter_length; filter_y++) { |
| // Load four pixels (16 bytes) together. |
| // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
| uint8x16_t src8 = vld1q_u8(&source_data_rows[filter_y][out_x << 2]); |
| |
| int16x8_t src16_01 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src8))); |
| int16x8_t src16_23 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src8))); |
| |
| accum0 = |
| vmlal_n_s16(accum0, vget_low_s16(src16_01), filter_values[filter_y]); |
| accum1 = |
| vmlal_n_s16(accum1, vget_high_s16(src16_01), filter_values[filter_y]); |
| accum2 = |
| vmlal_n_s16(accum2, vget_low_s16(src16_23), filter_values[filter_y]); |
| accum3 = |
| vmlal_n_s16(accum3, vget_high_s16(src16_23), filter_values[filter_y]); |
| } |
| |
| // Shift right for fixed point implementation. |
| // Packing 32 bits |accum| to 16 bits per channel (unsigned saturation). |
| int16x4_t accum16_0 = vqshrn_n_s32(accum0, ConvolutionFilter1D::kShiftBits); |
| int16x4_t accum16_1 = vqshrn_n_s32(accum1, ConvolutionFilter1D::kShiftBits); |
| int16x4_t accum16_2 = vqshrn_n_s32(accum2, ConvolutionFilter1D::kShiftBits); |
| int16x4_t accum16_3 = vqshrn_n_s32(accum3, ConvolutionFilter1D::kShiftBits); |
| |
| // [16] a1 b1 g1 r1 a0 b0 g0 r0 |
| int16x8_t accum16_low = vcombine_s16(accum16_0, accum16_1); |
| // [16] a3 b3 g3 r3 a2 b2 g2 r2 |
| int16x8_t accum16_high = vcombine_s16(accum16_2, accum16_3); |
| |
| // Packing 16 bits |accum| to 8 bits per channel (unsigned saturation). |
| // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
| uint8x16_t accum8 = |
| vcombine_u8(vqmovun_s16(accum16_low), vqmovun_s16(accum16_high)); |
| |
| if (has_alpha) { |
| // Compute the max(ri, gi, bi) for each pixel. |
| // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0 |
| uint8x16_t a = |
| vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 8)); |
| // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
| uint8x16_t b = vmaxq_u8(a, accum8); // Max of r and g |
| // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0 |
| a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 16)); |
| // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
| b = vmaxq_u8(a, b); // Max of r and g and b. |
| // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00 |
| b = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(b), 24)); |
| |
| // Make sure the value of alpha channel is always larger than maximum |
| // value of color channels. |
| accum8 = vmaxq_u8(b, accum8); |
| } else { |
| // Set value of alpha channels to 0xFF. |
| accum8 = vreinterpretq_u8_u32(vreinterpretq_u32_u8(accum8) | |
| vdupq_n_u32(0xFF000000)); |
| } |
| |
| // Store the convolution result (16 bytes) and advance the pixel pointers. |
| vst1q_u8(out_row, accum8); |
| out_row += 16; |
| } |
| |
| // Process the leftovers when the width of the output is not divisible |
| // by 4, that is at most 3 pixels. |
| int remainder = pixel_width & 3; |
| if (remainder) { |
| int32x4_t accum0 = vdupq_n_s32(0); |
| int32x4_t accum1 = vdupq_n_s32(0); |
| int32x4_t accum2 = vdupq_n_s32(0); |
| |
| for (int filter_y = 0; filter_y < filter_length; ++filter_y) { |
| // [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0 |
| uint8x16_t src8 = vld1q_u8(&source_data_rows[filter_y][width * 4]); |
| |
| int16x8_t src16_01 = vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(src8))); |
| int16x8_t src16_23 = vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(src8))); |
| |
| accum0 = |
| vmlal_n_s16(accum0, vget_low_s16(src16_01), filter_values[filter_y]); |
| accum1 = |
| vmlal_n_s16(accum1, vget_high_s16(src16_01), filter_values[filter_y]); |
| accum2 = |
| vmlal_n_s16(accum2, vget_low_s16(src16_23), filter_values[filter_y]); |
| } |
| |
| int16x4_t accum16_0 = vqshrn_n_s32(accum0, ConvolutionFilter1D::kShiftBits); |
| int16x4_t accum16_1 = vqshrn_n_s32(accum1, ConvolutionFilter1D::kShiftBits); |
| int16x4_t accum16_2 = vqshrn_n_s32(accum2, ConvolutionFilter1D::kShiftBits); |
| |
| int16x8_t accum16_low = vcombine_s16(accum16_0, accum16_1); |
| int16x8_t accum16_high = vcombine_s16(accum16_2, accum16_2); |
| |
| uint8x16_t accum8 = |
| vcombine_u8(vqmovun_s16(accum16_low), vqmovun_s16(accum16_high)); |
| |
| if (has_alpha) { |
| // Compute the max(ri, gi, bi) for each pixel. |
| // [8] xx a3 b3 g3 xx a2 b2 g2 xx a1 b1 g1 xx a0 b0 g0 |
| uint8x16_t a = |
| vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 8)); |
| // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
| uint8x16_t b = vmaxq_u8(a, accum8); // Max of r and g |
| // [8] xx xx a3 b3 xx xx a2 b2 xx xx a1 b1 xx xx a0 b0 |
| a = vreinterpretq_u8_u32(vshrq_n_u32(vreinterpretq_u32_u8(accum8), 16)); |
| // [8] xx xx xx max3 xx xx xx max2 xx xx xx max1 xx xx xx max0 |
| b = vmaxq_u8(a, b); // Max of r and g and b. |
| // [8] max3 00 00 00 max2 00 00 00 max1 00 00 00 max0 00 00 00 |
| b = vreinterpretq_u8_u32(vshlq_n_u32(vreinterpretq_u32_u8(b), 24)); |
| |
| // Make sure the value of alpha channel is always larger than maximum |
| // value of color channels. |
| accum8 = vmaxq_u8(b, accum8); |
| } else { |
| // Set value of alpha channels to 0xFF. |
| accum8 = vreinterpretq_u8_u32(vreinterpretq_u32_u8(accum8) | |
| vdupq_n_u32(0xFF000000)); |
| } |
| |
| switch (remainder) { |
| case 1: |
| vst1q_lane_u32(reinterpret_cast<uint32_t*>(out_row), |
| vreinterpretq_u32_u8(accum8), 0); |
| break; |
| case 2: |
| vst1_u32(reinterpret_cast<uint32_t*>(out_row), |
| vreinterpret_u32_u8(vget_low_u8(accum8))); |
| break; |
| case 3: |
| vst1_u32(reinterpret_cast<uint32_t*>(out_row), |
| vreinterpret_u32_u8(vget_low_u8(accum8))); |
| vst1q_lane_u32(reinterpret_cast<uint32_t*>(out_row + 8), |
| vreinterpretq_u32_u8(accum8), 2); |
| break; |
| } |
| } |
| } |
| |
| } // namespace skia |
