image2yuvconfig() should calculate uv_crop_width image2yuvconfig() should calculate uv_crop_width and uv_crop_height from y_crop_width and y_crop_height. Use the same formula to calculate uv_width and uv_height. The VP8 version of image2yuvconfig() also calculated y_width, y_height, uv_width, uv_height incorrectly based on img->d_w and img->d_h. This is a backport from libaom: 0607f6bef8 image2yuvconfig() should calculate uv_crop_width Bug: 517413744 Change-Id: I8c099dbd36fe3c2cf9696868da8331f83c19828b
diff --git a/vp8/vp8_cx_iface.c b/vp8/vp8_cx_iface.c index da060e8..b1ec0d1 100644 --- a/vp8/vp8_cx_iface.c +++ b/vp8/vp8_cx_iface.c
@@ -784,23 +784,19 @@ static vpx_codec_err_t image2yuvconfig(const vpx_image_t *img, YV12_BUFFER_CONFIG *yv12) { - const int y_w = img->d_w; - const int y_h = img->d_h; - const int uv_w = (img->d_w + 1) / 2; - const int uv_h = (img->d_h + 1) / 2; vpx_codec_err_t res = VPX_CODEC_OK; yv12->y_buffer = img->planes[VPX_PLANE_Y]; yv12->u_buffer = img->planes[VPX_PLANE_U]; yv12->v_buffer = img->planes[VPX_PLANE_V]; - yv12->y_crop_width = y_w; - yv12->y_crop_height = y_h; - yv12->y_width = y_w; - yv12->y_height = y_h; - yv12->uv_crop_width = uv_w; - yv12->uv_crop_height = uv_h; - yv12->uv_width = uv_w; - yv12->uv_height = uv_h; + yv12->y_crop_width = img->d_w; + yv12->y_crop_height = img->d_h; + yv12->y_width = img->w; + yv12->y_height = img->h; + yv12->uv_crop_width = (yv12->y_crop_width + 1) / 2; + yv12->uv_crop_height = (yv12->y_crop_height + 1) / 2; + yv12->uv_width = (yv12->y_width + 1) / 2; + yv12->uv_height = (yv12->y_height + 1) / 2; yv12->y_stride = img->stride[VPX_PLANE_Y]; assert(img->stride[VPX_PLANE_U] == img->stride[VPX_PLANE_V]);
diff --git a/vp9/vp9_iface_common.c b/vp9/vp9_iface_common.c index c0f8f09..b0bbd2f 100644 --- a/vp9/vp9_iface_common.c +++ b/vp9/vp9_iface_common.c
@@ -89,13 +89,17 @@ yv12->y_width = img->w; yv12->y_height = img->h; - yv12->uv_width = img->x_chroma_shift == 1 || img->fmt == VPX_IMG_FMT_NV12 - ? (1 + yv12->y_width) / 2 - : yv12->y_width; + // When reading the data, UV are in one plane for NV12 format, thus + // x_chroma_shift is 0. After converting, UV are in separate planes, and + // x_chroma_shift should be set to 1. + const unsigned int x_chroma_shift = + (img->fmt == VPX_IMG_FMT_NV12) ? 1 : img->x_chroma_shift; + yv12->uv_width = (yv12->y_width + x_chroma_shift) >> x_chroma_shift; yv12->uv_height = - img->y_chroma_shift == 1 ? (1 + yv12->y_height) / 2 : yv12->y_height; - yv12->uv_crop_width = yv12->uv_width; - yv12->uv_crop_height = yv12->uv_height; + (yv12->y_height + img->y_chroma_shift) >> img->y_chroma_shift; + yv12->uv_crop_width = (yv12->y_crop_width + x_chroma_shift) >> x_chroma_shift; + yv12->uv_crop_height = + (yv12->y_crop_height + img->y_chroma_shift) >> img->y_chroma_shift; yv12->y_stride = img->stride[VPX_PLANE_Y]; assert(img->stride[VPX_PLANE_U] == img->stride[VPX_PLANE_V]); @@ -128,11 +132,7 @@ #else yv12->border = (img->stride[VPX_PLANE_Y] - img->w) / 2; #endif // CONFIG_VP9_HIGHBITDEPTH - yv12->subsampling_x = img->x_chroma_shift; + yv12->subsampling_x = x_chroma_shift; yv12->subsampling_y = img->y_chroma_shift; - // When reading the data, UV are in one plane for NV12 format, thus - // x_chroma_shift is 0. After converting, UV are in separate planes, and - // subsampling_x should be set to 1. - if (img->fmt == VPX_IMG_FMT_NV12) yv12->subsampling_x = 1; return VPX_CODEC_OK; }