blob: 8fe97d7f59f72a0677c8460186ca65cc701036f0 [file] [log] [blame]
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef MEDIA_GPU_V4L2_V4L2_JPEG_ENCODE_ACCELERATOR_H_
#define MEDIA_GPU_V4L2_V4L2_JPEG_ENCODE_ACCELERATOR_H_
#include <stddef.h>
#include <stdint.h>
#include <memory>
#include <vector>
#include "base/containers/queue.h"
#include "base/macros.h"
#include "base/memory/ref_counted.h"
#include "base/memory/weak_ptr.h"
#include "base/single_thread_task_runner.h"
#include "base/threading/thread.h"
#include "components/chromeos_camera/jpeg_encode_accelerator.h"
#include "gpu/ipc/common/gpu_memory_buffer_support.h"
#include "media/base/bitstream_buffer.h"
#include "media/base/unaligned_shared_memory.h"
#include "media/base/video_frame.h"
#include "media/gpu/media_gpu_export.h"
#include "media/gpu/v4l2/v4l2_device.h"
#include "media/gpu/v4l2/v4l2_jpeg_encode_accelerator.h"
#include "media/parsers/jpeg_parser.h"
namespace {
// Input pixel format V4L2_PIX_FMT_YUV420M has 3 physical planes.
constexpr size_t kMaxI420Plane = 3;
constexpr size_t kMaxNV12Plane = 2;
// Output pixel format V4L2_PIX_FMT_JPEG(_RAW) has only one physical plane.
constexpr size_t kMaxJpegPlane = 1;
// This class can only handle V4L2_PIX_FMT_YUV420(M) as input, so
// kMaxI420Plane can only be 3.
static_assert(kMaxI420Plane == 3,
"kMaxI420Plane must be 3 as input may be V4L2_PIX_FMT_YUV420M");
// This class can only handle V4L2_PIX_FMT_JPEG(_RAW) as output, so
// kMaxJpegPlanes can only be 1.
static_assert(
kMaxJpegPlane == 1,
"kMaxJpegPlane must be 1 as output must be V4L2_PIX_FMT_JPEG(_RAW)");
} // namespace
namespace media {
class MEDIA_GPU_EXPORT V4L2JpegEncodeAccelerator
: public chromeos_camera::JpegEncodeAccelerator {
public:
V4L2JpegEncodeAccelerator(
const scoped_refptr<base::SingleThreadTaskRunner>& io_task_runner);
~V4L2JpegEncodeAccelerator() override;
// JpegEncodeAccelerator implementation.
chromeos_camera::JpegEncodeAccelerator::Status Initialize(
chromeos_camera::JpegEncodeAccelerator::Client* client) override;
size_t GetMaxCodedBufferSize(const gfx::Size& picture_size) override;
void Encode(scoped_refptr<media::VideoFrame> video_frame,
int quality,
BitstreamBuffer* exif_buffer,
BitstreamBuffer output_buffer) override;
void EncodeWithDmaBuf(scoped_refptr<VideoFrame> input_frame,
scoped_refptr<VideoFrame> output_frame,
int quality,
int32_t task_id,
BitstreamBuffer* exif_buffer) override;
private:
// Record for input buffers.
struct I420BufferRecord {
I420BufferRecord();
~I420BufferRecord();
void* address[kMaxI420Plane]; // mmap() address.
size_t length[kMaxI420Plane]; // mmap() length.
// Set true during QBUF and DQBUF. |address| will be accessed by hardware.
bool at_device;
};
// Record for output buffers.
struct JpegBufferRecord {
JpegBufferRecord();
~JpegBufferRecord();
void* address[kMaxJpegPlane]; // mmap() address.
size_t length[kMaxJpegPlane]; // mmap() length.
// Set true during QBUF and DQBUF. |address| will be accessed by hardware.
bool at_device;
};
// Job record. Jobs are processed in a FIFO order. This is separated from
// I420BufferRecord, because a I420BufferRecord of input may be returned
// before we dequeue the corresponding output buffer. It can't always be
// associated with a JpegBufferRecord of output immediately either, because at
// the time of submission we may not have one available (and don't need one
// to submit input to the device).
struct JobRecord {
JobRecord(scoped_refptr<VideoFrame> input_frame,
scoped_refptr<VideoFrame> output_frame,
int32_t task_id,
int quality,
BitstreamBuffer* exif_buffer);
JobRecord(scoped_refptr<VideoFrame> input_frame,
int quality,
BitstreamBuffer* exif_buffer,
BitstreamBuffer output_buffer);
~JobRecord();
// Input frame buffer.
scoped_refptr<VideoFrame> input_frame;
// Output frame buffer.
scoped_refptr<VideoFrame> output_frame;
// JPEG encode quality.
int quality;
// Encode task ID.
int32_t task_id;
// Memory mapped from |output_buffer|.
UnalignedSharedMemory output_shm;
// Offset used for |output_shm|.
off_t output_offset;
// Memory mapped from |exif_buffer|.
// It contains EXIF data to be inserted into JPEG image. If it's nullptr,
// the JFIF APP0 segment will be inserted.
std::unique_ptr<UnalignedSharedMemory> exif_shm;
// Offset used for |exif_shm|.
off_t exif_offset;
};
// TODO(wtlee): To be deprecated. (crbug.com/944705)
//
// Encode Instance. One EncodedInstance is used for a specific set of jpeg
// parameters. The stored parameters are jpeg quality and resolutions of input
// image.
// We execute all EncodedInstance methods on |encoder_task_runner_| except
// Initialize().
class EncodedInstance {
public:
EncodedInstance(V4L2JpegEncodeAccelerator* parent);
~EncodedInstance();
bool Initialize();
// Create V4L2 buffers for input and output.
bool CreateBuffers(gfx::Size input_coded_size, size_t output_buffer_size);
// Set up JPEG related parameters in V4L2 device.
bool SetUpJpegParameters(int quality, gfx::Size coded_size);
// Dequeue last frame and enqueue next frame.
void ServiceDevice();
// Destroy input and output buffers.
void DestroyTask();
base::queue<std::unique_ptr<JobRecord>> input_job_queue_;
base::queue<std::unique_ptr<JobRecord>> running_job_queue_;
private:
// Prepare full JPEG markers except SOI and EXIF/APP0 markers in
// |jpeg_markers_|.
void PrepareJpegMarkers(gfx::Size coded_size);
// Copy the encoded data from |output_buffer| to the |dst_ptr| provided by
// the client. Add JPEG Marks if needed. Add EXIF section by |exif_shm|.
size_t FinalizeJpegImage(uint8_t* dst_ptr,
const JpegBufferRecord& output_buffer,
size_t buffer_size,
std::unique_ptr<UnalignedSharedMemory> exif_shm);
bool SetInputBufferFormat(gfx::Size coded_size);
bool SetOutputBufferFormat(gfx::Size coded_size, size_t buffer_size);
bool RequestInputBuffers();
bool RequestOutputBuffers();
void EnqueueInput();
void EnqueueOutput();
void Dequeue();
bool EnqueueInputRecord();
bool EnqueueOutputRecord();
void DestroyInputBuffers();
void DestroyOutputBuffers();
// Return the number of input/output buffers enqueued to the device.
size_t InputBufferQueuedCount();
size_t OutputBufferQueuedCount();
void NotifyError(int32_t task_id, Status status);
// Fill the quantization table into |dst_table|. The value is scaled by
// the |quality| and |basic_table|.
// We use the the Independent JPEG Group's formula to scale scale table.
// http://www.ijg.org/
static void FillQuantizationTable(int quality,
const uint8_t* basic_table,
uint8_t* dst_table);
// The number of input buffers and output buffers.
const size_t kBufferCount = 2;
// Pointer back to the parent.
V4L2JpegEncodeAccelerator* parent_;
// The V4L2Device this class is operating upon.
scoped_refptr<V4L2Device> device_;
// Input queue state.
bool input_streamon_;
// Mapping of int index to an input buffer record.
std::vector<I420BufferRecord> input_buffer_map_;
// Indices of input buffers ready to use; LIFO since we don't care about
// ordering.
std::vector<int> free_input_buffers_;
// Output queue state.
bool output_streamon_;
// Mapping of int index to an output buffer record.
std::vector<JpegBufferRecord> output_buffer_map_;
// Indices of output buffers ready to use; LIFO since we don't care about
// ordering.
std::vector<int> free_output_buffers_;
// Pixel format of input buffer.
uint32_t input_buffer_pixelformat_;
// Number of physical planes the input buffers have.
size_t input_buffer_num_planes_;
// Pixel format of output buffer.
uint32_t output_buffer_pixelformat_;
// Height of input buffer returned by driver.
uint32_t input_buffer_height_;
// Bytes per line for each plane.
uint32_t bytes_per_line_[kMaxI420Plane];
// JPEG Quantization table for V4L2_PIX_FMT_JPEG_RAW.
JpegQuantizationTable quantization_table_[2];
// JPEG markers for V4L2_PIX_FMT_JPEG_RAW.
// We prepare markers in the EncodedInstance setup stage, and reuse it for
// every encoding.
std::vector<uint8_t> jpeg_markers_;
};
// Encode Instance. One EncodedInstance is used for a specific set of jpeg
// parameters. The stored parameters are jpeg quality and resolutions of input
// image.
// We execute all EncodedInstance methods on |encoder_task_runner_| except
// Initialize().
class EncodedInstanceDmaBuf {
public:
EncodedInstanceDmaBuf(V4L2JpegEncodeAccelerator* parent);
~EncodedInstanceDmaBuf();
bool Initialize();
// Create V4L2 buffers for input and output.
bool CreateBuffers(gfx::Size input_coded_size,
const VideoFrameLayout& input_layout,
size_t output_buffer_size);
// Set up JPEG related parameters in V4L2 device.
bool SetUpJpegParameters(int quality, gfx::Size coded_size);
// Dequeue last frame and enqueue next frame.
void ServiceDevice();
// Destroy input and output buffers.
void DestroyTask();
base::queue<std::unique_ptr<JobRecord>> input_job_queue_;
base::queue<std::unique_ptr<JobRecord>> running_job_queue_;
private:
// Prepare full JPEG markers except SOI and EXIF/APP0 markers in
// |jpeg_markers_|.
void PrepareJpegMarkers(gfx::Size coded_size);
// Combined the encoded data from |output_frame| with the JFIF/EXIF data.
// Add JPEG Marks if needed. Add EXIF section by |exif_shm|.
size_t FinalizeJpegImage(scoped_refptr<VideoFrame> output_frame,
size_t buffer_size,
std::unique_ptr<UnalignedSharedMemory> exif_shm);
bool SetInputBufferFormat(gfx::Size coded_size,
const VideoFrameLayout& input_layout);
bool SetOutputBufferFormat(gfx::Size coded_size, size_t buffer_size);
bool RequestInputBuffers();
bool RequestOutputBuffers();
void EnqueueInput();
void EnqueueOutput();
void Dequeue();
bool EnqueueInputRecord();
bool EnqueueOutputRecord();
void DestroyInputBuffers();
void DestroyOutputBuffers();
// Return the number of input/output buffers enqueued to the device.
size_t InputBufferQueuedCount();
size_t OutputBufferQueuedCount();
void NotifyError(int32_t task_id, Status status);
// Fill the quantization table into |dst_table|. The value is scaled by
// the |quality| and |basic_table|.
// We use the the Independent JPEG Group's formula to scale scale table.
// http://www.ijg.org/
static void FillQuantizationTable(int quality,
const uint8_t* basic_table,
uint8_t* dst_table);
// The number of input buffers and output buffers.
const size_t kBufferCount = 2;
// Pointer back to the parent.
V4L2JpegEncodeAccelerator* parent_;
// Layout that represents the input data.
base::Optional<VideoFrameLayout> device_input_layout_;
// The V4L2Device this class is operating upon.
scoped_refptr<V4L2Device> device_;
std::unique_ptr<gpu::GpuMemoryBufferSupport> gpu_memory_buffer_support_;
// Input queue state.
bool input_streamon_;
// Indices of input buffers ready to use; LIFO since we don't care about
// ordering.
std::vector<int> free_input_buffers_;
// Output queue state.
bool output_streamon_;
// Indices of output buffers ready to use; LIFO since we don't care about
// ordering.
std::vector<int> free_output_buffers_;
// Pixel format of input buffer.
uint32_t input_buffer_pixelformat_;
// Number of physical planes the input buffers have.
size_t input_buffer_num_planes_;
// Pixel format of output buffer.
uint32_t output_buffer_pixelformat_;
// Height of input buffer returned by driver.
uint32_t input_buffer_height_;
// JPEG Quantization table for V4L2_PIX_FMT_JPEG_RAW.
JpegQuantizationTable quantization_table_[2];
// JPEG markers for V4L2_PIX_FMT_JPEG_RAW.
// We prepare markers in the EncodedInstance setup stage, and reuse it for
// every encoding.
std::vector<uint8_t> jpeg_markers_;
};
void VideoFrameReady(int32_t task_id, size_t encoded_picture_size);
void NotifyError(int32_t task_id, Status status);
// Run on |encoder_thread_| to enqueue the incoming frame.
void EncodeTask(std::unique_ptr<JobRecord> job_record);
// TODO(wtlee): To be deprecated. (crbug.com/944705)
void EncodeTaskLegacy(std::unique_ptr<JobRecord> job_record);
// Run on |encoder_thread_| to trigger ServiceDevice of EncodedInstance class.
void ServiceDeviceTask();
// TODO(wtlee): To be deprecated. (crbug.com/944705)
void ServiceDeviceTaskLegacy();
// Run on |encoder_thread_| to destroy input and output buffers.
void DestroyTask();
// The |latest_input_buffer_coded_size_| and |latest_quality_| are used to
// check if we need to open new EncodedInstance.
// Latest coded size of input buffer.
gfx::Size latest_input_buffer_coded_size_;
// TODO(wtlee): To be deprecated. (crbug.com/944705)
gfx::Size latest_input_buffer_coded_size_legacy_;
// Latest encode quality.
int latest_quality_;
// TODO(wtlee): To be deprecated. (crbug.com/944705)
int latest_quality_legacy_;
// ChildThread's task runner.
scoped_refptr<base::SingleThreadTaskRunner> child_task_runner_;
// GPU IO task runner.
scoped_refptr<base::SingleThreadTaskRunner> io_task_runner_;
// The client of this class.
chromeos_camera::JpegEncodeAccelerator::Client* client_;
// Thread to communicate with the device.
base::Thread encoder_thread_;
// Encode task runner.
scoped_refptr<base::SingleThreadTaskRunner> encoder_task_runner_;
// All the below members except |weak_factory_| are accessed from
// |encoder_thread_| only (if it's running).
// JEA may open multiple devices for different input parameters.
// We handle the |encoded_instances_| by order for keeping user's input order.
std::queue<std::unique_ptr<EncodedInstance>> encoded_instances_;
std::queue<std::unique_ptr<EncodedInstanceDmaBuf>> encoded_instances_dma_buf_;
// Point to |this| for use in posting tasks from the encoder thread back to
// the ChildThread.
base::WeakPtr<V4L2JpegEncodeAccelerator> weak_ptr_;
// Weak factory for producing weak pointers on the child thread.
base::WeakPtrFactory<V4L2JpegEncodeAccelerator> weak_factory_;
DISALLOW_COPY_AND_ASSIGN(V4L2JpegEncodeAccelerator);
};
} // namespace media
#endif // MEDIA_GPU_V4L2_V4L2_JPEG_ENCODE_ACCELERATOR_H_