| // Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #ifndef MEDIA_GPU_H264_DECODER_H_ |
| #define MEDIA_GPU_H264_DECODER_H_ |
| |
| #include <stddef.h> |
| #include <stdint.h> |
| |
| #include <memory> |
| #include <vector> |
| |
| #include "base/containers/span.h" |
| #include "base/macros.h" |
| #include "base/memory/ref_counted.h" |
| #include "media/base/limits.h" |
| #include "media/base/subsample_entry.h" |
| #include "media/gpu/accelerated_video_decoder.h" |
| #include "media/gpu/h264_dpb.h" |
| #include "media/gpu/media_gpu_export.h" |
| #include "media/video/h264_parser.h" |
| #include "ui/gfx/geometry/rect.h" |
| #include "ui/gfx/geometry/size.h" |
| |
| namespace media { |
| |
| // Clients of this class are expected to pass H264 Annex-B byte stream |
| // and are expected to provide an implementation of H264Accelerator for |
| // offloading final steps of the decoding process. |
| // |
| // This class must be created, called and destroyed on a single thread, and |
| // does nothing internally on any other thread. |
| class MEDIA_GPU_EXPORT H264Decoder : public AcceleratedVideoDecoder { |
| public: |
| class MEDIA_GPU_EXPORT H264Accelerator { |
| public: |
| // Methods may return kTryAgain if they need additional data (provided |
| // independently) in order to proceed. Examples are things like not having |
| // an appropriate key to decode encrypted content, or needing to wait |
| // until hardware buffers are available. This is not considered an |
| // unrecoverable error, but rather a pause to allow an application to |
| // independently provide the required data. When H264Decoder::Decode() |
| // is called again, it will attempt to resume processing of the stream |
| // by calling the same method again. |
| enum class Status { |
| // Operation completed successfully. |
| kOk, |
| |
| // Operation failed. |
| kFail, |
| |
| // Operation failed because some external data is missing. Retry the same |
| // operation later, once the data has been provided. |
| kTryAgain, |
| |
| // Operation is not supported. Used by SetStream() to indicate that the |
| // Accelerator can not handle this operation. |
| kNotSupported, |
| }; |
| |
| H264Accelerator(); |
| virtual ~H264Accelerator(); |
| |
| // Create a new H264Picture that the decoder client can use for decoding |
| // and pass back to this accelerator for decoding or reference. |
| // When the picture is no longer needed by decoder, it will just drop |
| // its reference to it, and it may do so at any time. |
| // Note that this may return nullptr if accelerator is not able to provide |
| // any new pictures at given time. The decoder is expected to handle |
| // this situation as normal and return from Decode() with kRanOutOfSurfaces. |
| virtual scoped_refptr<H264Picture> CreateH264Picture() = 0; |
| |
| // Submit metadata for the current frame, providing the current |sps| and |
| // |pps| for it, |dpb| has to contain all the pictures in DPB for current |
| // frame, and |ref_pic_p0/b0/b1| as specified in the H264 spec. Note that |
| // depending on the frame type, either p0, or b0 and b1 are used. |pic| |
| // contains information about the picture for the current frame. |
| // Note that this does not run decode in the accelerator and the decoder |
| // is expected to follow this call with one or more SubmitSlice() calls |
| // before calling SubmitDecode(). |
| // Returns kOk if successful, kFail if there are errors, or kTryAgain if |
| // the accelerator needs additional data before being able to proceed. |
| virtual Status SubmitFrameMetadata( |
| const H264SPS* sps, |
| const H264PPS* pps, |
| const H264DPB& dpb, |
| const H264Picture::Vector& ref_pic_listp0, |
| const H264Picture::Vector& ref_pic_listb0, |
| const H264Picture::Vector& ref_pic_listb1, |
| scoped_refptr<H264Picture> pic) = 0; |
| |
| // Used for handling CENCv1 streams where the entire slice header, except |
| // for the NALU type byte, is encrypted. |data| represents the encrypted |
| // ranges which will include any SEI NALUs along with the encrypted slice |
| // NALU. |subsamples| specifies what is encrypted and should have just a |
| // single clear byte for each and the rest is encrypted. |sps_nalu_data| |
| // and |pps_nalu_data| are the SPS and PPS NALUs respectively. |
| // |slice_header_out| should have its fields filled in upon successful |
| // return. Returns kOk if successful, kFail if there are errors, or |
| // kTryAgain if the accelerator needs additional data before being able to |
| // proceed. |
| virtual Status ParseEncryptedSliceHeader( |
| const std::vector<base::span<const uint8_t>>& data, |
| const std::vector<SubsampleEntry>& subsamples, |
| const std::vector<uint8_t>& sps_nalu_data, |
| const std::vector<uint8_t>& pps_nalu_data, |
| H264SliceHeader* slice_header_out); |
| |
| // Submit one slice for the current frame, passing the current |pps| and |
| // |pic| (same as in SubmitFrameMetadata()), the parsed header for the |
| // current slice in |slice_hdr|, and the reordered |ref_pic_listX|, |
| // as per H264 spec. |
| // |data| pointing to the full slice (including the unparsed header) of |
| // |size| in bytes. |
| // |subsamples| specifies which part of the slice data is encrypted. |
| // This must be called one or more times per frame, before SubmitDecode(). |
| // Note that |data| does not have to remain valid after this call returns. |
| // Returns kOk if successful, kFail if there are errors, or kTryAgain if |
| // the accelerator needs additional data before being able to proceed. |
| virtual Status SubmitSlice( |
| const H264PPS* pps, |
| const H264SliceHeader* slice_hdr, |
| const H264Picture::Vector& ref_pic_list0, |
| const H264Picture::Vector& ref_pic_list1, |
| scoped_refptr<H264Picture> pic, |
| const uint8_t* data, |
| size_t size, |
| const std::vector<SubsampleEntry>& subsamples) = 0; |
| |
| // Execute the decode in hardware for |pic|, using all the slices and |
| // metadata submitted via SubmitFrameMetadata() and SubmitSlice() since |
| // the previous call to SubmitDecode(). |
| // Returns kOk if successful, kFail if there are errors, or kTryAgain if |
| // the accelerator needs additional data before being able to proceed. |
| virtual Status SubmitDecode(scoped_refptr<H264Picture> pic) = 0; |
| |
| // Schedule output (display) of |pic|. Note that returning from this |
| // method does not mean that |pic| has already been outputted (displayed), |
| // but guarantees that all pictures will be outputted in the same order |
| // as this method was called for them. Decoder may drop its reference |
| // to |pic| after calling this method. |
| // Return true if successful. |
| virtual bool OutputPicture(scoped_refptr<H264Picture> pic) = 0; |
| |
| // Reset any current state that may be cached in the accelerator, dropping |
| // any cached parameters/slices that have not been committed yet. |
| virtual void Reset() = 0; |
| |
| // Notifies the accelerator whenever there is a new stream to process. |
| // |stream| is the data in annex B format, which may include SPS and PPS |
| // NALUs when there is a configuration change. The first frame must contain |
| // the SPS and PPS NALUs. SPS and PPS NALUs may not be encrypted. |
| // |decrypt_config| is the config for decrypting the stream. The accelerator |
| // should use |decrypt_config| to keep track of the parts of |stream| that |
| // are encrypted. If kTryAgain is returned, the decoder will retry this call |
| // later. This method has a default implementation that returns |
| // kNotSupported. |
| virtual Status SetStream(base::span<const uint8_t> stream, |
| const DecryptConfig* decrypt_config); |
| |
| private: |
| DISALLOW_COPY_AND_ASSIGN(H264Accelerator); |
| }; |
| |
| H264Decoder(std::unique_ptr<H264Accelerator> accelerator, |
| VideoCodecProfile profile, |
| const VideoColorSpace& container_color_space = VideoColorSpace()); |
| ~H264Decoder() override; |
| |
| // AcceleratedVideoDecoder implementation. |
| void SetStream(int32_t id, const DecoderBuffer& decoder) override; |
| bool Flush() override WARN_UNUSED_RESULT; |
| void Reset() override; |
| DecodeResult Decode() override WARN_UNUSED_RESULT; |
| gfx::Size GetPicSize() const override; |
| gfx::Rect GetVisibleRect() const override; |
| VideoCodecProfile GetProfile() const override; |
| uint8_t GetBitDepth() const override; |
| size_t GetRequiredNumOfPictures() const override; |
| size_t GetNumReferenceFrames() const override; |
| |
| // Return true if we need to start a new picture. |
| static bool IsNewPrimaryCodedPicture(const H264Picture* curr_pic, |
| int curr_pps_id, |
| const H264SPS* sps, |
| const H264SliceHeader& slice_hdr); |
| |
| // Fill a H264Picture in |pic| from given |sps| and |slice_hdr|. Return false |
| // when there is an error. |
| static bool FillH264PictureFromSliceHeader(const H264SPS* sps, |
| const H264SliceHeader& slice_hdr, |
| H264Picture* pic); |
| |
| private: |
| // Internal state of the decoder. |
| enum State { |
| // After initialization, need an SPS. |
| kNeedStreamMetadata, |
| // Ready to decode from any point. |
| kDecoding, |
| // After Reset(), need a resume point. |
| kAfterReset, |
| // The following keep track of what step is next in Decode() processing |
| // in order to resume properly after H264Decoder::kTryAgain (or another |
| // retryable error) is returned. The next time Decode() is called the call |
| // that previously failed will be retried and execution continues from |
| // there (if possible). |
| kParseSliceHeader, |
| kTryPreprocessCurrentSlice, |
| kEnsurePicture, |
| kTryNewFrame, |
| kTryCurrentSlice, |
| // Error in decode, can't continue. |
| kError, |
| }; |
| |
| // Process H264 stream structures. |
| bool ProcessSPS(int sps_id, bool* need_new_buffers); |
| |
| // Processes a CENCv1 encrypted slice header and fills in |curr_slice_hdr_| |
| // with the relevant parsed fields. |
| H264Accelerator::Status ProcessEncryptedSliceHeader( |
| const std::vector<SubsampleEntry>& subsamples); |
| |
| // Process current slice header to discover if we need to start a new picture, |
| // finishing up the current one. |
| H264Accelerator::Status PreprocessCurrentSlice(); |
| // Process current slice as a slice of the current picture. |
| H264Accelerator::Status ProcessCurrentSlice(); |
| |
| // Initialize the current picture according to data in |slice_hdr|. |
| bool InitCurrPicture(const H264SliceHeader* slice_hdr); |
| |
| // Initialize |pic| as a "non-existing" picture (see spec) with |frame_num|, |
| // to be used for frame gap concealment. |
| bool InitNonexistingPicture(scoped_refptr<H264Picture> pic, int frame_num); |
| |
| // Calculate picture order counts for |pic| on initialization |
| // of a new frame (see spec). |
| bool CalculatePicOrderCounts(scoped_refptr<H264Picture> pic); |
| |
| // Update PicNum values in pictures stored in DPB on creation of |
| // a picture with |frame_num|. |
| void UpdatePicNums(int frame_num); |
| |
| bool UpdateMaxNumReorderFrames(const H264SPS* sps); |
| |
| // Prepare reference picture lists for the current frame. |
| void PrepareRefPicLists(); |
| // Prepare reference picture lists for the given slice. |
| bool ModifyReferencePicLists(const H264SliceHeader* slice_hdr, |
| H264Picture::Vector* ref_pic_list0, |
| H264Picture::Vector* ref_pic_list1); |
| |
| // Construct initial reference picture lists for use in decoding of |
| // P and B pictures (see 8.2.4 in spec). |
| void ConstructReferencePicListsP(); |
| void ConstructReferencePicListsB(); |
| |
| // Helper functions for reference list construction, per spec. |
| int PicNumF(const H264Picture& pic); |
| int LongTermPicNumF(const H264Picture& pic); |
| |
| // Perform the reference picture lists' modification (reordering), as |
| // specified in spec (8.2.4). |
| // |
| // |list| indicates list number and should be either 0 or 1. |
| bool ModifyReferencePicList(const H264SliceHeader* slice_hdr, |
| int list, |
| H264Picture::Vector* ref_pic_listx); |
| |
| // Perform reference picture memory management operations (marking/unmarking |
| // of reference pictures, long term picture management, discarding, etc.). |
| // See 8.2.5 in spec. |
| bool HandleMemoryManagementOps(scoped_refptr<H264Picture> pic); |
| bool ReferencePictureMarking(scoped_refptr<H264Picture> pic); |
| bool SlidingWindowPictureMarking(); |
| |
| // Handle a gap in frame_num in the stream up to |frame_num|, by creating |
| // "non-existing" pictures (see spec). |
| bool HandleFrameNumGap(int frame_num); |
| |
| // Start processing a new frame. |
| H264Accelerator::Status StartNewFrame(const H264SliceHeader* slice_hdr); |
| |
| // All data for a frame received, process it and decode. |
| H264Accelerator::Status FinishPrevFrameIfPresent(); |
| |
| // Called after we are done processing |pic|. Performs all operations to be |
| // done after decoding, including DPB management, reference picture marking |
| // and memory management operations. |
| // This will also output pictures if any have become ready to be outputted |
| // after processing |pic|. |
| bool FinishPicture(scoped_refptr<H264Picture> pic); |
| |
| // Clear DPB contents and remove all surfaces in DPB from *in_use_ list. |
| // Cleared pictures will be made available for decode, unless they are |
| // at client waiting to be displayed. |
| void ClearDPB(); |
| |
| // Commits all pending data for HW decoder and starts HW decoder. |
| H264Accelerator::Status DecodePicture(); |
| |
| // Notifies client that a picture is ready for output. |
| bool OutputPic(scoped_refptr<H264Picture> pic); |
| |
| // Output all pictures in DPB that have not been outputted yet. |
| bool OutputAllRemainingPics(); |
| |
| // Decoder state. |
| State state_; |
| |
| // The colorspace for the h264 container. |
| const VideoColorSpace container_color_space_; |
| |
| // Parser in use. |
| H264Parser parser_; |
| |
| // Most recent call to SetStream(). |
| const uint8_t* current_stream_ = nullptr; |
| size_t current_stream_size_ = 0; |
| |
| // Decrypting config for the most recent data passed to SetStream(). |
| std::unique_ptr<DecryptConfig> current_decrypt_config_; |
| |
| // Keep track of when SetStream() is called so that |
| // H264Accelerator::SetStream() can be called. |
| bool current_stream_has_been_changed_ = false; |
| |
| // DPB in use. |
| H264DPB dpb_; |
| |
| // Current stream buffer id; to be assigned to pictures decoded from it. |
| int32_t stream_id_ = -1; |
| |
| // Picture currently being processed/decoded. |
| scoped_refptr<H264Picture> curr_pic_; |
| |
| // Reference picture lists, constructed for each frame. |
| H264Picture::Vector ref_pic_list_p0_; |
| H264Picture::Vector ref_pic_list_b0_; |
| H264Picture::Vector ref_pic_list_b1_; |
| |
| // Global state values, needed in decoding. See spec. |
| int max_frame_num_; |
| int max_pic_num_; |
| int max_long_term_frame_idx_; |
| size_t max_num_reorder_frames_; |
| |
| int prev_frame_num_; |
| int prev_ref_frame_num_; |
| int prev_frame_num_offset_; |
| bool prev_has_memmgmnt5_; |
| |
| // Values related to previously decoded reference picture. |
| bool prev_ref_has_memmgmnt5_; |
| int prev_ref_top_field_order_cnt_; |
| int prev_ref_pic_order_cnt_msb_; |
| int prev_ref_pic_order_cnt_lsb_; |
| H264Picture::Field prev_ref_field_; |
| |
| // Currently active SPS and PPS. |
| int curr_sps_id_; |
| int curr_pps_id_; |
| |
| // Last PPS that was parsed. Used for full sample encryption, which has the |
| // assumption this is streaming content which does not switch between |
| // different PPSes in the stream (they are present once in the container for |
| // the stream). |
| int last_parsed_pps_id_; |
| |
| // Copies of the last SPS and PPS NALUs, used for full sample encryption. |
| std::vector<uint8_t> last_sps_nalu_; |
| std::vector<uint8_t> last_pps_nalu_; |
| |
| // Current NALU and slice header being processed. |
| std::unique_ptr<H264NALU> curr_nalu_; |
| std::unique_ptr<H264SliceHeader> curr_slice_hdr_; |
| |
| // Encrypted SEI NALUs preceding a fully encrypted slice NALU. We need to |
| // save these that are part of a single sample so they can all be decrypted |
| // together. |
| std::vector<base::span<const uint8_t>> encrypted_sei_nalus_; |
| std::vector<SubsampleEntry> sei_subsamples_; |
| |
| // These are absl::nullopt unless get recovery point SEI message after Reset. |
| // A frame_num of the frame at output order that is correct in content. |
| absl::optional<int> recovery_frame_num_; |
| // A value in the recovery point SEI message to compute |recovery_frame_num_| |
| // later. |
| absl::optional<int> recovery_frame_cnt_; |
| |
| // Output picture size. |
| gfx::Size pic_size_; |
| // Output visible cropping rect. |
| gfx::Rect visible_rect_; |
| |
| // Profile of input bitstream. |
| VideoCodecProfile profile_; |
| // Bit depth of input bitstream. |
| uint8_t bit_depth_ = 0; |
| |
| // PicOrderCount of the previously outputted frame. |
| int last_output_poc_; |
| |
| const std::unique_ptr<H264Accelerator> accelerator_; |
| |
| DISALLOW_COPY_AND_ASSIGN(H264Decoder); |
| }; |
| |
| } // namespace media |
| |
| #endif // MEDIA_GPU_H264_DECODER_H_ |