blob: 718000e961e8718998cc2435ad30c11d316dc7eb [file] [log] [blame]
/* Copyright 2018 The WebRTC project authors. All Rights Reserved.
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
// This is EXPERIMENTAL interface for media transport.
// The goal is to refactor WebRTC code so that audio and video frames
// are sent / received through the media transport interface. This will
// enable different media transport implementations, including QUIC-based
// media transport.
#include <api/transport/network_control.h>
#include <memory>
#include <string>
#include <utility>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/rtc_error.h"
#include "api/transport/media/audio_transport.h"
#include "api/transport/media/video_transport.h"
#include "api/units/data_rate.h"
#include "common_types.h" // NOLINT(build/include)
#include "rtc_base/copy_on_write_buffer.h"
#include "rtc_base/network_route.h"
namespace rtc {
class PacketTransportInternal;
class Thread;
} // namespace rtc
namespace webrtc {
class RtcEventLog;
class AudioPacketReceivedObserver {
virtual ~AudioPacketReceivedObserver() = default;
// Invoked for the first received audio packet on a given channel id.
// It will be invoked once for each channel id.
virtual void OnFirstAudioPacketReceived(int64_t channel_id) = 0;
// Used to configure stream allocations.
struct MediaTransportAllocatedBitrateLimits {
DataRate min_pacing_rate = DataRate::Zero();
DataRate max_padding_bitrate = DataRate::Zero();
DataRate max_total_allocated_bitrate = DataRate::Zero();
// Used to configure target bitrate constraints.
// If the value is provided, the constraint is updated.
// If the value is omitted, the value is left unchanged.
struct MediaTransportTargetRateConstraints {
absl::optional<DataRate> min_bitrate;
absl::optional<DataRate> max_bitrate;
absl::optional<DataRate> starting_bitrate;
// A collection of settings for creation of media transport.
struct MediaTransportSettings final {
MediaTransportSettings(const MediaTransportSettings&);
MediaTransportSettings& operator=(const MediaTransportSettings&);
// Group calls are not currently supported, in 1:1 call one side must set
// is_caller = true and another is_caller = false.
bool is_caller;
// Must be set if a pre-shared key is used for the call.
// TODO( This should become zero buffer in the distant
// future.
absl::optional<std::string> pre_shared_key;
// If present, this is a config passed from the caller to the answerer in the
// offer. Each media transport knows how to understand its own parameters.
absl::optional<std::string> remote_transport_parameters;
// If present, provides the event log that media transport should use.
// Media transport does not own it. The lifetime of |event_log| will exceed
// the lifetime of the instance of MediaTransportInterface instance.
RtcEventLog* event_log = nullptr;
// Callback to notify about network route changes.
class MediaTransportNetworkChangeCallback {
virtual ~MediaTransportNetworkChangeCallback() = default;
// Called when the network route is changed, with the new network route.
virtual void OnNetworkRouteChanged(
const rtc::NetworkRoute& new_network_route) = 0;
// State of the media transport. Media transport begins in the pending state.
// It transitions to writable when it is ready to send media. It may transition
// back to pending if the connection is blocked. It may transition to closed at
// any time. Closed is terminal: a transport will never re-open once closed.
enum class MediaTransportState {
// Callback invoked whenever the state of the media transport changes.
class MediaTransportStateCallback {
virtual ~MediaTransportStateCallback() = default;
// Invoked whenever the state of the media transport changes.
virtual void OnStateChanged(MediaTransportState state) = 0;
// Callback for RTT measurements on the receive side.
// TODO(nisse): Related interfaces: CallStatsObserver and RtcpRttStats. It's
// somewhat unclear what type of measurement is needed. It's used to configure
// NACK generation and playout buffer. Either raw measurement values or recent
// maximum would make sense for this use. Need consolidation of RTT signalling.
class MediaTransportRttObserver {
virtual ~MediaTransportRttObserver() = default;
// Invoked when a new RTT measurement is available, typically once per ACK.
virtual void OnRttUpdated(int64_t rtt_ms) = 0;
// Supported types of application data messages.
enum class DataMessageType {
// Application data buffer with the binary bit unset.
// Application data buffer with the binary bit set.
// Transport-agnostic control messages, such as open or open-ack messages.
// Parameters for sending data. The parameters may change from message to
// message, even within a single channel. For example, control messages may be
// sent reliably and in-order, even if the data channel is configured for
// unreliable delivery.
struct SendDataParams {
SendDataParams(const SendDataParams&);
DataMessageType type = DataMessageType::kText;
// Whether to deliver the message in order with respect to other ordered
// messages with the same channel_id.
bool ordered = false;
// If set, the maximum number of times this message may be
// retransmitted by the transport before it is dropped.
// Setting this value to zero disables retransmission.
// Must be non-negative. |max_rtx_count| and |max_rtx_ms| may not be set
// simultaneously.
absl::optional<int> max_rtx_count;
// If set, the maximum number of milliseconds for which the transport
// may retransmit this message before it is dropped.
// Setting this value to zero disables retransmission.
// Must be non-negative. |max_rtx_count| and |max_rtx_ms| may not be set
// simultaneously.
absl::optional<int> max_rtx_ms;
// Sink for callbacks related to a data channel.
class DataChannelSink {
virtual ~DataChannelSink() = default;
// Callback issued when data is received by the transport.
virtual void OnDataReceived(int channel_id,
DataMessageType type,
const rtc::CopyOnWriteBuffer& buffer) = 0;
// Callback issued when a remote data channel begins the closing procedure.
// Messages sent after the closing procedure begins will not be transmitted.
virtual void OnChannelClosing(int channel_id) = 0;
// Callback issued when a (remote or local) data channel completes the closing
// procedure. Closing channels become closed after all pending data has been
// transmitted.
virtual void OnChannelClosed(int channel_id) = 0;
// Media transport interface for sending / receiving encoded audio/video frames
// and receiving bandwidth estimate update from congestion control.
class MediaTransportInterface {
virtual ~MediaTransportInterface();
// Retrieves callers config (i.e. media transport offer) that should be passed
// to the callee, before the call is connected. Such config is opaque to SDP
// (sdp just passes it through). The config is a binary blob, so SDP may
// choose to use base64 to serialize it (or any other approach that guarantees
// that the binary blob goes through). This should only be called for the
// caller's perspective.
// This may return an unset optional, which means that the given media
// transport is not supported / disabled and shouldn't be reported in SDP.
// It may also return an empty string, in which case the media transport is
// supported, but without any extra settings.
// TODO(psla): Make abstract.
virtual absl::optional<std::string> GetTransportParametersOffer() const;
// Connect the media transport to the ICE transport.
// The implementation must be able to ignore incoming packets that don't
// belong to it.
// TODO(psla): Make abstract.
virtual void Connect(rtc::PacketTransportInternal* packet_transport);
// Start asynchronous send of audio frame. The status returned by this method
// only pertains to the synchronous operations (e.g.
// serialization/packetization), not to the asynchronous operation.
virtual RTCError SendAudioFrame(uint64_t channel_id,
MediaTransportEncodedAudioFrame frame) = 0;
// Start asynchronous send of video frame. The status returned by this method
// only pertains to the synchronous operations (e.g.
// serialization/packetization), not to the asynchronous operation.
virtual RTCError SendVideoFrame(
uint64_t channel_id,
const MediaTransportEncodedVideoFrame& frame) = 0;
// Used by video sender to be notified on key frame requests.
virtual void SetKeyFrameRequestCallback(
MediaTransportKeyFrameRequestCallback* callback);
// Requests a keyframe for the particular channel (stream). The caller should
// check that the keyframe is not present in a jitter buffer already (i.e.
// don't request a keyframe if there is one that you will get from the jitter
// buffer in a moment).
virtual RTCError RequestKeyFrame(uint64_t channel_id) = 0;
// Sets audio sink. Sink must be unset by calling SetReceiveAudioSink(nullptr)
// before the media transport is destroyed or before new sink is set.
virtual void SetReceiveAudioSink(MediaTransportAudioSinkInterface* sink) = 0;
// Registers a video sink. Before destruction of media transport, you must
// pass a nullptr.
virtual void SetReceiveVideoSink(MediaTransportVideoSinkInterface* sink) = 0;
// Adds a target bitrate observer. Before media transport is destructed
// the observer must be unregistered (by calling
// RemoveTargetTransferRateObserver).
// A newly registered observer will be called back with the latest recorded
// target rate, if available.
virtual void AddTargetTransferRateObserver(
TargetTransferRateObserver* observer);
// Removes an existing |observer| from observers. If observer was never
// registered, an error is logged and method does nothing.
virtual void RemoveTargetTransferRateObserver(
TargetTransferRateObserver* observer);
// Sets audio packets observer, which gets informed about incoming audio
// packets. Before destruction, the observer must be unregistered by setting
// nullptr.
// This method may be temporary, when the multiplexer is implemented (or
// multiplexer may use it to demultiplex channel ids).
virtual void SetFirstAudioPacketReceivedObserver(
AudioPacketReceivedObserver* observer);
// Intended for receive side. AddRttObserver registers an observer to be
// called for each RTT measurement, typically once per ACK. Before media
// transport is destructed the observer must be unregistered.
virtual void AddRttObserver(MediaTransportRttObserver* observer);
virtual void RemoveRttObserver(MediaTransportRttObserver* observer);
// Returns the last known target transfer rate as reported to the above
// observers.
virtual absl::optional<TargetTransferRate> GetLatestTargetTransferRate();
// Gets the audio packet overhead in bytes. Returned overhead does not include
// transport overhead (ipv4/6, turn channeldata, tcp/udp, etc.).
// If the transport is capable of fusing packets together, this overhead
// might not be a very accurate number.
// TODO(nisse): Deprecated.
virtual size_t GetAudioPacketOverhead() const;
// Corresponding observers for audio and video overhead. Before destruction,
// the observers must be unregistered by setting nullptr.
// TODO(nisse): Should move to per-stream objects, since packetization
// overhead can vary per stream, e.g., depending on negotiated extensions. In
// addition, we should move towards reporting total overhead including all
// layers. Currently, overhead of the lower layers is reported elsewhere,
// e.g., on route change between IPv4 and IPv6.
virtual void SetAudioOverheadObserver(OverheadObserver* observer) {}
// Registers an observer for network change events. If the network route is
// already established when the callback is added, |callback| will be called
// immediately with the current network route. Before media transport is
// destroyed, the callback must be removed.
virtual void AddNetworkChangeCallback(
MediaTransportNetworkChangeCallback* callback);
virtual void RemoveNetworkChangeCallback(
MediaTransportNetworkChangeCallback* callback);
// Sets a state observer callback. Before media transport is destroyed, the
// callback must be unregistered by setting it to nullptr.
// A newly registered callback will be called with the current state.
// Media transport does not invoke this callback concurrently.
virtual void SetMediaTransportStateCallback(
MediaTransportStateCallback* callback) = 0;
// Updates allocation limits.
// TODO(psla): Make abstract when downstream implementation implement it.
virtual void SetAllocatedBitrateLimits(
const MediaTransportAllocatedBitrateLimits& limits);
// Sets starting rate.
// TODO(psla): Make abstract when downstream implementation implement it.
virtual void SetTargetBitrateLimits(
const MediaTransportTargetRateConstraints& target_rate_constraints) {}
// Opens a data |channel_id| for sending. May return an error if the
// specified |channel_id| is unusable. Must be called before |SendData|.
virtual RTCError OpenChannel(int channel_id) = 0;
// Sends a data buffer to the remote endpoint using the given send parameters.
// |buffer| may not be larger than 256 KiB. Returns an error if the send
// fails.
virtual RTCError SendData(int channel_id,
const SendDataParams& params,
const rtc::CopyOnWriteBuffer& buffer) = 0;
// Closes |channel_id| gracefully. Returns an error if |channel_id| is not
// open. Data sent after the closing procedure begins will not be
// transmitted. The channel becomes closed after pending data is transmitted.
virtual RTCError CloseChannel(int channel_id) = 0;
// Sets a sink for data messages and channel state callbacks. Before media
// transport is destroyed, the sink must be unregistered by setting it to
// nullptr.
virtual void SetDataSink(DataChannelSink* sink) = 0;
// TODO(sukhanov): RtcEventLogs.
// If media transport factory is set in peer connection factory, it will be
// used to create media transport for sending/receiving encoded frames and
// this transport will be used instead of default RTP/SRTP transport.
// Currently Media Transport negotiation is not supported in SDP.
// If application is using media transport, it must negotiate it before
// setting media transport factory in peer connection.
class MediaTransportFactory {
virtual ~MediaTransportFactory() = default;
// Creates media transport.
// - Does not take ownership of packet_transport or network_thread.
// - Does not support group calls, in 1:1 call one side must set
// is_caller = true and another is_caller = false.
virtual RTCErrorOr<std::unique_ptr<MediaTransportInterface>>
CreateMediaTransport(rtc::PacketTransportInternal* packet_transport,
rtc::Thread* network_thread,
const MediaTransportSettings& settings);
// Creates a new Media Transport in a disconnected state. If the media
// transport for the caller is created, one can then call
// MediaTransportInterface::GetTransportParametersOffer on that new instance.
// TODO(psla): Make abstract.
virtual RTCErrorOr<std::unique_ptr<webrtc::MediaTransportInterface>>
CreateMediaTransport(rtc::Thread* network_thread,
const MediaTransportSettings& settings);
// Gets a transport name which is supported by the implementation.
// Different factories should return different transport names, and at runtime
// it will be checked that different names were used.
// For example, "rtp" or "generic" may be returned by two different
// implementations.
// The value returned by this method must never change in the lifetime of the
// factory.
// TODO(psla): Make abstract.
virtual std::string GetTransportName() const;
} // namespace webrtc