From d059b2be30bb362d96bd53137167f39953b213c1 Mon Sep 17 00:00:00 2001 From: Guido Urdaneta Date: Mon, 4 Mar 2024 23:43:40 +0100 Subject: [PATCH] Add RTCRtpEncodedSource --- index.html | 336 ++++++++++++++++++++++++++++++++++++++----- webrtc-extensions.js | 2 +- 2 files changed, 300 insertions(+), 38 deletions(-) diff --git a/index.html b/index.html index b2a400f..82edcc3 100644 --- a/index.html +++ b/index.html @@ -1710,43 +1710,305 @@

Modifications to existing procedures

-
-

Event summary

-

- The following events fire on {{RTCIceTransport}} objects:

- - - - - - - - - - - - - - - - - - - - - - - - - -
Event nameInterfaceFired when...
icecandidatepairadd{{RTCIceCandidatePairEvent}} - The [= ICE agent =] has formed a candidate pair and is making it available to the script. -
icecandidatepairremove{{RTCIceCandidatePairEvent}} - The [= ICE agent =] has picked a candidate pair to remove, and unless the operation is canceled by invoking the preventDefault() method on the event, it will be removed. -
icecandidatepairnominate{{RTCIceCandidatePairEvent}} - The [= ICE agent =] has picked a valid candidate pair to {{nominate}}, and unless the operation is canceled by - invoking the preventDefault() method on the event, it will be {{nominated}}. -
+
+

Encoded frame source for RTCRtpSender

+
+

{{RTCRtpSenderEncodedSource}} interface

+

+ The RTCRtpSenderEncodedSource + interface allows applications to inject encoded frames into an + {{RTCRtpSender}}. This is intended to support use cases + where an application already has access to the encoded frames it wants + to send. An example is forwarding encoded frames coming from one or more + peer connections to another peer connection. In this case, the + application can use an encoded transform on the RTCRtpReceivers of the + input peer connections to get the encoded frames and inject them on the + RTCRtpSender of the output peer connection without having to decode and + reencode the frames. +

+ 
+        [Exposed=DedicatedWorker]
+        interface RTCRtpSenderEncodedSource {
+          constructor(DOMString kind);
+          readonly attribute RTCRtpSenderEncodedSourceHandle handle;
+          undefined enqueue((RTCEncodedVideoFrame or RTCEncodedAudioFrame) encodedFrame);
+          undefined stop();
+        };
+
+

Constructor

+

+ When the {{RTCRtpSenderEncodedSource}} constructor + is invoked with a |kind| argument, the user agent MUST run the following steps: +

    +
  1. If |kind| is neither the string `"audio"` nor the string `"video"`, throw a {{TypeError}}.
    2. +
  2. Let |source| be the newly created {{RTCRtpSenderEncodedSource}} object
    3. +
  3. Let |source| have an [[\IsStopped]] + internal slot, initialized to false.
    4. +
  4. Let |source| have a [[\Handle]] + internal slot
    5. +
  5. [=Create an RTCRtpSenderEncodedSourceHandle=], given |source| as argument and assign it to + |source|.{{RTCRtpSenderEncodedSource/[[Handle]]}}
    6. +
  6. Let |source| have a [[\Kind]] + internal slot, initialized to |kind|.
    7. +
  7. Let |source| have a [[\Sender]] + internal slot, initialized to null.
    8. +
+

+

Attributes

+
+
+ handle of type + {{RTCRtpSenderEncodedSourceHandle}}, readonly +
+
+

+ This attribute is the [=handle associated with=] this {{RTCRtpSenderEncodedSource}}. +

+

+ On getting, if {{RTCRtpSenderEncodedSource/[[Handle]]}}.{{RTCRtpSenderEncodedSourceHandle/[[IsDetached]]}} + is `true`, the attribute MUST return `null`. Otherwise, it MUST + return the value of the {{RTCRtpSenderEncodedSource/[[Handle]]}} slot. +

+
+
+

Methods

+
+
+ stop +
+
+

+ The User Agent MUST run the following steps: +

    +
  1. Let |source| be the object on which the method is invoked.
    2. +
  2. If |source|.{{RTCRtpSenderEncodedSource/[[IsStopped]]}} is true, abort these steps. +
  3. Set |source|.{{RTCRtpSenderEncodedSource/[[IsStopped]]}} to true. +
+

+
+
+ enqueue +
+
+

+ The User Agent MUST run the following steps: +

    +
  1. Let |source| be the object on which the method is invoked.
    2. +
  2. Let |encodedFrame| be the argument given to this method
    3. +
  3. If |source|.{{RTCRtpSenderEncodedSource/[[IsStopped]]}} is true, + throw an {{InvalidStateError}} {{DOMException}}, and abort these steps.
    4. +
  4. If |source|.{{RTCRtpSenderEncodedSource/[[Kind]]}} is `"video"` + and |encodedFrame| is not an {{RTCEncodedVideoFrame}}, throw an + {{InvalidAccessError}} {{DOMException}}, and abort these steps.
    5. +
  5. If |source|.{{RTCRtpSenderEncodedSource/[[Kind]]}} is `"audio"` + and |encodedFrame| is not an {{RTCEncodedAudioFrame}}, throw an + {{InvalidAccessError}} {{DOMException}}, and abort these steps.
    6. +
  6. Let |data| be |encodedFrame|.`[[data]]`
    7. +
  7. Let |handle| be the [=handle associated with=] |source|. + |handle| and |source| might exist in different realms.
    8. +
  8. If |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}} is `null`, throw an + {{InvalidStateError}} {{DOMException}}, and abort these steps.
    9. +
  9. Let |serializedFrame| be [=StructuredSerializeWithTransfer=](|encodedFrame|,|data|).
    10. +
  10. Let |frameCopy| be [=StructuredDeserialize=](|serializedFrame|, |handle|’s [=relevant realm=]).
    11. +
  11. Enqueue |frameCopy| to |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}}'s + packetizer, to be processed [=in parallel=].
    12. +
+

+
+
+
+
d +

RTCRtpSenderEncodedSourceHandle interface

+ Interface that acts as an opaque cross-realm reference to an {{RTCRtpSenderEncodedSource}}. + It is transferable to the {{Window}} context and can be assigned to an {{RTCRtpSender}} + using {{RTCRtpSender/replaceTrack}}. An {{RTCRtpSenderEncodedSource}} always + stays associated to its {{RTCRtpSenderEncodedSourceHandle}} regardless of the + {{RTCRtpSenderEncodedSourceHandle}}'s realm, similar to the way a {{MediaStreamTrack}} + source maintains references to the tracks it backs. The main difference is that an + {{RTCRtpSenderEncodedSource}} has exactly one associated handle. + 
+        [Exposed=(Window,DedicatedWorker), Transferable]
+        interface RTCRtpSenderEncodedSourceHandle {};
+
+ To create an RTCRtpSenderEncodedSourceHandle object, + given an {{RTCRtpSenderEncodedSource}} |source|: +
    +
  1. Let |handle| be a new {{RTCRtpSenderEncodedSourceHandle}} object.
    2. +
  2. Let |handle| have a [[\Sender]] + internal slot, initialized to `null`.
    3. +
  3. Let |handle| have a [[\Kind]] + internal slot, initialized to |source|.{{RTCRtpSenderEncodedSource/[[Kind]]}}.
    4. +
  4. Let |handle| have an [[\IsDetached]] + internal slot, initialized to `false`.
    5. +
  5. Let |handle| have an [[\Id]] + internal slot, initialized to a unique string.
    6. +
  6. Return |handle|.
    7. +
+ {{RTCRtpSenderEncodedSourceHandle}} objects are transferable. + Their transfer steps, given |value|, + and |dataHolder|, are: +
    +
  1. If |value|.{{RTCRtpSenderEncodedSourceHandle/[[IsDetached]]}} is `true`, + throw a {{DataCloneError}} {{DOMException}}.
    2. +
  2. Set |dataHolder|.`[[Kind]]` to |value|.{{RTCRtpSenderEncodedSourceHandle/[[Kind]]}}
    3. +
  3. Set |dataHolder|.`[[Id]]` to |value|.{{RTCRtpSenderEncodedSourceHandle/[[Id]]}}
    4. +
  4. Set |value|.{{RTCRtpSenderEncodedSourceHandle/[[IsDetached]]}} to `true`.
    5. +
+ The {{RTCRtpSenderEncodedSourceHandle}} transfer-receiving steps, + given |dataHolder| and |handle|, are: +
    +
  1. Set |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Kind]]}} to |dataHolder|.`[[Kind]]`
    2. +
  2. Set |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Id]]}} to |dataHolder|.`[[Id]]`
    3. +
  3. Set |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}} to `null`
    4. +
+ We define the handle associated with an + {{RTCRtpSenderEncodedSource}} |source| to be an + {{RTCRtpSenderEncodedSourceHandle}} object |handle| for which the following holds: +
    +
  1. |handle|.{{RTCRtpSenderEncodedSourceHandle/[[Id]]}} equals + |source|.{{RTCRtpSenderEncodedSource/[[Handle]]}}.{{RTCRtpSenderEncodedSourceHandle/[[Id]]}}
    2. +
  2. |handle|.{{RTCRtpSenderEncodedSourceHandle/[[IsDetached]]}} is `false`
    3. +
+ Given that {{RTCRtpSenderEncodedSourceHandle}} is + transferable, but not + serializable, there is at + most one [=handle associated with=] a {{RTCRtpSenderEncodedSource}} and + it is possible that they exist in different realms. The user agent can use + whatever mechanism works best to keep track of the potentially cross-realm + association between an encoded source and its associated handle, similarly + to the way a {{MediaStreamTrack}} and its corresponding + source are connected. +
+
+

RTCRtpSender interface extensions

+ 
+        partial interface RTCRtpSender {
+          undefined replaceTrack(RTCRtpSenderEncodedSourceHandle withSourceHandle);
+          readonly attribute RTCRtpSenderEncodedSourceHandle encodedSourceHandle;
+      };
+
+

Attributes

+
+
+ encodedSourceHandle of type + {{RTCRtpSenderEncodedSourceHandle}}, readonly +
+
+ This attribute is the handle for the {{RTCRtpSenderEncodedSource}} associated with this + {{RTCRtpSender}} object. If the encoded source is not `null` and is stopped, + the {{RTCRtpSender}} SHOULD send one black (video) frame per second and MUST NOT + send (audio). If {{RTCRtpSender/encodedSourceHandle}} is `null` + and {{RTCRtpSender/track}} is `null` then the {{RTCRtpSender}} does not send. + On getting, the attribute MUST return the value of the + {{RTCRtpSender/[[SenderEncodedSourceHandle]]}} slot. +
+
+

Methods

+
+
+ replaceTrack +
+
+

+ Attempts to replace the {{RTCRtpSender}}'s current {{RTCRtpSender/track}} or + {{RTCRtpSender/encodedSourceHandle}} with + another encoded source without renegotiation. When this method is invoked, + the user agent MUST run the following steps: +

    +
  1. Let |sender| be the {{RTCRtpSender}} object on which the method is invoked.
    2. +
  2. Let |transceiver| be the {{RTCRtpTransceiver}} object associated with |sender|.
    3. +
  3. Let |connection| be the {{RTCPeerConnection}} object associated with |sender|.
    4. +
  4. Let |withSourceHandle| be the argument to this method.
    5. +
  5. If |withSourceHandle| is non-null and |withSourceHandle|.{{RTCRtpSenderEncodedSourceHandle/[[Kind]]}} + differs from the transceiver kind of transceiver, return a {{Promise}} rejected with a + newly created {{TypeError}}.
    6. +
  6. Return the result of chaining the following steps to |connection|'s [=operations chain=]: +
      +
    1. If |transceiver|.`[[Stopping]]` + is `true`, return a {{Promise}} rejected with a newly created + {{InvalidStateError}} {{DOMException}}.
      2. +
    2. Let |p| be a new {{Promise}}.
      3. +
    3. Let |sending| be `true` if |transceiver|.`[[CurrentDirection]]` + is "sendrecv" or "sendonly", and `false` otherwise.
      4. +
    4. Run the following steps [=in parallel=]: +
        +
      1. If |sending| is `true`, and |withSourceHandle| is `null`, have the sender stop sending.
        2. +
      2. Queue a task that runs the following steps: +
          +
        1. If |connection|.{{RTCPeerConnection/[[IsClosed]]}} is `true`, abort these steps.
          2. +
        2. Set |sender|.`[[SenderTrack]]` to `null`.
          3. +
        3. Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}} to |withSourceHandle|.
          4. +
        4. set |withSourceHandle|.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}} to |sender|.
          5. +
        5. Resolve |p| with `undefined`.
          6. +
        +
        3. +
      3. Return |p|.
        4. +
      +
      5. +
    +
    7. +
+

+
+
+
+
+

Modifications to existing procedures

+
    +
  • Make the following modification to the [=Create an RTCRtpSender=] algorithm: +
    • After step 2, insert the following step: +
      • Let sender have a [[\SenderEncodedSourceHandle]] + internal slot initialized to `null`.
      +
    +
    • +
  • Make the following modification to {{RTCPeerConnection}}.{{RTCPeerConnection/addTrack()}}: +
    • After step 8.1, insert the following step: +
        +
      • Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}} to `null`
        • +
      +
    +
    • +
  • Make the following modifications to {{RTCRtpSender}}.`removeTrack()`: +
      +
    1. + Replace step 9 with the following step: +
      • If |sender|.`[[SenderTrack]]` is `null` and + |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}} is `null`, + abort these steps.
      +
      2. +
    2. + After step 10, insert the following steps: +
        +
      1. Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}}.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}} to `null`
        2. +
      2. Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}} to `null`.
        3. +
      +
      3. +
    +
    • +
  • + Make the following modifications to {{RTCRtpSender}}.{{RTCRtpSender/setParameters()}}: +
      + Replace step 8 with the following step: +
        +
      • In parallel, configure the media stack to use parameters to transmit + |sender|.`[[SenderTrack]]` or |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}}.
        • +
      +
    +
    • +
  • + Make the following modifications to {{RTCRtpSender}}.`replaceTrack()`: +
      + After step 6.4.4.2, insert the following steps: +
        +
      1. Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}}.{{RTCRtpSenderEncodedSourceHandle/[[Sender]]}} to `null`.
        2. +
      2. Set |sender|.{{RTCRtpSender/[[SenderEncodedSourceHandle]]}} to `null`.
        3. +
      +
    +
    • +
+

diff --git a/webrtc-extensions.js b/webrtc-extensions.js index b755faa..5742f31 100644 --- a/webrtc-extensions.js +++ b/webrtc-extensions.js @@ -5,7 +5,7 @@ var respecConfig = { repoURL: "https://github.com/w3c/webrtc-extensions/", branch: "main" }, - "xref": ["html", "webidl", "webrtc", "hr-time", "mediacapture-streams", "webrtc-stats", "infra", "dom"], + "xref": ["html", "webidl", "webrtc", "hr-time", "mediacapture-streams", "webrtc-stats", "infra", "dom", "webrtc-encoded-transform"], "shortName": "webrtc-extensions", "specStatus": "ED", "subjectPrefix": "[webrtc-extensions]",