Fix bugs in related to browser connecting via WebRTC

examples.livemd

@@ -27,8 +27,7 @@ File.mkdir_p!(input_dir)
 out_dir = "#{data_dir}/output"
 File.mkdir_p!(out_dir)
 
-samples_url =
-  "https://raw.githubusercontent.com/membraneframework/static/gh-pages/samples"
+samples_url = "https://raw.githubusercontent.com/membraneframework/static/gh-pages/samples"
 
 unless File.exists?("#{input_dir}/bun.mp4") do
   %{status: 200, body: bbb_mp4} = Req.get!("#{samples_url}/big-buck-bunny/bun33s.mp4")

lib/boombox/pipeline.ex

@@ -305,8 +305,8 @@ defmodule Boombox.Pipeline do
 
   @spec create_input(Boombox.input(), Membrane.Pipeline.CallbackContext.t(), State.t()) ::
           Ready.t() | Wait.t()
-  defp create_input({:webrtc, signaling}, _ctx, state) do
-    Boombox.WebRTC.create_input(signaling, state.output, state)
+  defp create_input({:webrtc, signaling}, ctx, state) do
+    Boombox.WebRTC.create_input(signaling, state.output, ctx, state)
   end
 
   defp create_input({:mp4, location, opts}, _ctx, _state) do
@@ -331,8 +331,8 @@ defmodule Boombox.Pipeline do
 
   @spec create_output(Boombox.output(), Membrane.Pipeline.CallbackContext.t(), State.t()) ::
           {Ready.t() | Wait.t(), State.t()}
-  defp create_output({:webrtc, signaling}, _ctx, state) do
-    Boombox.WebRTC.create_output(signaling, state)
+  defp create_output({:webrtc, signaling}, ctx, state) do
+    Boombox.WebRTC.create_output(signaling, ctx, state)
   end
 
   defp create_output(_output, _ctx, state) do

lib/boombox/transcoder/video.ex

@@ -3,12 +3,16 @@ defmodule Boombox.Transcoder.Video do
 
   import Membrane.ChildrenSpec
   require Membrane.Logger
+  alias Membrane.RemoteStream
   alias Membrane.{ChildrenSpec, H264, H265, RawVideo, VP8}
 
   @type video_stream_format :: VP8.t() | H264.t() | H265.t() | RawVideo.t()
 
   defguard is_video_format(format)
-           when is_struct(format) and format.__struct__ in [VP8, H264, H265, RawVideo]
+           when is_struct(format) and
+                  (format.__struct__ in [VP8, H264, H265, RawVideo] or
+                     (format.__struct__ == RemoteStream and format.content_format == VP8 and
+                        format.type == :packetized))
 
   @spec plug_video_transcoding(
           ChildrenSpec.builder(),
@@ -70,6 +74,13 @@ defmodule Boombox.Transcoder.Video do
     builder |> child(:vp8_decoder, %VP8.Decoder{})
   end
 
+  defp maybe_plug_parser_and_decoder(builder, %RemoteStream{
+         content_format: VP8,
+         type: :packetized
+       }) do
+    builder |> child(:vp8_decoder, %VP8.Decoder{})
+  end
+
   defp maybe_plug_parser_and_decoder(builder, %RawVideo{}) do
     builder
   end

lib/boombox/webrtc.ex

@@ -3,16 +3,19 @@ defmodule Boombox.WebRTC do
 
   import Membrane.ChildrenSpec
   require Membrane.Pad, as: Pad
+  alias Membrane.WebRTC.SimpleWebSocketServer
   alias Boombox.Pipeline.{Ready, State, Wait}
-  alias Membrane.H264
-  alias Membrane.VP8
+  alias Membrane.{H264, RemoteStream, VP8}
+  alias Membrane.Pipeline.CallbackContext
+  alias Membrane.WebRTC.SignalingChannel
 
   @type output_webrtc_state :: %{negotiated_video_codecs: [:vp8 | :h264] | nil}
   @type webrtc_sink_new_tracks :: [%{id: term, kind: :audio | :video}]
 
-  @spec create_input(Boombox.webrtc_signaling(), Boombox.output(), State.t()) :: Wait.t()
-  def create_input(signaling, output, state) do
-    signaling = resolve_signaling(signaling)
+  @spec create_input(Boombox.webrtc_signaling(), Boombox.output(), CallbackContext.t(), State.t()) ::
+          Wait.t()
+  def create_input(signaling, output, ctx, state) do
+    signaling = resolve_signaling(signaling, ctx.utility_supervisor)
 
     keyframe_interval =
       case output do
@@ -69,9 +72,10 @@ defmodule Boombox.WebRTC do
     %Ready{track_builders: track_builders}
   end
 
-  @spec create_output(Boombox.webrtc_signaling(), State.t()) :: {Ready.t() | Wait.t(), State.t()}
-  def create_output(signaling, state) do
-    signaling = resolve_signaling(signaling)
+  @spec create_output(Boombox.webrtc_signaling(), CallbackContext.t(), State.t()) ::
+          {Ready.t() | Wait.t(), State.t()}
+  def create_output(signaling, ctx, state) do
+    signaling = resolve_signaling(signaling, ctx.utility_supervisor)
     startup_tracks = if webrtc_input?(state), do: [:audio, :video], else: []
 
     spec =
@@ -83,10 +87,17 @@ defmodule Boombox.WebRTC do
 
     state = %{state | output_webrtc_state: %{negotiated_video_codecs: nil}}
 
-    status =
-      if webrtc_input?(state),
-        do: %Wait{actions: [spec: spec]},
-        else: %Ready{actions: [spec: spec]}
+    {status, state} =
+      if webrtc_input?(state) do
+        # let's spawn websocket server for webrtc source before the source starts
+        {:webrtc, input_signaling} = state.input
+        signaling_channel = resolve_signaling(input_signaling, ctx.utility_supervisor)
+        state = %{state | input: {:webrtc, signaling_channel}}
+
+        {%Wait{actions: [spec: spec]}, state}
+      else
+        {%Ready{actions: [spec: spec]}, state}
+      end
 
     {status, state}
   end
@@ -160,6 +171,9 @@ defmodule Boombox.WebRTC do
               %VP8{} = vp8 when vp8_negotiated? ->
                 vp8
 
+              %RemoteStream{content_format: VP8, type: :packetized} when vp8_negotiated? ->
+                VP8
+
               _format when h264_negotiated? ->
                 %H264{alignment: :nalu, stream_structure: :annexb}
 
@@ -175,14 +189,16 @@ defmodule Boombox.WebRTC do
     %Ready{actions: [spec: spec], eos_info: Map.values(tracks)}
   end
 
-  defp resolve_signaling(%Membrane.WebRTC.SignalingChannel{} = signaling) do
+  defp resolve_signaling(%SignalingChannel{} = signaling, _utility_supervisor) do
     signaling
   end
 
-  defp resolve_signaling(uri) when is_binary(uri) do
+  defp resolve_signaling(uri, utility_supervisor) when is_binary(uri) do
     uri = URI.new!(uri)
     {:ok, ip} = :inet.getaddr(~c"#{uri.host}", :inet)
-    {:websocket, ip: ip, port: uri.port}
+    opts = [ip: ip, port: uri.port]
+
+    SimpleWebSocketServer.start_link_supervised(utility_supervisor, opts)
   end
 
   defp webrtc_input?(%{input: {:webrtc, _signalling}}), do: true

mix.exs

@@ -46,13 +46,13 @@ defmodule Boombox.Mixfile do
   defp deps do
     [
       {:membrane_core, "~> 1.1"},
-      {:membrane_webrtc_plugin, "~> 0.23.1"},
+      {:membrane_webrtc_plugin, "~> 0.23.2"},
       {:membrane_opus_plugin, "~> 0.20.3"},
       {:membrane_aac_plugin, "~> 0.19.0"},
       {:membrane_aac_fdk_plugin, "~> 0.18.0"},
-      {:membrane_vpx_plugin, "~> 0.2.0"},
+      {:membrane_vpx_plugin, "~> 0.3.0"},
       {:membrane_h26x_plugin, "~> 0.10.0"},
-      {:membrane_h264_ffmpeg_plugin, "~> 0.32.0"},
+      {:membrane_h264_ffmpeg_plugin, "~> 0.32.5"},
       {:membrane_h265_ffmpeg_plugin, "~> 0.4.2"},
       {:membrane_mp4_plugin, "~> 0.35.2"},
       {:membrane_realtimer_plugin, "~> 0.9.0"},
@@ -68,8 +68,9 @@ defmodule Boombox.Mixfile do
       {:membrane_ffmpeg_swresample_plugin, "~> 0.20.0"},
       {:membrane_hackney_plugin, "~> 0.11.0"},
       {:membrane_ffmpeg_swscale_plugin, "~> 0.16.2"},
-      {:membrane_simple_rtsp_server, "~> 0.1.3"},
       {:image, "~> 0.54.0"},
+      {:membrane_simple_rtsp_server, "~> 0.1.3", only: :test},
+      {:playwright, "~> 1.49.1-alpha.1", only: :test},
       {:burrito, "~> 1.0", runtime: burrito?(), optional: true},
       {:ex_doc, ">= 0.0.0", only: :dev, runtime: false},
       {:dialyxir, ">= 0.0.0", only: :dev, runtime: false},