diff --git a/lib/pinchflat/downloading/download_option_builder.ex b/lib/pinchflat/downloading/download_option_builder.ex index 333d1d5e..1b927b94 100644 --- a/lib/pinchflat/downloading/download_option_builder.ex +++ b/lib/pinchflat/downloading/download_option_builder.ex @@ -34,21 +34,38 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilder do @doc """ Builds the output path for yt-dlp to download media based on the given source's - media profile. Uses the source's override output path template if it exists. + or media_item's media profile. Uses the source's override output path template if it exists. Accepts a %MediaItem{} or %Source{} struct. If a %Source{} struct is passed, it will use a default %MediaItem{} struct with the given source. Returns binary() """ + def build_output_path_for(%Source{} = source_with_preloads) do + build_output_path_for(%MediaItem{source: source_with_preloads}) + end + def build_output_path_for(%MediaItem{} = media_item_with_preloads) do output_path_template = Sources.output_path_template(media_item_with_preloads.source) build_output_path(output_path_template, media_item_with_preloads) end - def build_output_path_for(%Source{} = source_with_preloads) do - build_output_path_for(%MediaItem{source: source_with_preloads}) + @doc """ + Builds the quality options for yt-dlp to download media based on the given source's + or media_item's media profile. Useful for helping predict final filepath of downloaded + media. + + returns [Keyword.t()] + """ + def build_quality_options_for(%Source{} = source_with_preloads) do + build_quality_options_for(%MediaItem{source: source_with_preloads}) + end + + def build_quality_options_for(%MediaItem{} = media_item_with_preloads) do + media_profile = media_item_with_preloads.source.media_profile + + quality_options(media_profile) end defp default_options(override_opts) do diff --git a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex index 67540e6d..9daeda1e 100644 --- a/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex +++ b/lib/pinchflat/fast_indexing/fast_indexing_helpers.ex @@ -15,6 +15,7 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do alias Pinchflat.FastIndexing.YoutubeRss alias Pinchflat.FastIndexing.YoutubeApi alias Pinchflat.Downloading.DownloadingHelpers + alias Pinchflat.Downloading.DownloadOptionBuilder alias Pinchflat.YtDlp.Media, as: YtDlpMedia @@ -27,6 +28,10 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do downloaded_. """ def kickoff_download_tasks_from_youtube_rss_feed(%Source{} = source) do + # The media_profile is needed to determine the quality options to _then_ determine a more + # accurate predicted filepath + source = Repo.preload(source, [:media_profile]) + {:ok, media_ids} = get_recent_media_ids(source) existing_media_items = list_media_items_by_media_id_for(source, media_ids) new_media_ids = media_ids -- Enum.map(existing_media_items, & &1.media_id) @@ -68,7 +73,11 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpers do defp create_media_item_from_media_id(source, media_id) do url = "https://www.youtube.com/watch?v=#{media_id}" - case YtDlpMedia.get_media_attributes(url, use_cookies: source.use_cookies) do + command_opts = + [output: DownloadOptionBuilder.build_output_path_for(source)] ++ + DownloadOptionBuilder.build_quality_options_for(source) + + case YtDlpMedia.get_media_attributes(url, command_opts, use_cookies: source.use_cookies) do {:ok, media_attrs} -> Media.create_media_item_from_backend_attrs(source, media_attrs) diff --git a/lib/pinchflat/media/media_item.ex b/lib/pinchflat/media/media_item.ex index f85fcfba..7a4fb328 100644 --- a/lib/pinchflat/media/media_item.ex +++ b/lib/pinchflat/media/media_item.ex @@ -31,6 +31,7 @@ defmodule Pinchflat.Media.MediaItem do :uploaded_at, :upload_date_index, :duration_seconds, + :predicted_media_filepath, # these fields are captured only on download :media_downloaded_at, :media_filepath, @@ -76,6 +77,7 @@ defmodule Pinchflat.Media.MediaItem do field :duration_seconds, :integer field :playlist_index, :integer, default: 0 + field :predicted_media_filepath, :string field :media_filepath, :string field :media_size_bytes, :integer field :thumbnail_filepath, :string diff --git a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex index 7b82a1fc..74e5f4c3 100644 --- a/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex +++ b/lib/pinchflat/slow_indexing/slow_indexing_helpers.ex @@ -16,6 +16,7 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do alias Pinchflat.YtDlp.MediaCollection alias Pinchflat.Downloading.DownloadingHelpers alias Pinchflat.SlowIndexing.FileFollowerServer + alias Pinchflat.Downloading.DownloadOptionBuilder alias Pinchflat.SlowIndexing.MediaCollectionIndexingWorker alias Pinchflat.YtDlp.Media, as: YtDlpMedia @@ -56,6 +57,9 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do Returns [%MediaItem{} | %Ecto.Changeset{}] """ def index_and_enqueue_download_for_media_items(%Source{} = source) do + # The media_profile is needed to determine the quality options to _then_ determine a more + # accurate predicted filepath + source = Repo.preload(source, [:media_profile]) # See the method definition below for more info on how file watchers work # (important reading if you're not familiar with it) {:ok, media_attributes} = setup_file_watcher_and_kickoff_indexing(source) @@ -94,8 +98,13 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpers do {:ok, pid} = FileFollowerServer.start_link() handler = fn filepath -> setup_file_follower_watcher(pid, filepath, source) end + + command_opts = + [output: DownloadOptionBuilder.build_output_path_for(source)] ++ + DownloadOptionBuilder.build_quality_options_for(source) + runner_opts = [file_listener_handler: handler, use_cookies: source.use_cookies] - result = MediaCollection.get_media_attributes_for_collection(source.original_url, runner_opts) + result = MediaCollection.get_media_attributes_for_collection(source.original_url, command_opts, runner_opts) FileFollowerServer.stop(pid) diff --git a/lib/pinchflat/yt_dlp/media.ex b/lib/pinchflat/yt_dlp/media.ex index fda79843..cdd8dea2 100644 --- a/lib/pinchflat/yt_dlp/media.ex +++ b/lib/pinchflat/yt_dlp/media.ex @@ -11,7 +11,8 @@ defmodule Pinchflat.YtDlp.Media do :livestream, :short_form_content, :uploaded_at, - :duration_seconds + :duration_seconds, + :predicted_media_filepath ] defstruct [ @@ -23,7 +24,8 @@ defmodule Pinchflat.YtDlp.Media do :short_form_content, :uploaded_at, :duration_seconds, - :playlist_index + :playlist_index, + :predicted_media_filepath ] alias __MODULE__ @@ -63,15 +65,17 @@ defmodule Pinchflat.YtDlp.Media do @doc """ Returns a map representing the media at the given URL. + Optionally takes a list of additional command options to pass to yt-dlp + or configuration-related options to pass to the runner. Returns {:ok, %Media{}} | {:error, any, ...}. """ - def get_media_attributes(url, addl_opts \\ []) do + def get_media_attributes(url, command_opts \\ [], addl_opts \\ []) do runner = Application.get_env(:pinchflat, :yt_dlp_runner) - command_opts = [:simulate, :skip_download] + all_command_opts = [:simulate, :skip_download] ++ command_opts output_template = indexing_output_template() - case runner.run(url, command_opts, output_template, addl_opts) do + case runner.run(url, all_command_opts, output_template, addl_opts) do {:ok, output} -> output |> Phoenix.json_library().decode!() @@ -91,7 +95,7 @@ defmodule Pinchflat.YtDlp.Media do if something is a short via the URL again """ def indexing_output_template do - "%(.{id,title,live_status,original_url,description,aspect_ratio,duration,upload_date,timestamp,playlist_index})j" + "%(.{id,title,live_status,original_url,description,aspect_ratio,duration,upload_date,timestamp,playlist_index,filename})j" end @doc """ @@ -110,7 +114,8 @@ defmodule Pinchflat.YtDlp.Media do duration_seconds: response["duration"] && round(response["duration"]), short_form_content: response["original_url"] && short_form_content?(response), uploaded_at: response["upload_date"] && parse_uploaded_at(response), - playlist_index: response["playlist_index"] || 0 + playlist_index: response["playlist_index"] || 0, + predicted_media_filepath: response["filename"] } end diff --git a/lib/pinchflat/yt_dlp/media_collection.ex b/lib/pinchflat/yt_dlp/media_collection.ex index 1b0035ab..fe920722 100644 --- a/lib/pinchflat/yt_dlp/media_collection.ex +++ b/lib/pinchflat/yt_dlp/media_collection.ex @@ -11,20 +11,23 @@ defmodule Pinchflat.YtDlp.MediaCollection do @doc """ Returns a list of maps representing the media in the collection. + Optionally takes a list of additional command options to pass to yt-dlp + or configuration-related options to pass to the runner. - Options: + Runner Options: - :file_listener_handler - a function that will be called with the path to the file that will be written to when yt-dlp is done. This is useful for setting up a file watcher to know when the file is ready to be read. + - :use_cookies - whether or not to use user-provided cookies when fetching the media details Returns {:ok, [map()]} | {:error, any, ...}. """ - def get_media_attributes_for_collection(url, addl_opts \\ []) do + def get_media_attributes_for_collection(url, command_opts \\ [], addl_opts \\ []) do runner = Application.get_env(:pinchflat, :yt_dlp_runner) # `ignore_no_formats_error` is necessary because yt-dlp will error out if # the first video has not released yet (ie: is a premier). We don't care about # available formats since we're just getting the media details - command_opts = [:simulate, :skip_download, :ignore_no_formats_error, :no_warnings] + all_command_opts = [:simulate, :skip_download, :ignore_no_formats_error, :no_warnings] ++ command_opts use_cookies = Keyword.get(addl_opts, :use_cookies, false) output_template = YtDlpMedia.indexing_output_template() output_filepath = FilesystemUtils.generate_metadata_tmpfile(:json) @@ -35,7 +38,7 @@ defmodule Pinchflat.YtDlp.MediaCollection do file_listener_handler.(output_filepath) end - case runner.run(url, command_opts, output_template, runner_opts) do + case runner.run(url, all_command_opts, output_template, runner_opts) do {:ok, output} -> parsed_lines = output diff --git a/priv/repo/erd.png b/priv/repo/erd.png index 06525bc0..c6566d80 100644 Binary files a/priv/repo/erd.png and b/priv/repo/erd.png differ diff --git a/priv/repo/migrations/20241107201850_add_predicted_media_filepath_to_media_items.exs b/priv/repo/migrations/20241107201850_add_predicted_media_filepath_to_media_items.exs new file mode 100644 index 00000000..630b8562 --- /dev/null +++ b/priv/repo/migrations/20241107201850_add_predicted_media_filepath_to_media_items.exs @@ -0,0 +1,9 @@ +defmodule Pinchflat.Repo.Migrations.AddPredictedMediaFilepathToMediaItems do + use Ecto.Migration + + def change do + alter table(:media_items) do + add :predicted_media_filepath, :string + end + end +end diff --git a/test/pinchflat/downloading/download_option_builder_test.exs b/test/pinchflat/downloading/download_option_builder_test.exs index 98aff6f5..97ad1181 100644 --- a/test/pinchflat/downloading/download_option_builder_test.exs +++ b/test/pinchflat/downloading/download_option_builder_test.exs @@ -461,6 +461,22 @@ defmodule Pinchflat.Downloading.DownloadOptionBuilderTest do end end + describe "build_quality_options_for/1" do + test "builds quality options for a media item", %{media_item: media_item} do + options = DownloadOptionBuilder.build_quality_options_for(media_item) + + assert {:format_sort, "res:1080,+codec:avc:m4a"} in options + assert {:remux_video, "mp4"} in options + end + + test "builds quality options for a source", %{media_item: media_item} do + options = DownloadOptionBuilder.build_quality_options_for(media_item.source) + + assert {:format_sort, "res:1080,+codec:avc:m4a"} in options + assert {:remux_video, "mp4"} in options + end + end + defp update_media_profile_attribute(media_item_with_preloads, attrs) do media_item_with_preloads.source.media_profile |> Profiles.change_media_profile(attrs) diff --git a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs index d2620f6c..54220183 100644 --- a/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs +++ b/test/pinchflat/fast_indexing/fast_indexing_helpers_test.exs @@ -61,6 +61,18 @@ defmodule Pinchflat.FastIndexing.FastIndexingHelpersTest do assert [_] = Tasks.list_tasks_for(media_item, "MediaDownloadWorker") end + test "passes the source's download options to the yt-dlp runner", %{source: source} do + expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) + + expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl_opts -> + assert {:output, "/tmp/test/media/%(title)S.%(ext)S"} in opts + assert {:remux_video, "mp4"} in opts + {:ok, media_attributes_return_fixture()} + end) + + FastIndexingHelpers.kickoff_download_tasks_from_youtube_rss_feed(source) + end + test "sets use_cookies if the source uses cookies" do expect(HTTPClientMock, :get, fn _url -> {:ok, "test_1"} end) diff --git a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs index a4650e8a..36abcd71 100644 --- a/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs +++ b/test/pinchflat/slow_indexing/slow_indexing_helpers_test.exs @@ -202,6 +202,16 @@ defmodule Pinchflat.SlowIndexing.SlowIndexingHelpersTest do assert %Ecto.Changeset{} = changeset end + test "passes the source's download options to the yt-dlp runner", %{source: source} do + expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl_opts -> + assert {:output, "/tmp/test/media/%(title)S.%(ext)S"} in opts + assert {:remux_video, "mp4"} in opts + {:ok, source_attributes_return_fixture()} + end) + + SlowIndexingHelpers.index_and_enqueue_download_for_media_items(source) + end + test "sets use_cookies if the source uses cookies" do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts -> assert {:use_cookies, true} in addl_opts diff --git a/test/pinchflat/yt_dlp/media_collection_test.exs b/test/pinchflat/yt_dlp/media_collection_test.exs index 5d14b7cf..6f112b3e 100644 --- a/test/pinchflat/yt_dlp/media_collection_test.exs +++ b/test/pinchflat/yt_dlp/media_collection_test.exs @@ -35,6 +35,16 @@ defmodule Pinchflat.YtDlp.MediaCollectionTest do assert {:error, "Big issue", 1} = MediaCollection.get_media_attributes_for_collection(@channel_url) end + test "passes long additional command options" do + expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl_opts -> + assert :foo in opts + + {:ok, ""} + end) + + assert {:ok, _} = MediaCollection.get_media_attributes_for_collection(@channel_url, [:foo]) + end + test "passes additional args to runner" do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl_opts -> assert [{:output_filepath, filepath} | _] = addl_opts @@ -56,7 +66,7 @@ defmodule Pinchflat.YtDlp.MediaCollectionTest do end assert {:ok, _} = - MediaCollection.get_media_attributes_for_collection(@channel_url, file_listener_handler: handler) + MediaCollection.get_media_attributes_for_collection(@channel_url, [], file_listener_handler: handler) assert_receive {:handler, filename} assert String.ends_with?(filename, ".json") diff --git a/test/pinchflat/yt_dlp/media_test.exs b/test/pinchflat/yt_dlp/media_test.exs index 03668273..8a3d9ae2 100644 --- a/test/pinchflat/yt_dlp/media_test.exs +++ b/test/pinchflat/yt_dlp/media_test.exs @@ -120,13 +120,22 @@ defmodule Pinchflat.YtDlp.MediaTest do assert {:ok, _} = Media.get_media_attributes(@media_url) end + test "passes along additional command options" do + expect(YtDlpRunnerMock, :run, fn _url, opts, _ot, _addl -> + assert [:simulate, :skip_download, :custom_arg] = opts + {:ok, media_attributes_return_fixture()} + end) + + assert {:ok, _} = Media.get_media_attributes(@media_url, [:custom_arg]) + end + test "passes along additional options" do expect(YtDlpRunnerMock, :run, fn _url, _opts, _ot, addl -> assert [addl_arg: true] = addl {:ok, media_attributes_return_fixture()} end) - assert {:ok, _} = Media.get_media_attributes(@media_url, addl_arg: true) + assert {:ok, _} = Media.get_media_attributes(@media_url, [], addl_arg: true) end test "returns the error straight through when the command fails" do @@ -139,7 +148,7 @@ defmodule Pinchflat.YtDlp.MediaTest do describe "indexing_output_template/0" do test "contains all the greatest hits" do attrs = - ~w(id title live_status original_url description aspect_ratio duration upload_date timestamp playlist_index)a + ~w(id title live_status original_url description aspect_ratio duration upload_date timestamp playlist_index filename)a formatted_attrs = "%(.{#{Enum.join(attrs, ",")}})j" @@ -159,7 +168,8 @@ defmodule Pinchflat.YtDlp.MediaTest do "duration" => 60, "upload_date" => "20210101", "timestamp" => 1_600_000_000, - "playlist_index" => 1 + "playlist_index" => 1, + "filename" => "TiZPUDkDYbk.mp4" } assert %Media{ @@ -171,7 +181,8 @@ defmodule Pinchflat.YtDlp.MediaTest do short_form_content: false, uploaded_at: ~U[2020-09-13 12:26:40Z], duration_seconds: 60, - playlist_index: 1 + playlist_index: 1, + predicted_media_filepath: "TiZPUDkDYbk.mp4" } == Media.response_to_struct(response) end