From 6764b0e2e256fd79cd718ec6d90fbcccedecea9c Mon Sep 17 00:00:00 2001 From: Mishig Davaadorj Date: Wed, 22 Nov 2023 10:49:19 +0100 Subject: [PATCH 1/4] [Task & Icon] for Image-to-Video --- packages/hub/src/types/public.d.ts | 1 + packages/tasks/src/const.ts | 1 + packages/tasks/src/pipelines.ts | 5 +++++ packages/tasks/src/tasksData.ts | 1 + .../widgets/src/lib/components/Icons/IconImageToVideo.svelte | 5 +++++ .../src/lib/components/PipelineIcon/PipelineIcon.svelte | 2 ++ 6 files changed, 15 insertions(+) create mode 100644 packages/widgets/src/lib/components/Icons/IconImageToVideo.svelte diff --git a/packages/hub/src/types/public.d.ts b/packages/hub/src/types/public.d.ts index 6b2df94e2..e9a7fed93 100644 --- a/packages/hub/src/types/public.d.ts +++ b/packages/hub/src/types/public.d.ts @@ -69,6 +69,7 @@ export type Task = | "text-to-image" | "image-to-text" | "image-to-image" + | "image-to-video" | "unconditional-image-generation" | "video-classification" | "reinforcement-learning" diff --git a/packages/tasks/src/const.ts b/packages/tasks/src/const.ts index 34fb9b24a..d7fde6c72 100644 --- a/packages/tasks/src/const.ts +++ b/packages/tasks/src/const.ts @@ -18,6 +18,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "image-segmentation": ["transformers", "transformers.js"], "image-to-image": [], "image-to-text": ["transformers.js"], + "image-to-video": [], // TODO: diffusers will soon support image-to-video "video-classification": [], "multiple-choice": ["transformers"], "object-detection": ["transformers", "transformers.js"], diff --git a/packages/tasks/src/pipelines.ts b/packages/tasks/src/pipelines.ts index ae487d5e3..cd483757d 100644 --- a/packages/tasks/src/pipelines.ts +++ b/packages/tasks/src/pipelines.ts @@ -438,6 +438,11 @@ export const PIPELINE_DATA = { modality: "cv", color: "indigo", }, + "image-to-video": { + name: "Image-to-Video", + modality: "multimodal", + color: "indigo", + }, "unconditional-image-generation": { name: "Unconditional Image Generation", modality: "cv", diff --git a/packages/tasks/src/tasksData.ts b/packages/tasks/src/tasksData.ts index db2609971..2da36cefa 100644 --- a/packages/tasks/src/tasksData.ts +++ b/packages/tasks/src/tasksData.ts @@ -54,6 +54,7 @@ export const TASKS_DATA: Record = { "image-segmentation": getData("image-segmentation", imageSegmentation), "image-to-image": getData("image-to-image", imageToImage), "image-to-text": getData("image-to-text", imageToText), + "image-to-video": undefined, "multiple-choice": undefined, "object-detection": getData("object-detection", objectDetection), "video-classification": getData("video-classification", videoClassification), diff --git a/packages/widgets/src/lib/components/Icons/IconImageToVideo.svelte b/packages/widgets/src/lib/components/Icons/IconImageToVideo.svelte new file mode 100644 index 000000000..3ae57df09 --- /dev/null +++ b/packages/widgets/src/lib/components/Icons/IconImageToVideo.svelte @@ -0,0 +1,5 @@ + + + diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte index 990019763..199dacbb4 100644 --- a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte +++ b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte @@ -23,6 +23,7 @@ import IconTextGeneration from "../Icons/IconTextGeneration.svelte"; import IconTextToImage from "../Icons/IconTextToImage.svelte"; import IconImageToText from "../Icons/IconImageToText.svelte"; + import IconImageToVideo from "../Icons/IconImageToVideo.svelte"; import IconTextToSpeech from "../Icons/IconTextToSpeech.svelte"; import IconTextToVideo from "../Icons/IconTextToVideo.svelte"; import IconTokenClassification from "../Icons/IconTokenClassification.svelte"; @@ -72,6 +73,7 @@ "text-to-image": IconTextToImage, "image-to-text": IconImageToText, "image-to-image": IconImageToImage, + "image-to-video": IconImageToVideo, "unconditional-image-generation": IconUnconditionalImageGeneration, "reinforcement-learning": IconReinforcementLearning, robotics: IconRobotics, From e1d2b92c9207e7162f423169b4af437966a80ae5 Mon Sep 17 00:00:00 2001 From: Mishig Davaadorj Date: Wed, 22 Nov 2023 12:18:08 +0100 Subject: [PATCH 2/4] Add diffusers for all tasks --- packages/tasks/src/const.ts | 6 +++--- packages/widgets/src/lib/interfaces/LibrariesToTasks.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tasks/src/const.ts b/packages/tasks/src/const.ts index d7fde6c72..3167125bb 100644 --- a/packages/tasks/src/const.ts +++ b/packages/tasks/src/const.ts @@ -16,9 +16,9 @@ export const TASKS_MODEL_LIBRARIES: Record = { "graph-ml": ["transformers"], "image-classification": ["keras", "timm", "transformers", "transformers.js"], "image-segmentation": ["transformers", "transformers.js"], - "image-to-image": [], + "image-to-image": ["diffusers"], "image-to-text": ["transformers.js"], - "image-to-video": [], // TODO: diffusers will soon support image-to-video + "image-to-video": ["diffusers"], "video-classification": [], "multiple-choice": ["transformers"], "object-detection": ["transformers", "transformers.js"], @@ -36,7 +36,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "text-classification": ["adapter-transformers", "spacy", "transformers", "transformers.js"], "text-generation": ["transformers", "transformers.js"], "text-retrieval": [], - "text-to-image": [], + "text-to-image": ["diffusers"], "text-to-speech": ["espnet", "tensorflowtts", "transformers"], "text-to-audio": ["transformers"], "text-to-video": [], diff --git a/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts b/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts index 780323f9f..0e7887110 100644 --- a/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts +++ b/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts @@ -15,7 +15,7 @@ export const LIBRARY_TASK_MAPPING_EXCLUDING_TRANSFORMERS: Partial Date: Wed, 22 Nov 2023 13:38:00 +0100 Subject: [PATCH 3/4] Revert "Add diffusers for all tasks" This reverts commit e1d2b92c9207e7162f423169b4af437966a80ae5. --- packages/tasks/src/const.ts | 6 +++--- packages/widgets/src/lib/interfaces/LibrariesToTasks.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tasks/src/const.ts b/packages/tasks/src/const.ts index 3167125bb..d7fde6c72 100644 --- a/packages/tasks/src/const.ts +++ b/packages/tasks/src/const.ts @@ -16,9 +16,9 @@ export const TASKS_MODEL_LIBRARIES: Record = { "graph-ml": ["transformers"], "image-classification": ["keras", "timm", "transformers", "transformers.js"], "image-segmentation": ["transformers", "transformers.js"], - "image-to-image": ["diffusers"], + "image-to-image": [], "image-to-text": ["transformers.js"], - "image-to-video": ["diffusers"], + "image-to-video": [], // TODO: diffusers will soon support image-to-video "video-classification": [], "multiple-choice": ["transformers"], "object-detection": ["transformers", "transformers.js"], @@ -36,7 +36,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "text-classification": ["adapter-transformers", "spacy", "transformers", "transformers.js"], "text-generation": ["transformers", "transformers.js"], "text-retrieval": [], - "text-to-image": ["diffusers"], + "text-to-image": [], "text-to-speech": ["espnet", "tensorflowtts", "transformers"], "text-to-audio": ["transformers"], "text-to-video": [], diff --git a/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts b/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts index 0e7887110..780323f9f 100644 --- a/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts +++ b/packages/widgets/src/lib/interfaces/LibrariesToTasks.ts @@ -15,7 +15,7 @@ export const LIBRARY_TASK_MAPPING_EXCLUDING_TRANSFORMERS: Partial Date: Wed, 22 Nov 2023 13:38:44 +0100 Subject: [PATCH 4/4] "image-to-video": ["diffusers"], only --- packages/tasks/src/const.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/const.ts b/packages/tasks/src/const.ts index d7fde6c72..025b582ba 100644 --- a/packages/tasks/src/const.ts +++ b/packages/tasks/src/const.ts @@ -18,7 +18,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "image-segmentation": ["transformers", "transformers.js"], "image-to-image": [], "image-to-text": ["transformers.js"], - "image-to-video": [], // TODO: diffusers will soon support image-to-video + "image-to-video": ["diffusers"], "video-classification": [], "multiple-choice": ["transformers"], "object-detection": ["transformers", "transformers.js"],