From c05f9a0d9a8d74875e53568b725db0a391a989c5 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Mon, 4 Nov 2024 10:26:31 +0100 Subject: [PATCH 01/13] Add a separate ov nightly GHA workflow Update workflow name Trigger Tests Fix workflow file Fix Introduce more readable job names Revert basic test changes --- .github/workflows/test_openvino_full.yml | 83 ++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .github/workflows/test_openvino_full.yml diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml new file mode 100644 index 0000000000..728b5aade7 --- /dev/null +++ b/.github/workflows/test_openvino_full.yml @@ -0,0 +1,83 @@ +name: OpenVINO - Full Test + +on: + workflow_dispatch: + schedule: + - cron: "41 3 * * *" # run every day at 3:41 + push: + branches: + - v*-release + pull_request: + types: [opened, synchronize, reopened, labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + if: ${{ (github.event_name == 'workflow_dispatch') || (github.event_name == 'schedule') || (github.event_name == 'push') || contains( github.event.pull_request.labels.*.name, 'openvino-test') }} + strategy: + fail-fast: false + matrix: + include: + - python-version: "3.12" + os: "ubuntu-22.04" + transformers-version: "latest" + openvino: "ov-release" + nncf: "nncf-release" + - python-version: "3.12" + os: "ubuntu-22.04" + transformers-version: "latest" + openvino: "ov-nightly" + nncf: "nncf-release" + - python-version: "3.12" + os: "ubuntu-22.04" + transformers-version: "latest" + openvino: "ov-nightly" + nncf: "nncf-develop" + + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install lowest compatible transformers version + if: ${{ matrix.transformers-version != 'latest' }} + run: pip install transformers==${{ matrix.transformers-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install PyTorch CPU to prevent unnecessary downloading/installing of CUDA packages + pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu + pip install .[tests] + + - name: Install openvino-nightly + if: ${{ matrix.openvino == 'ov-nightly' }} + run: pip install --pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + + - name: Install openvino release + if: ${{ matrix.openvino != 'ov-nightly' }} + run: pip install .[openvino] + + - name: Install nncf develop + if: ${{ matrix.nncf == 'nncf-develop' }} + run: pip install git+https://github.com/openvinotoolkit/nncf.git + + - name: Install nncf release + if: ${{ matrix.nncf != 'nncf-develop' }} + run: pip install .[nncf] + + - name: Pip freeze + run: pip freeze + + - name: OpenVINO tests + run: pytest tests/openvino -s --durations=0 + env: + RUN_SLOW: 1 + HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} From 0b706516be0291681807c9de5449346a7e6075c9 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 6 Nov 2024 10:20:15 +0100 Subject: [PATCH 02/13] Remove ov-nightly and nncf-develop from slow tests --- .github/workflows/test_openvino_full.yml | 10 +++++----- .github/workflows/test_openvino_slow.yml | 9 +-------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index 728b5aade7..efffde513a 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -24,13 +24,13 @@ jobs: - python-version: "3.12" os: "ubuntu-22.04" transformers-version: "latest" - openvino: "ov-release" - nncf: "nncf-release" + openvino: "ov-stable" + nncf: "nncf-stable" - python-version: "3.12" os: "ubuntu-22.04" transformers-version: "latest" openvino: "ov-nightly" - nncf: "nncf-release" + nncf: "nncf-stable" - python-version: "3.12" os: "ubuntu-22.04" transformers-version: "latest" @@ -62,7 +62,7 @@ jobs: run: pip install --pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - name: Install openvino release - if: ${{ matrix.openvino != 'ov-nightly' }} + if: ${{ matrix.openvino == 'ov-stable' }} run: pip install .[openvino] - name: Install nncf develop @@ -70,7 +70,7 @@ jobs: run: pip install git+https://github.com/openvinotoolkit/nncf.git - name: Install nncf release - if: ${{ matrix.nncf != 'nncf-develop' }} + if: ${{ matrix.nncf == 'nncf-stable' }} run: pip install .[nncf] - name: Pip freeze diff --git a/.github/workflows/test_openvino_slow.yml b/.github/workflows/test_openvino_slow.yml index bf52413a7d..42aa11cdaf 100644 --- a/.github/workflows/test_openvino_slow.yml +++ b/.github/workflows/test_openvino_slow.yml @@ -25,9 +25,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-22.04", "windows-2019"] - openvino-version: ["stable", "nightly"] transformers-version: ["4.36.0", "latest"] - nncf: ["nncf", "git+https://github.com/openvinotoolkit/nncf.git"] runs-on: ${{ matrix.os }} @@ -47,11 +45,6 @@ jobs: pip install .[openvino,tests] transformers[testing] pip uninstall -y nncf - - if: ${{ matrix.openvino-version == 'nightly' }} - name: Install nightly OpenVINO - run: | - pip install openvino openvino-tokenizers --pre --upgrade --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly - - if: ${{ matrix.transformers-version != 'latest' }} name: Downgrade Transformers and Accelerate run: pip install transformers==${{ matrix.transformers-version }} accelerate==0.* @@ -65,7 +58,7 @@ jobs: - name: Install dependencies (slow) run: | - pip install ${{ matrix.nncf }} + pip install .[nncf] - name: Test with Pytest (slow) run: | From 0eb3b4cdeffc41fa7e227b5c87e08406329f9caa Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 6 Nov 2024 10:22:59 +0100 Subject: [PATCH 03/13] Use Python 3.9 --- .github/workflows/test_openvino_full.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index efffde513a..c0a268d97d 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -21,17 +21,17 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.12" + - python-version: "3.9" os: "ubuntu-22.04" transformers-version: "latest" openvino: "ov-stable" nncf: "nncf-stable" - - python-version: "3.12" + - python-version: "3.9" os: "ubuntu-22.04" transformers-version: "latest" openvino: "ov-nightly" nncf: "nncf-stable" - - python-version: "3.12" + - python-version: "3.9" os: "ubuntu-22.04" transformers-version: "latest" openvino: "ov-nightly" From 9c05797e92d78c18208feb49fb082dfd3ae829d7 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Wed, 6 Nov 2024 10:24:28 +0100 Subject: [PATCH 04/13] Add a combination for ov-stable and nncf-develop --- .github/workflows/test_openvino_full.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index c0a268d97d..0a884015da 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -31,6 +31,11 @@ jobs: transformers-version: "latest" openvino: "ov-nightly" nncf: "nncf-stable" + - python-version: "3.9" + os: "ubuntu-22.04" + transformers-version: "latest" + openvino: "ov-stable" + nncf: "nncf-develop" - python-version: "3.9" os: "ubuntu-22.04" transformers-version: "latest" From 3e8ce546597d9ec51e565669730de513d515f3c9 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Sun, 24 Nov 2024 15:50:48 +0100 Subject: [PATCH 05/13] Remove large model after testing --- tests/openvino/test_diffusion.py | 65 ++++++++++++++++++-------------- tests/openvino/utils_tests.py | 13 +++++++ 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/tests/openvino/test_diffusion.py b/tests/openvino/test_diffusion.py index 1467e5ed1f..4ebea5ed0e 100644 --- a/tests/openvino/test_diffusion.py +++ b/tests/openvino/test_diffusion.py @@ -27,7 +27,7 @@ from diffusers.utils import load_image from parameterized import parameterized from transformers.testing_utils import slow -from utils_tests import MODEL_NAMES, SEED +from utils_tests import MODEL_NAMES, SEED, remove_model_from_cache from optimum.intel.openvino import ( OVDiffusionPipeline, @@ -352,19 +352,22 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + try: + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + finally: + remove_model_from_cache(model_id) class OVPipelineForImage2ImageTest(unittest.TestCase): @@ -577,19 +580,22 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + try: + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + finally: + remove_model_from_cache(model_id) class OVPipelineForInpaintingTest(unittest.TestCase): @@ -810,16 +816,19 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + try: + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + finally: + remove_model_from_cache(model_id) diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 2d2d713305..97e7b2fc8a 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os +import shutil +from pathlib import Path import numpy as np import torch @@ -203,3 +206,13 @@ def get_num_quantized_nodes(ov_model): if type_name == "f8e8m0": num_weight_nodes["f8e8m0"] += 1 return num_fake_quantize, num_weight_nodes + + +def remove_model_from_cache(model_id: str): + hf_home = os.environ.get("HF_HOME", "~/.cache/huggingface") + hf_hub_cache = os.environ.get("HF_HUB_CACHE", os.path.join(hf_home, "hub")) + hf_hub_cache = Path(hf_hub_cache).expanduser() + model_id_namespace, model_id_label = model_id.split("/") + model_path = hf_hub_cache / f"models--{model_id_namespace}--{model_id_label}" + if model_path.exists(): + shutil.rmtree(model_path) From c1d46da675853e38380bdad77b0282b0a9bdfa40 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Mon, 25 Nov 2024 11:54:28 +0100 Subject: [PATCH 06/13] Revert "Remove large model after testing" This reverts commit 3e8ce546597d9ec51e565669730de513d515f3c9. --- tests/openvino/test_diffusion.py | 65 ++++++++++++++------------------ tests/openvino/utils_tests.py | 13 ------- 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/tests/openvino/test_diffusion.py b/tests/openvino/test_diffusion.py index f961971ea0..2baeba9a42 100644 --- a/tests/openvino/test_diffusion.py +++ b/tests/openvino/test_diffusion.py @@ -28,7 +28,7 @@ from diffusers.utils import load_image from parameterized import parameterized from transformers.testing_utils import slow -from utils_tests import MODEL_NAMES, SEED, remove_model_from_cache +from utils_tests import MODEL_NAMES, SEED from optimum.intel.openvino import ( OVDiffusionPipeline, @@ -379,22 +379,19 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - try: - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) - finally: - remove_model_from_cache(model_id) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForImage2ImageTest(unittest.TestCase): @@ -607,22 +604,19 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - try: - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) - finally: - remove_model_from_cache(model_id) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForInpaintingTest(unittest.TestCase): @@ -843,19 +837,16 @@ def test_textual_inversion(self): model_id = "runwayml/stable-diffusion-v1-5" ti_id = "sd-concepts-library/cat-toy" - try: - inputs = self.generate_inputs() - inputs["prompt"] = "A backpack" + inputs = self.generate_inputs() + inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) - finally: - remove_model_from_cache(model_id) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) diff --git a/tests/openvino/utils_tests.py b/tests/openvino/utils_tests.py index 272c0aec2a..b646b5b52a 100644 --- a/tests/openvino/utils_tests.py +++ b/tests/openvino/utils_tests.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import shutil -from pathlib import Path import numpy as np import openvino as ov @@ -220,13 +217,3 @@ def get_num_quantized_nodes(model): if type_name == "nf4": num_weight_nodes["nf4"] += 1 return num_fake_quantize, num_weight_nodes - - -def remove_model_from_cache(model_id: str): - hf_home = os.environ.get("HF_HOME", "~/.cache/huggingface") - hf_hub_cache = os.environ.get("HF_HUB_CACHE", os.path.join(hf_home, "hub")) - hf_hub_cache = Path(hf_hub_cache).expanduser() - model_id_namespace, model_id_label = model_id.split("/") - model_path = hf_hub_cache / f"models--{model_id_namespace}--{model_id_label}" - if model_path.exists(): - shutil.rmtree(model_path) From 3fe8ed2c7a3ba6725758a5d9834b58a92fa28099 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Mon, 25 Nov 2024 14:11:32 +0100 Subject: [PATCH 07/13] Temporary cache dir --- tests/openvino/test_diffusion.py | 57 ++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/tests/openvino/test_diffusion.py b/tests/openvino/test_diffusion.py index 5f151cb839..c2e6212f92 100644 --- a/tests/openvino/test_diffusion.py +++ b/tests/openvino/test_diffusion.py @@ -392,16 +392,21 @@ def test_textual_inversion(self): inputs = self.generate_inputs() inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + with TemporaryDirectory() as temp_dir: + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( + model_id, safety_checker=None, cache_dir=temp_dir + ) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained( + model_id, compile=False, safety_checker=None, cache_dir=temp_dir + ) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForImage2ImageTest(unittest.TestCase): @@ -636,16 +641,21 @@ def test_textual_inversion(self): inputs = self.generate_inputs(model_type="stable-diffusion") inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + with TemporaryDirectory() as temp_dir: + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( + model_id, safety_checker=None, cache_dir=temp_dir + ) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained( + model_id, compile=False, safety_checker=None, cache_dir=temp_dir + ) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForInpaintingTest(unittest.TestCase): @@ -880,13 +890,18 @@ def test_textual_inversion(self): inputs = self.generate_inputs() inputs["prompt"] = "A backpack" - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) - diffusers_pipeline.load_textual_inversion(ti_id) + with TemporaryDirectory() as temp_dir: + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( + model_id, safety_checker=None, cache_dir=temp_dir + ) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained( + model_id, compile=False, safety_checker=None, cache_dir=temp_dir + ) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) From 0ab79e0eff9fd2cbe7f23d12f33a3d59168bfab7 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Tue, 26 Nov 2024 13:59:42 +0100 Subject: [PATCH 08/13] Revert "Temporary cache dir" This reverts commit 3fe8ed2c7a3ba6725758a5d9834b58a92fa28099. --- tests/openvino/test_diffusion.py | 57 ++++++++++++-------------------- 1 file changed, 21 insertions(+), 36 deletions(-) diff --git a/tests/openvino/test_diffusion.py b/tests/openvino/test_diffusion.py index c2e6212f92..5f151cb839 100644 --- a/tests/openvino/test_diffusion.py +++ b/tests/openvino/test_diffusion.py @@ -392,21 +392,16 @@ def test_textual_inversion(self): inputs = self.generate_inputs() inputs["prompt"] = "A backpack" - with TemporaryDirectory() as temp_dir: - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( - model_id, safety_checker=None, cache_dir=temp_dir - ) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained( - model_id, compile=False, safety_checker=None, cache_dir=temp_dir - ) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForImage2ImageTest(unittest.TestCase): @@ -641,21 +636,16 @@ def test_textual_inversion(self): inputs = self.generate_inputs(model_type="stable-diffusion") inputs["prompt"] = "A backpack" - with TemporaryDirectory() as temp_dir: - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( - model_id, safety_checker=None, cache_dir=temp_dir - ) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained( - model_id, compile=False, safety_checker=None, cache_dir=temp_dir - ) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) class OVPipelineForInpaintingTest(unittest.TestCase): @@ -890,18 +880,13 @@ def test_textual_inversion(self): inputs = self.generate_inputs() inputs["prompt"] = "A backpack" - with TemporaryDirectory() as temp_dir: - diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained( - model_id, safety_checker=None, cache_dir=temp_dir - ) - diffusers_pipeline.load_textual_inversion(ti_id) + diffusers_pipeline = self.AUTOMODEL_CLASS.from_pretrained(model_id, safety_checker=None) + diffusers_pipeline.load_textual_inversion(ti_id) - ov_pipeline = self.OVMODEL_CLASS.from_pretrained( - model_id, compile=False, safety_checker=None, cache_dir=temp_dir - ) - ov_pipeline.load_textual_inversion(ti_id) + ov_pipeline = self.OVMODEL_CLASS.from_pretrained(model_id, compile=False, safety_checker=None) + ov_pipeline.load_textual_inversion(ti_id) - diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images - ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images + diffusers_output = diffusers_pipeline(**inputs, generator=get_generator("pt", SEED)).images + ov_output = ov_pipeline(**inputs, generator=get_generator("pt", SEED)).images - np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) + np.testing.assert_allclose(ov_output, diffusers_output, atol=1e-4, rtol=1e-2) From 50058549f596fdbdc6f4f08f7fd7b02ee2c575eb Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Tue, 26 Nov 2024 14:51:38 +0100 Subject: [PATCH 09/13] Lower verbosity --- .github/workflows/test_openvino_full.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index 0a884015da..43cc5f1e8d 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -82,7 +82,7 @@ jobs: run: pip freeze - name: OpenVINO tests - run: pytest tests/openvino -s --durations=0 + run: pytest tests/openvino --durations=0 env: RUN_SLOW: 1 HF_HUB_READ_TOKEN: ${{ secrets.HF_HUB_READ_TOKEN }} From bdc1102f2be9b2853cc30d5ac06fc12ab5116e32 Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Fri, 29 Nov 2024 19:03:10 +0100 Subject: [PATCH 10/13] Reorder install commands --- .github/workflows/test_openvino_full.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index 43cc5f1e8d..aace2aca0a 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -51,10 +51,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install lowest compatible transformers version - if: ${{ matrix.transformers-version != 'latest' }} - run: pip install transformers==${{ matrix.transformers-version }} - - name: Install dependencies run: | python -m pip install --upgrade pip @@ -62,6 +58,10 @@ jobs: pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu pip install .[tests] + - name: Install lowest compatible transformers version + if: ${{ matrix.transformers-version != 'latest' }} + run: pip install transformers==${{ matrix.transformers-version }} + - name: Install openvino-nightly if: ${{ matrix.openvino == 'ov-nightly' }} run: pip install --pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly From c3e7a60fd71cc3e59648f3aec315b31c04a4d99e Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Sat, 30 Nov 2024 16:08:56 +0100 Subject: [PATCH 11/13] Trigger Tests From b6e181321fbc81f9c0c4eae786af95dd3ddb17fa Mon Sep 17 00:00:00 2001 From: Nikita Savelyev Date: Tue, 3 Dec 2024 12:57:03 +0100 Subject: [PATCH 12/13] Move transformers installation to the end --- .github/workflows/test_openvino_full.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_openvino_full.yml b/.github/workflows/test_openvino_full.yml index aace2aca0a..914035b750 100644 --- a/.github/workflows/test_openvino_full.yml +++ b/.github/workflows/test_openvino_full.yml @@ -58,10 +58,6 @@ jobs: pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu pip install .[tests] - - name: Install lowest compatible transformers version - if: ${{ matrix.transformers-version != 'latest' }} - run: pip install transformers==${{ matrix.transformers-version }} - - name: Install openvino-nightly if: ${{ matrix.openvino == 'ov-nightly' }} run: pip install --pre -U openvino openvino-tokenizers --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly @@ -78,6 +74,10 @@ jobs: if: ${{ matrix.nncf == 'nncf-stable' }} run: pip install .[nncf] + - name: Install the lowest compatible transformers version + if: ${{ matrix.transformers-version != 'latest' }} + run: pip install transformers==${{ matrix.transformers-version }} + - name: Pip freeze run: pip freeze From 5df88b2da4a1ae2e416ec11f2dc65abc0c5dc2ae Mon Sep 17 00:00:00 2001 From: Ilyas Moutawwakil <57442720+IlyasMoutawwakil@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:29:12 +0100 Subject: [PATCH 13/13] Update .github/workflows/test_openvino_slow.yml --- .github/workflows/test_openvino_slow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test_openvino_slow.yml b/.github/workflows/test_openvino_slow.yml index 42aa11cdaf..6f9c6167c4 100644 --- a/.github/workflows/test_openvino_slow.yml +++ b/.github/workflows/test_openvino_slow.yml @@ -60,6 +60,10 @@ jobs: run: | pip install .[nncf] + - if: ${{ matrix.transformers-version != 'latest' }} + name: Downgrade Transformers and Accelerate + run: pip install transformers==${{ matrix.transformers-version }} accelerate==0.* + - name: Test with Pytest (slow) run: | pytest tests/openvino -m "run_slow" --durations=0