From 60d780f99e6cf8274e10bc371b51a47a90e30ae8 Mon Sep 17 00:00:00 2001 From: mzr1996 Date: Thu, 20 Jul 2023 10:21:15 +0800 Subject: [PATCH 1/4] Fix docs --- configs/llava/README.md | 2 +- docs/en/notes/changelog.md | 2 +- docs/en/useful_tools/shape_bias.md | 10 +++++----- mmpretrain/models/multimodal/flamingo/flamingo.py | 2 +- mmpretrain/models/multimodal/minigpt4/minigpt4.py | 2 +- mmpretrain/models/multimodal/otter/otter.py | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/llava/README.md b/configs/llava/README.md index 31275f7748c..7aaf57d7d13 100644 --- a/configs/llava/README.md +++ b/configs/llava/README.md @@ -21,7 +21,7 @@ Instruction tuning large language models (LLMs) using machine-generated instruct According to the license of LLaMA, we cannot provide the merged checkpoint directly. Please use the below script to download and get the merged the checkpoint. -```baseh +```shell python tools/model_converters/llava-delta2mmpre.py huggyllama/llama-7b liuhaotian/LLaVA-Lightning-7B-delta-v1-1 ./LLaVA-Lightning-7B-delta-v1-1.pth ``` diff --git a/docs/en/notes/changelog.md b/docs/en/notes/changelog.md index 219797b4f8f..f25923d4f62 100644 --- a/docs/en/notes/changelog.md +++ b/docs/en/notes/changelog.md @@ -7,7 +7,7 @@ - Support inference of more **multi-modal** algorithms, such as **LLaVA**, **MiniGPT-4**, **Otter**, etc. - Support around **10 multi-modal datasets**! - Add **iTPN**, **SparK** self-supervised learning algorithms. -- Provide examples of [New Config](./mmpretrain/configs/) and [DeepSpeed/FSDP](./configs/mae/benchmarks/). +- Provide examples of [New Config](https://github.com/open-mmlab/mmpretrain/tree/main/mmpretrain/configs/) and [DeepSpeed/FSDP](https://github.com/open-mmlab/mmpretrain/tree/main/configs/mae/benchmarks/). ### New Features diff --git a/docs/en/useful_tools/shape_bias.md b/docs/en/useful_tools/shape_bias.md index ea4f96c46be..907bde61ee7 100644 --- a/docs/en/useful_tools/shape_bias.md +++ b/docs/en/useful_tools/shape_bias.md @@ -1,10 +1,10 @@ -## Shape Bias Tool Usage +# Shape Bias Tool Usage Shape bias measures how a model relies the shapes, compared to texture, to sense the semantics in images. For more details, we recommend interested readers to this [paper](https://arxiv.org/abs/2106.07411). MMPretrain provide an off-the-shelf toolbox to obtain the shape bias of a classification model. You can following these steps below: -### Prepare the dataset +## Prepare the dataset First you should download the [cue-conflict](https://github.com/bethgelab/model-vs-human/releases/download/v0.1/cue-conflict.tar.gz) to `data` folder, and then unzip this dataset. After that, you `data` folder should have the following structure: @@ -18,7 +18,7 @@ data | |── truck ``` -### Modify the config for classification +## Modify the config for classification We run the shape-bias tool on a ViT-base model with masked autoencoder pretraining. Its config file is `configs/mae/benchmarks/vit-base-p16_8xb128-coslr-100e_in1k.py`, and its checkpoint is downloaded from [this link](https://download.openmmlab.com/mmselfsup/1.x/mae/mae_vit-base-p16_8xb512-fp16-coslr-1600e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k/vit-base-p16_ft-8xb128-coslr-100e_in1k_20220825-cf70aa21.pth). Replace the original test_pipeline, test_dataloader and test_evaluation with the following configurations: @@ -55,7 +55,7 @@ test_evaluator = dict( Please note you should make custom modifications to the `csv_dir` and `model_name` above. I renamed my modified sample config file as `vit-base-p16_8xb128-coslr-100e_in1k_shape-bias.py` in the folder `configs/mae/benchmarks/`. -### Inference your model with above modified config file +## Inference your model with above modified config file Then you should inferece your model on the `cue-conflict` dataset with the your modified config file. @@ -77,7 +77,7 @@ bash tools/dist_test.sh configs/mae/benchmarks/vit-base-p16_8xb128-coslr-100e_in After that, you should obtain a csv file in `csv_dir` folder, named `cue-conflict_model-name_session-1.csv`. Besides this file, you should also download these [csv files](https://github.com/bethgelab/model-vs-human/tree/master/raw-data/cue-conflict) to the `csv_dir`. -### Plot shape bias +## Plot shape bias Then we can start to plot the shape bias: diff --git a/mmpretrain/models/multimodal/flamingo/flamingo.py b/mmpretrain/models/multimodal/flamingo/flamingo.py index abdd03328f4..1c19875b8b4 100644 --- a/mmpretrain/models/multimodal/flamingo/flamingo.py +++ b/mmpretrain/models/multimodal/flamingo/flamingo.py @@ -23,7 +23,7 @@ class Flamingo(BaseModel): zeroshot_prompt (str): Prompt used for zero-shot inference. Defaults to 'Output:'. shot_prompt_tmpl (str): Prompt used for few-shot inference. - Defaults to 'Output:{caption}<|endofchunk|>'. + Defaults to ``Output:{caption}<|endofchunk|>``. final_prompt_tmpl (str): Final part of prompt used for inference. Defaults to 'Output:'. generation_cfg (dict): The extra generation config, accept the keyword diff --git a/mmpretrain/models/multimodal/minigpt4/minigpt4.py b/mmpretrain/models/multimodal/minigpt4/minigpt4.py index d23203603ec..4616c2e2597 100644 --- a/mmpretrain/models/multimodal/minigpt4/minigpt4.py +++ b/mmpretrain/models/multimodal/minigpt4/minigpt4.py @@ -36,7 +36,7 @@ class MiniGPT4(BaseModel): raw_prompts (list): Prompts for training. Defaults to None. max_txt_len (int): Max token length while doing tokenization. Defaults to 32. - end_sym (str): Ended symbol of the sequence. Defaults to '\n'. + end_sym (str): Ended symbol of the sequence. Defaults to '\\n'. generation_cfg (dict): The config of text generation. Defaults to dict(). data_preprocessor (:obj:`BaseDataPreprocessor`): Used for diff --git a/mmpretrain/models/multimodal/otter/otter.py b/mmpretrain/models/multimodal/otter/otter.py index 2fed1a4d27c..f1eb61baead 100644 --- a/mmpretrain/models/multimodal/otter/otter.py +++ b/mmpretrain/models/multimodal/otter/otter.py @@ -20,8 +20,8 @@ class Otter(Flamingo): zeroshot_prompt (str): Prompt used for zero-shot inference. Defaults to an. shot_prompt_tmpl (str): Prompt used for few-shot inference. - Defaults to 'User:Please describe the image. - GPT:{caption}<|endofchunk|>'. + Defaults to ``User:Please describe the image. + GPT:{caption}<|endofchunk|>``. final_prompt_tmpl (str): Final part of prompt used for inference. Defaults to 'User:Please describe the image. GPT:'. generation_cfg (dict): The extra generation config, accept the keyword From 2b8d8eecb2c8b5f8a8e97b5556fc8cf0c99a2316 Mon Sep 17 00:00:00 2001 From: liyl <32213940+liyunlong10@users.noreply.github.com> Date: Tue, 25 Jul 2023 11:25:32 +0800 Subject: [PATCH 2/4] [Fix] Fix the issue #1711 "GaussianBlur doesn't work" (#1722) * Fix issue 1711. GaussianBlur. * Fix UT --------- Co-authored-by: mzr1996 --- mmpretrain/datasets/transforms/auto_augment.py | 2 +- tests/test_datasets/test_transforms/test_auto_augment.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/mmpretrain/datasets/transforms/auto_augment.py b/mmpretrain/datasets/transforms/auto_augment.py index 03b057b850a..4705d5ec04e 100644 --- a/mmpretrain/datasets/transforms/auto_augment.py +++ b/mmpretrain/datasets/transforms/auto_augment.py @@ -1169,7 +1169,7 @@ def transform(self, results): img = results['img'] pil_img = Image.fromarray(img) - pil_img.filter(ImageFilter.GaussianBlur(radius=radius)) + pil_img = pil_img.filter(ImageFilter.GaussianBlur(radius=radius)) results['img'] = np.array(pil_img, dtype=img.dtype) return results diff --git a/tests/test_datasets/test_transforms/test_auto_augment.py b/tests/test_datasets/test_transforms/test_auto_augment.py index fa0761016f4..d9f65c3d993 100644 --- a/tests/test_datasets/test_transforms/test_auto_augment.py +++ b/tests/test_datasets/test_transforms/test_auto_augment.py @@ -1285,9 +1285,10 @@ def test_initialize(self): def test_transform(self): transform_func = 'PIL.ImageFilter.GaussianBlur' + from PIL.ImageFilter import GaussianBlur # test params inputs - with patch(transform_func, autospec=True) as mock: + with patch(transform_func, wraps=GaussianBlur) as mock: cfg = { **self.DEFAULT_ARGS, 'radius': 0.5, @@ -1297,7 +1298,7 @@ def test_transform(self): mock.assert_called_once_with(radius=0.5) # test prob - with patch(transform_func, autospec=True) as mock: + with patch(transform_func, wraps=GaussianBlur) as mock: cfg = { **self.DEFAULT_ARGS, 'radius': 0.5, @@ -1307,7 +1308,7 @@ def test_transform(self): mock.assert_not_called() # test magnitude_range - with patch(transform_func, autospec=True) as mock: + with patch(transform_func, wraps=GaussianBlur) as mock: cfg = { **self.DEFAULT_ARGS, 'magnitude_range': (0.1, 2), From 4d1dbafaa28af29f5cb907588c019ae4878c2d24 Mon Sep 17 00:00:00 2001 From: Nripesh Niketan <86844847+NripeshN@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:21:00 +0530 Subject: [PATCH 3/4] [Enhance] Add GPU Acceleration Apple silicon mac (#1699) * Add GPU Acceleration Apple silicon mac * lint fix * Update launch.py * Use to refactor the device selection. * Update launch.py --------- Co-authored-by: mzr1996 --- mmpretrain/engine/hooks/swav_hook.py | 5 ++++- mmpretrain/models/heads/itpn_clip_head.py | 4 ++-- projects/gradio_demo/launch.py | 19 ++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/mmpretrain/engine/hooks/swav_hook.py b/mmpretrain/engine/hooks/swav_hook.py index 71c82ad1e0c..be5f3a36bdd 100644 --- a/mmpretrain/engine/hooks/swav_hook.py +++ b/mmpretrain/engine/hooks/swav_hook.py @@ -3,6 +3,7 @@ from typing import Dict, List, Optional, Sequence import torch +from mmengine.device import get_device from mmengine.dist import get_rank, get_world_size, is_distributed from mmengine.hooks import Hook from mmengine.logging import MMLogger @@ -97,11 +98,13 @@ def before_train_epoch(self, runner) -> None: if self.queue_length > 0 \ and runner.epoch >= self.epoch_queue_starts \ and self.queue is None: + self.queue = torch.zeros( len(self.crops_for_assign), self.queue_length // runner.world_size, self.feat_dim, - ).cuda() + device=get_device(), + ) # set the boolean type of use_the_queue get_ori_model(runner.model).head.loss_module.queue = self.queue diff --git a/mmpretrain/models/heads/itpn_clip_head.py b/mmpretrain/models/heads/itpn_clip_head.py index 7465d7c2a89..52c49b8c013 100644 --- a/mmpretrain/models/heads/itpn_clip_head.py +++ b/mmpretrain/models/heads/itpn_clip_head.py @@ -3,6 +3,7 @@ import torch import torch.nn as nn +from mmengine.device import get_device from mmengine.model import BaseModule from mmpretrain.registry import MODELS @@ -43,8 +44,7 @@ def loss(self, feats: torch.Tensor, target: torch.Tensor, target (torch.Tensor): Target generated by target_generator. mask (torch.Tensor): Generated mask for pretraing. """ - - mask = mask.to(torch.device('cuda'), non_blocking=True) + mask = mask.to(get_device(), non_blocking=True) mask = mask.flatten(1).to(torch.bool) target = target[mask] diff --git a/projects/gradio_demo/launch.py b/projects/gradio_demo/launch.py index 191ae094e31..61bccee54a2 100644 --- a/projects/gradio_demo/launch.py +++ b/projects/gradio_demo/launch.py @@ -20,25 +20,26 @@ logger = MMLogger('mmpretrain', logger_name='mmpre') if torch.cuda.is_available(): - gpus = [ + devices = [ torch.device(f'cuda:{i}') for i in range(torch.cuda.device_count()) ] - logger.info(f'Available GPUs: {len(gpus)}') + logger.info(f'Available GPUs: {len(devices)}') +elif hasattr(torch.backends, 'mps') and torch.backends.mps.is_available(): + devices = [torch.device('mps')] + logger.info('Available MPS.') else: - gpus = None - logger.info('No available GPU.') + devices = [torch.device('cpu')] + logger.info('Available CPU.') def get_free_device(): - if gpus is None: - return torch.device('cpu') if hasattr(torch.cuda, 'mem_get_info'): - free = [torch.cuda.mem_get_info(gpu)[0] for gpu in gpus] + free = [torch.cuda.mem_get_info(gpu)[0] for gpu in devices] select = max(zip(free, range(len(free))))[1] else: import random - select = random.randint(0, len(gpus) - 1) - return gpus[select] + select = random.randint(0, len(devices) - 1) + return devices[select] class InferencerCache: From c5248b17b729227d3966233c151160715c0a1cf2 Mon Sep 17 00:00:00 2001 From: Yinlei Sun Date: Fri, 28 Jul 2023 13:39:38 +0800 Subject: [PATCH 4/4] [Enhance] Adapt test cases on Ascend NPU. (#1728) --- tests/test_engine/test_hooks/test_densecl_hook.py | 3 ++- tests/test_engine/test_hooks/test_ema_hook.py | 5 +++-- tests/test_engine/test_hooks/test_simsiam_hook.py | 3 ++- tests/test_engine/test_hooks/test_swav_hook.py | 3 ++- tests/test_engine/test_hooks/test_switch_recipe_hook.py | 9 +++++---- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tests/test_engine/test_hooks/test_densecl_hook.py b/tests/test_engine/test_hooks/test_densecl_hook.py index f567f643ccd..645d10216b5 100644 --- a/tests/test_engine/test_hooks/test_densecl_hook.py +++ b/tests/test_engine/test_hooks/test_densecl_hook.py @@ -5,6 +5,7 @@ import torch import torch.nn as nn +from mmengine.device import get_device from mmengine.logging import MMLogger from mmengine.model import BaseModule from mmengine.optim import OptimWrapper @@ -79,7 +80,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_densecl_hook(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() dummy_dataset = DummyDataset() toy_model = ToyModel().to(device) densecl_hook = DenseCLHook(start_iters=1) diff --git a/tests/test_engine/test_hooks/test_ema_hook.py b/tests/test_engine/test_hooks/test_ema_hook.py index c7eead147da..05207256201 100644 --- a/tests/test_engine/test_hooks/test_ema_hook.py +++ b/tests/test_engine/test_hooks/test_ema_hook.py @@ -8,6 +8,7 @@ import torch import torch.nn as nn +from mmengine.device import get_device from mmengine.evaluator import Evaluator from mmengine.logging import MMLogger from mmengine.model import BaseModel @@ -70,7 +71,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_load_state_dict(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) ema_hook = EMAHook() runner = Runner( @@ -95,7 +96,7 @@ def test_load_state_dict(self): def test_evaluate_on_ema(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) # Test validate on ema model diff --git a/tests/test_engine/test_hooks/test_simsiam_hook.py b/tests/test_engine/test_hooks/test_simsiam_hook.py index 6101c0769f8..29bda933c0c 100644 --- a/tests/test_engine/test_hooks/test_simsiam_hook.py +++ b/tests/test_engine/test_hooks/test_simsiam_hook.py @@ -5,6 +5,7 @@ import torch import torch.nn as nn +from mmengine.device import get_device from mmengine.logging import MMLogger from mmengine.model import BaseModule from mmengine.runner import Runner @@ -79,7 +80,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_simsiam_hook(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() dummy_dataset = DummyDataset() toy_model = ToyModel().to(device) simsiam_hook = SimSiamHook( diff --git a/tests/test_engine/test_hooks/test_swav_hook.py b/tests/test_engine/test_hooks/test_swav_hook.py index 2c2d005453b..2239ccb764a 100644 --- a/tests/test_engine/test_hooks/test_swav_hook.py +++ b/tests/test_engine/test_hooks/test_swav_hook.py @@ -5,6 +5,7 @@ import torch import torch.nn as nn +from mmengine.device import get_device from mmengine.logging import MMLogger from mmengine.model import BaseModule from mmengine.optim import OptimWrapper @@ -86,7 +87,7 @@ def tearDown(self): self.temp_dir.cleanup() def test_swav_hook(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() dummy_dataset = DummyDataset() toy_model = ToyModel().to(device) swav_hook = SwAVHook( diff --git a/tests/test_engine/test_hooks/test_switch_recipe_hook.py b/tests/test_engine/test_hooks/test_switch_recipe_hook.py index e1b4ebff316..c8c7b564ac5 100644 --- a/tests/test_engine/test_hooks/test_switch_recipe_hook.py +++ b/tests/test_engine/test_hooks/test_switch_recipe_hook.py @@ -10,6 +10,7 @@ import torch.nn as nn from mmcv.transforms import Compose from mmengine.dataset import BaseDataset, ConcatDataset, RepeatDataset +from mmengine.device import get_device from mmengine.logging import MMLogger from mmengine.model import BaseDataPreprocessor, BaseModel from mmengine.optim import OptimWrapper @@ -130,7 +131,7 @@ def test_init(self): self.assertIsNone(hook.schedule[1]['batch_augments']) def test_do_switch(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) loss = CrossEntropyLoss(use_soft=True) @@ -205,7 +206,7 @@ def test_do_switch(self): # runner.train() def test_resume(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) loss = CrossEntropyLoss(use_soft=True) @@ -275,7 +276,7 @@ def test_resume(self): logs.output) def test_switch_train_pipeline(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) runner = Runner( @@ -324,7 +325,7 @@ def test_switch_train_pipeline(self): pipeline) def test_switch_loss(self): - device = 'cuda:0' if torch.cuda.is_available() else 'cpu' + device = get_device() model = SimpleModel().to(device) runner = Runner(