diff --git a/test_convert_llm.sh b/test_convert_llm.sh index 5c46ef378854650570d8aa8f62bf9c34d2be6c43..b29c9a462286bf071f4b443823f82f4d59c1b51c 100644 --- a/test_convert_llm.sh +++ b/test_convert_llm.sh @@ -65,6 +65,14 @@ rm -rf tests cd .. echo "..............................................done apply transformers" +rm -rf peft/ +git clone -b v0.7.1 https://github.com/huggingface/peft.git --depth=1 +if [ $? -ne 0 ]; then + echo "Error:git clone peft failed" + exit 1 +fi +echo "------------------------------------done peft" + echo "..............................................start code_convert" MindSpeed_Core_MS_PATH=$(pwd) echo ${MindSpeed_Core_MS_PATH} @@ -72,7 +80,8 @@ echo ${MindSpeed_Core_MS_PATH} python3 tools/transfer.py \ --megatron_path ${MindSpeed_Core_MS_PATH}/Megatron-LM/megatron/ \ --mindspeed_path ${MindSpeed_Core_MS_PATH}/MindSpeed/mindspeed/ \ ---mindspeed_llm_path ${MindSpeed_Core_MS_PATH}/MindSpeed-LLM/ +--mindspeed_llm_path ${MindSpeed_Core_MS_PATH}/MindSpeed-LLM/ \ +--peft_path ${MindSpeed_Core_MS_PATH}/peft/ export PYTHONPATH=${MindSpeed_Core_MS_PATH}/msadapter/mindtorch:${MindSpeed_Core_MS_PATH}/Megatron-LM:${MindSpeed_Core_MS_PATH}/MindSpeed:${MindSpeed_Core_MS_PATH}/MindSpeed-LLM:${MindSpeed_Core_MS_PATH}/transformers/src/:$PYTHONPATH echo $PYTHONPATH diff --git a/tools/rules/line_rules.py b/tools/rules/line_rules.py index cbbe60ddb6cd8620e7a4dd26189e9ae033d28469..0a9a8572bd1f137350398752ff637474008a255c 100644 --- a/tools/rules/line_rules.py +++ b/tools/rules/line_rules.py @@ -1756,5 +1756,180 @@ def create_worker_group_scheduler(name, world_size, name_prefix): else:""","""- torch_npu.npu.empty_cache() + # torch_npu.npu.empty_cache()"""], - } + }, + "peft":{ +"src/peft/__init__.py": [""" LoraModel, + LoHaConfig, + LoHaModel, +- LoKrConfig, +- LoKrModel, + IA3Config, + IA3Model, + AdaLoraConfig, +""" +], +"src/peft/mapping.py": [""" IA3Model, + LoHaConfig, + LoHaModel, +- LoKrConfig, +- LoKrModel, + LoraConfig, + LoraModel, + MultitaskPromptTuningConfig,""",""" "P_TUNING": PromptEncoderConfig, + "LORA": LoraConfig, + "LOHA": LoHaConfig, +- "LOKR": LoKrConfig, + "ADALORA": AdaLoraConfig, + "IA3": IA3Config, + "MULTITASK_PROMPT_TUNING": MultitaskPromptTuningConfig,""",""" PEFT_TYPE_TO_TUNER_MAPPING = { + "LORA": LoraModel, + "LOHA": LoHaModel, +- "LOKR": LoKrModel, + "ADALORA": AdaLoraModel, + "IA3": IA3Model, + "OFT": OFTModel, +""" +], +"src/peft/mixed_model.py": [""" from typing import Any, Optional, Union + + import torch +-from accelerate.hooks import remove_hook_from_submodules ++#from accelerate.hooks import remove_hook_from_submodules + from torch import nn + from transformers.utils import PushToHubMixin + """,""" AdaLoraModel, + IA3Model, + LoHaModel, +- LoKrModel, + LoraModel, + MixedModel, + OFTModel,""",""" PEFT_TYPE_TO_MODEL_MAPPING = { + PeftType.LORA: LoraModel, + PeftType.LOHA: LoHaModel, +- PeftType.LOKR: LoKrModel, + PeftType.ADALORA: AdaLoraModel, + PeftType.IA3: IA3Model, + PeftType.OFT: OFTModel, +""" +], +"src/peft/peft_model.py": [""" import packaging.version + import torch + import transformers +-from accelerate import dispatch_model, infer_auto_device_map +-from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules +-from accelerate.utils import get_balanced_memory ++#from accelerate import dispatch_model, infer_auto_device_map ++#from accelerate.hooks import AlignDevicesHook, add_hook_to_module, remove_hook_from_submodules ++#from accelerate.utils import get_balanced_memory + from huggingface_hub import ModelCard, ModelCardData, hf_hub_download + from safetensors.torch import save_file as safe_save_file + from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss""",""" AdaptionPromptModel, + IA3Model, + LoHaModel, +- LoKrModel, + LoraModel, + MultitaskPromptEmbedding, + OFTModel,""",""" PEFT_TYPE_TO_MODEL_MAPPING = { + PeftType.LORA: LoraModel, + PeftType.LOHA: LoHaModel, +- PeftType.LOKR: LoKrModel, + PeftType.PROMPT_TUNING: PromptEmbedding, + PeftType.P_TUNING: PromptEncoder, + PeftType.PREFIX_TUNING: PrefixEncoder, +""" +], +"src/peft/tuners/__init__.py": [""" from .adaption_prompt import AdaptionPromptConfig, AdaptionPromptModel + from .lora import LoraConfig, LoraModel, LoftQConfig + from .loha import LoHaConfig, LoHaModel +-from .lokr import LoKrConfig, LoKrModel ++#from .lokr import LoKrConfig, LoKrModel + from .ia3 import IA3Config, IA3Model + from .adalora import AdaLoraConfig, AdaLoraModel + from .p_tuning import PromptEncoder, PromptEncoderConfig, PromptEncoderReparameterizationType +""" +], +"src/peft/tuners/mixed/model.py": [""" from torch import nn + from tqdm import tqdm + +-from peft.tuners import adalora, loha, lokr, lora, oft ++from peft.tuners import lora ++#from peft.tuners import adalora, loha, lokr, lora, oft + from peft.tuners.tuners_utils import BaseTuner, BaseTunerLayer, check_target_module_exists + from peft.utils import ( + TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING,""",""" + # Collection of constants used for all tuners + COMPATIBLE_TUNER_TYPES = (PeftType.LORA, PeftType.LOHA, PeftType.LOKR, PeftType.ADALORA, PeftType.OFT) +-PREFIXES = [lora.LoraModel.prefix, lokr.LoKrModel.prefix, loha.LoHaModel.prefix, oft.OFTModel.prefix] +-Configs = Union[lora.LoraConfig, loha.LoHaConfig, lokr.LoKrConfig, adalora.AdaLoraConfig, oft.OFTConfig] +-Layers = (lora.layer.LoraLayer, loha.layer.LoHaLayer, lokr.layer.LoKrLayer, adalora.layer.AdaLoraLayer, oft.OFTLayer) +- ++#PREFIXES = [lora.LoraModel.prefix, lokr.LoKrModel.prefix, loha.LoHaModel.prefix, oft.OFTModel.prefix] ++#Configs = Union[lora.LoraConfig, loha.LoHaConfig, lokr.LoKrConfig, adalora.AdaLoraConfig, oft.OFTConfig] ++#Layers = (lora.layer.LoraLayer, loha.layer.LoHaLayer, lokr.layer.LoKrLayer, adalora.layer.AdaLoraLayer, oft.OFTLayer) ++PREFIXES = [lora.LoraModel.prefix] ++Configs = Union[lora.LoraConfig] ++Layers = (lora.layer.LoraLayer) + + class MixedModel(BaseTuner): + \"\"\"""",""" *args: Any, + **kwargs: Any, + ) -> None: ++ ++ if isinstance(config, lora.LoraConfig): ++ lora.LoraModel._create_and_replace(self, config, *args, **kwargs) ++ else: ++ raise ValueError(f"Unsupported config type {type(config)}, should be one of {COMPATIBLE_TUNER_TYPES}.") ++ ''' + if isinstance(config, adalora.AdaLoraConfig): + adalora.AdaLoraModel._create_and_replace(self, config, *args, **kwargs) + elif isinstance(config, lora.LoraConfig):""",""" lokr.LoKrModel._create_and_replace(self, config, *args, **kwargs) + elif isinstance(config, oft.OFTConfig): + oft.OFTModel._create_and_replace(self, config, *args, **kwargs) ++ + else: + raise ValueError(f"Unsupported config type {type(config)}, should be one of {COMPATIBLE_TUNER_TYPES}.") ++ ''' + + def _replace_module(self, parent, child_name, new_module, child) -> None: + setattr(parent, child_name, new_module)""",""" loaded_in_4bit = kwargs.pop("loaded_in_4bit", False) + if loaded_in_8bit or loaded_in_4bit: + raise ValueError(f"8bit and 4bit quantization not supported for {config.peft_type.value} (yet).") +- ++ ++ if isinstance(config, lora.LoraConfig): ++ new_module = lora.LoraModel._create_new_module(config, adapter_name, target, **kwargs) ++ else: ++ raise ValueError(f"Unknown config type {type(config)}, should be one of {COMPATIBLE_TUNER_TYPES}.") ++ ''' + if isinstance(config, adalora.AdaLoraConfig): + new_module = adalora.AdaLoraModel._create_new_module(config, adapter_name, target, **kwargs) + elif isinstance(config, lora.LoraConfig):""",""" new_module = lokr.LoKrModel._create_new_module(config, adapter_name, target, **kwargs) + elif isinstance(config, oft.OFTConfig): + new_module = oft.OFTModel._create_new_module(config, adapter_name, target, **kwargs) ++ + else: + raise ValueError(f"Unknown config type {type(config)}, should be one of {COMPATIBLE_TUNER_TYPES}.") ++ ''' + return new_module + + def _set_adapter_layers(self, enabled=True): +""" +], +"src/peft/utils/other.py": [""" import warnings + from typing import Optional, Tuple + +-import accelerate ++#import accelerate + import torch +-from accelerate.hooks import add_hook_to_module, remove_hook_from_module +-from accelerate.utils import is_npu_available, is_xpu_available ++#from accelerate.hooks import add_hook_to_module, remove_hook_from_module ++#from accelerate.utils import is_npu_available, is_xpu_available + from safetensors.torch import storage_ptr, storage_size + + from ..import_utils import is_auto_gptq_available, is_torch_tpu_available +""" +], + +} } \ No newline at end of file diff --git a/tools/transfer.py b/tools/transfer.py index 9dae9139d48b44878b68c0dcab932e86dad088ac..90ed9981e826389f38b524cf5f7d92c71a5f0404 100644 --- a/tools/transfer.py +++ b/tools/transfer.py @@ -109,7 +109,7 @@ def convert_special_rules_by_line(origin_path, save_path, package_name="megatron def convert_package(origin_path, save_path, package_name="megatron"): """convert_package""" - if package_name == "MindSpeed-LLM": + if package_name in ["MindSpeed-LLM", "peft"]: convert_special_rules_by_line(origin_path, save_path, package_name=package_name) else: convert_special_rules_by_line(origin_path, save_path, package_name=package_name) @@ -150,6 +150,10 @@ if __name__ == "__main__": help="origin einops package path") parser.add_argument("--convert_einops_path", type=str, default=None, help="convert einops package path") + parser.add_argument("--peft_path", type=str, default=None, + help="origin peft package path") + parser.add_argument("--convert_peft_path", type=str, default=None, + help="convert peft package path") parser.add_argument("--is_rl", action="store_true", help="is rl") args = parser.parse_args() @@ -197,3 +201,8 @@ if __name__ == "__main__": origin_path = args.einops_path save_path = origin_path if not args.convert_einops_path else args.convert_einops_path convert_special_rules(origin_path, save_path, package_name="einops") + + if args.peft_path: + origin_path = args.peft_path + save_path = origin_path if not args.convert_peft_path else args.convert_peft_path + convert_package(origin_path, save_path, "peft")