diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/README.md b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ec90e559253839cace085a654f48716047341aaa --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/README.md @@ -0,0 +1,148 @@ +# IndexKits + +[TOC] + +## Introduction + +Index Kits (`index_kits`) for streaming Arrow data. + +* Supports creating datasets from configuration files. +* Supports creating index v2 from arrows. +* Supports the creation, reading, and usage of index files. +* Supports the creation, reading, and usage of multi-bucket, multi-resolution indexes. + +`index_kits` has the following advantages: + +* Optimizes memory usage for streaming reads, supporting the reading of data at the billion level. +* Optimizes memory usage and reading speed of Index files. +* Supports the creation of Base/Multireso Index V2 datasets, including data filtering, repeating, and deduplication during creation. +* Supports loading various dataset types (Arrow files, Base Index V2, multiple Base Index V2, Multireso Index V2, multiple Multireso Index V2). +* Uses a unified API interface to access images, text, and other attributes for all dataset types. +* Built-in shuffle, `resize_and_crop`, and returns the coordinates of the crop. + +## Installation + +* **Install from pre-compiled whl (recommended)** + +* Install from source + + ```shell + cd IndexKits + pip install -e . + ``` + +## Usage + +### Loading an Index V2 File + +```python +from index_kits import ArrowIndexV2 + +index_manager = ArrowIndexV2('data.json') + +# You can shuffle the dataset using a seed. If a seed is provided (i.e., the seed is not None), this shuffle will not affect any random state. +index_manager.shuffle(1234, fast=True) + +for i in range(len(index_manager)): + # Get an image (requires the arrow to contain an image/binary column): + pil_image = index_manager.get_image(i) + # Get text (requires the arrow to contain a text_zh column): + text = index_manager.get_attribute(i, column='text_zh') + # Get MD5 (requires the arrow to contain an md5 column): + md5 = index_manager.get_md5(i) + # Get any attribute by specifying the column (must be contained in the arrow): + ocr_num = index_manager.get_attribute(i, column='ocr_num') + # Get multiple attributes at once + item = index_manager.get_data(i, columns=['text_zh', 'md5']) # i: in-dataset index + print(item) + # { + # 'index': 3, # in-json index + # 'in_arrow_index': 3, # in-arrow index + # 'arrow_name': '/HunYuanDiT/dataset/porcelain/00000.arrow', + # 'text_zh': 'Fortune arrives with the auspicious green porcelain tea cup', + # 'md5': '1db68f8c0d4e95f97009d65fdf7b441c' + # } +``` + + +### Loading a Set of Arrow Files + +If you have a small batch of arrow files, you can directly use `IndexV2Builder` to load these arrow files without creating an Index V2 file. + +```python +from index_kits import IndexV2Builder + +index_manager = IndexV2Builder(arrow_files).to_index_v2() +``` + +### Loading a Multi-Bucket (Multi-Resolution) Index V2 File + +When using a multi-bucket (multi-resolution) index file, refer to the following example code, especially the definition of SimpleDataset and the usage of MultiResolutionBucketIndexV2. + +```python +from torch.utils.data import DataLoader, Dataset +import torchvision.transforms as T + +from index_kits import MultiResolutionBucketIndexV2 +from index_kits.sampler import BlockDistributedSampler + +class SimpleDataset(Dataset): + def __init__(self, index_file, batch_size, world_size): + # When using multi-bucket (multi-resolution), batch_size and world_size need to be specified for underlying data alignment. + self.index_manager = MultiResolutionBucketIndexV2(index_file, batch_size, world_size) + + self.flip_norm = T.Compose( + [ + T.RandomHorizontalFlip(), + T.ToTensor(), + T.Normalize([0.5], [0.5]), + ] + ) + + def shuffle(self, seed, fast=False): + self.index_manager.shuffle(seed, fast=fast) + + def __len__(self): + return len(self.index_manager) + + def __getitem__(self, idx): + image = self.index_manager.get_image(idx) + original_size = image.size # (w, h) + target_size = self.index_manager.get_target_size(idx) # (w, h) + image, crops_coords_left_top = self.index_manager.resize_and_crop(image, target_size) + image = self.flip_norm(image) + text = self.index_manager.get_attribute(idx, column='text_zh') + return image, text, (original_size, target_size, crops_coords_left_top) + +batch_size = 8 # batch_size per GPU +world_size = 8 # total number of GPUs +rank = 0 # rank of the current process +num_workers = 4 # customizable based on actual conditions +shuffle = False # must be set to False +drop_last = True # must be set to True + +# Correct batch_size and world_size must be passed in, otherwise, the multi-resolution data cannot be aligned correctly. +dataset = SimpleDataset('data_multireso.json', batch_size=batch_size, world_size=world_size) +# Must use BlockDistributedSampler to ensure the samples in a batch have the same resolution. +sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, + shuffle=shuffle, drop_last=drop_last, batch_size=batch_size) +loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, sampler=sampler, + num_workers=num_workers, pin_memory=True, drop_last=drop_last) + +for epoch in range(10): + # Please use the shuffle method provided by the dataset, not the DataLoader's shuffle parameter. + dataset.shuffle(epoch, fast=True) + for batch in loader: + pass +``` + + +## Fast Shuffle + +When your index v2 file contains hundreds of millions of samples, using the default shuffle can be quite slow. Therefore, it’s recommended to enable `fast=True` mode: + +```python +index_manager.shuffle(seed=1234, fast=True) +``` + +This will globally shuffle without keeping the indices of the same arrow together. Although this may reduce reading speed, it requires a trade-off based on the model’s forward time. \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/bin/idk b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/bin/idk new file mode 100644 index 0000000000000000000000000000000000000000..0ed34413004a4b82f3c598bf77c8c62c3c6d3f36 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/bin/idk @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +import argparse + +from index_kits.dataset.make_dataset_core import startup, make_multireso +from index_kits.common import show_index_info +from index_kits import __version__ + + +def common_args(parser): + parser.add_argument('-t', '--target', type=str, required=True, help='Save path') + + +def get_args(): + parser = argparse.ArgumentParser(description=""" + IndexKits is a tool to build and manage index files for large-scale datasets. + It supports both base index and multi-resolution index. + + Introduction + ------------ + This command line tool provides the following functionalities: + 1. Show index v2 information + 2. Build base index v2 + 3. Build multi-resolution index v2 + + Examples + -------- + 1. Show index v2 information + index_kits show /path/to/index.json + + 2. Build base index v2 + Default usage: + index_kits base -c /path/to/config.yaml -t /path/to/index.json + + Use multiple processes: + index_kits base -c /path/to/config.yaml -t /path/to/index.json -w 40 + + 3. Build multi-resolution index v2 + + Build with a configuration file: + index_kits multireso -c /path/to/config.yaml -t /path/to/index_mb_gt512.json + + Build by specifying arguments without a configuration file: + index_kits multireso --src /path/to/index.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + + Build by specifying target-ratios: + index_kits multireso --src /path/to/index.json --base-size 512 --target-ratios 1:1 4:3 3:4 16:9 9:16 --min-size 512 -t /path/to/index_mb_gt512.json + + Build with multiple source index files. + index_kits multireso --src /path/to/index1.json /path/to/index2.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + """, formatter_class=argparse.RawTextHelpFormatter) + sub_parsers = parser.add_subparsers(dest='task', required=True) + + # Show index message + show_parser = sub_parsers.add_parser('show', description=""" + Show base/multireso index v2 information. + + Example + ------- + index_kits show /path/to/index.json + """, formatter_class=argparse.RawTextHelpFormatter) + show_parser.add_argument('src', type=str, help='Path to a base/multireso index file.') + show_parser.add_argument('--arrow-files', action='store_true', help='Show arrow files only.') + show_parser.add_argument('--depth', type=int, default=1, + help='Arrow file depth. Default is 1, the level of last folder in the arrow file path. ' + 'Set it to 0 to show the full path including `xxx/last_folder/*.arrow`.') + + # Single resolution bucket + base_parser = sub_parsers.add_parser('base', description=""" + Build base index v2. + + Example + ------- + index_kits base -c /path/to/config.yaml -t /path/to/index.json + """, formatter_class=argparse.RawTextHelpFormatter) + base_parser.add_argument('-c', '--config', type=str, required=True, help='Configuration file path') + common_args(base_parser) + base_parser.add_argument('-w', '--world-size', type=int, default=1) + base_parser.add_argument('--work-dir', type=str, default='.', help='Work directory') + base_parser.add_argument('--use-cache', action='store_true', help='Use cache to avoid reprocessing. ' + 'Perform merge pkl results directly.') + + # Multi-resolution bucket + mo_parser = sub_parsers.add_parser('multireso', description=""" + Build multi-resolution index v2 + + Example + ------- + Build with a configuration file: + index_kits multireso -c /path/to/config.yaml -t /path/to/index_mb_gt512.json + + Build by specifying arguments without a configuration file: + index_kits multireso --src /path/to/index.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + + Build by specifying target-ratios: + index_kits multireso --src /path/to/index.json --base-size 512 --target-ratios 1:1 4:3 3:4 16:9 9:16 --min-size 512 -t /path/to/index_mb_gt512.json + + Build with multiple source index files. + index_kits multireso --src /path/to/index1.json /path/to/index2.json --base-size 512 --reso-step 32 --min-size 512 -t /path/to/index_mb_gt512.json + """, formatter_class=argparse.RawTextHelpFormatter) + mo_parser.add_argument('-c', '--config', type=str, default=None, + help='Configuration file path in a yaml format. Either --config or --src must be provided.') + mo_parser.add_argument('-s', '--src', type=str, nargs='+', default=None, + help='Source index files. Either --config or --src must be provided.') + common_args(mo_parser) + mo_parser.add_argument('--base-size', type=int, default=None, help="Base size. Typically set as 256/512/1024 according to image size you train model.") + mo_parser.add_argument('--reso-step', type=int, default=None, + help="Resolution step. Either reso_step or target_ratios must be provided.") + mo_parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios. Either reso_step or target_ratios must be provided.") + mo_parser.add_argument('--md5-file', type=str, default=None, + help='You can provide an md5 to height and width file to accelerate the process. ' + 'It is a pickle file that contains a dict, which maps md5 to (height, width) tuple.') + mo_parser.add_argument('--align', type=int, default=16, help="Used when --target-ratios is provided. Align size of source image height and width.") + mo_parser.add_argument('--min-size', type=int, default=0, + help="Minimum size. Images smaller than this size will be ignored.") + + # Common + parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {__version__}') + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = get_args() + if args.task == 'show': + show_index_info(args.src, + args.arrow_files, + args.depth, + ) + elif args.task == 'base': + startup(args.config, + args.target, + args.world_size, + args.work_dir, + use_cache=args.use_cache, + ) + elif args.task == 'multireso': + make_multireso(args.target, + args.config, + args.src, + args.base_size, + args.reso_step, + args.target_ratios, + args.align, + args.min_size, + args.md5_file, + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/MakeDataset.md b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/MakeDataset.md new file mode 100644 index 0000000000000000000000000000000000000000..ae23e766f4526d0e92f5cfba72febb10387ba027 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/MakeDataset.md @@ -0,0 +1,357 @@ +# Use the IndexKits library to create datasets. + +[TOC] + +## Introdution + +The IndexKits library offers a command-line tool `idk` for creating datasets and viewing their statistical information. You can view the instructions by using `idk -h`. Here, it refers to creating an Index V2 format dataset from a series of arrow files. + +## 1. Creating a Base Index V2 Dataset + +### 1.1 Create a Base Index V2 dataset using `idk` + +When creating a Base Index V2 dataset, you need to specify the path to a configuration file using the `-c` parameter and a save path using the `-t` parameter. + +```shell +idk base -c base_config.yaml -t base_dataset.json +``` + +### 1.2 Basic Configuration + +Next, let’s discuss how to write the configuration file. The configuration file is in `yaml` format, and below is a basic example: + +Filename: `base_config.yaml` + +```yaml +source: + - /HunYuanDiT/dataset/porcelain/arrows/00000.arrow +``` + +| Field Name | Type | Description | +|:---------:|:---:|:----------:| +| source | Optional | Arrow List | + + +We provide an example that includes all features and fields in [full_config.yaml](./docs/full_config.yaml). + +### 1.3 Filtering Criteria + +`idk` offers two types of filtering capabilities during the dataset creation process: (1) based on columns in Arrow, and (2) based on MD5 files. + +To enable filtering criteria, add the `filter` field in the configuration file. + +#### 1.3.1 Column Filtering + +To enable column filtering, add the `column` field under the `filter` section. +Multiple column filtering criteria can be applied simultaneously, with the intersection of multiple conditions being taken. + +For example, to select data where both the length and width are greater than or equal to 512, +with the default being 1024 if the length and width are invalid: + +```yaml +filter: + column: + - name: height + type: int + action: ge + target: 512 + default: 1024 + - name: width + type: int + action: ge + target: 512 + default: 1024 +``` + +This filtering condition is equivalent to `table['height'].to_int(default=1024) >= 512 && table['width'].to_int(default=1024) >= 512`. + +Each filtering criterion includes the following fields: + +| Field Name | Type | Description | Value Range | +|:------------------:|:---:|:---------------:|:----------------------:| +| name | Required | Column name in Arrow | Column in Arrow | +| type | Required | Type of Elements in the Column | `int`, `float` or `str` | +| action | Required | Filtering Criteria | See the table below for possible values | +| target | Required | Filtering Criteria | Numeric or String | +| default | Required | Default value when the element is invalid | Numeric or String | +| arrow_file_keyword | Optional | Keywords in the Arrow file path | - | + +Below are the specific meanings of “action” and the optional values under different circumstances:: + +| Action | Description | Type | Action | Description | Type | +|:-------------:|:----------------------------------------------------:|:---------------------:|:------------:|:--------------------------------------------:|:---------------------:| +| eq | equal, `==` | `int`, `float`, `str` | ne | not equal, `!=` | `int`, `float`, `str` | +| gt | great than, `>` | `int`, `float` | lt | less than, `<` | `int`, `float` | +| ge | great or equal, `>=` | `int`, `float` | le | less or equal, `<=` | `int`, `float` | +| len_eq | str length equal, `str.len()==` | `str` | len_ne | str length not equal, `str.len()!=` | `str` | +| len_gt | str length great than, `str.len()>` | `str` | len_lt | str length less than, `str.len()<` | `str` | +| len_ge | str length great or equal, `str.len()>=` | `str` | len_le | str length less or equal, `str.len()<=` | `str` | +| contains | str contains, `str.contains(target)` | `str` | not_contains | str not contains, `str.not_contains(target)` | `str` | +| in | str in, `str.in(target)` | `str` | not_in | str not in, `str.not_in(target)` | `str` | +| lower_last_in | lower str last char in, `str.lower()[-1].in(target)` | `str` | | | | + +#### 1.3.2 MD5 Filtering + +Add an `md5` field under the `filter` section to initiate MD5 filtering. Multiple MD5 filtering criteria can be applied simultaneously, with the intersection of multiple conditions being taken. + +例如: +* `badcase.txt` is a list of MD5s, aiming to filter out the entries listed in these lists. +* `badcase.json` is a dictionary, where the key is the MD5 and the value is a text-related `tag`. The goal is to filter out specific `tags`. + +```shell +filter: + md5: + - name: badcase1 + path: + - badcase1.txt + type: list + action: in + is_valid: false + - name: badcase2 + path: badcase2.json + type: dict + action: eq + target: 'Specified tag' + is_valid: false +``` + +Each filtering criterion includes five fields: + +| Field Name | Type | Description | Value Range | +|:------------------:|:---:|:-----------------------------------------------------------------:|:------------------------------------------------------------------------------------:| +| name | Required | The name of the filtering criterion, which can be customized for ease of statistics. | - | +| path | Required | The path to the filtering file, which can be a single path or multiple paths provided in a list format. Supports `.txt`, `.json`, `.pkl` formats. | - | +| type | Required | The type of records in the filtering file. | `list` or `dict` | +| action | Required | The filtering action. | For `list`: `in`, `not_in`; For `dict`: `eq`, `ne`, `gt`, `lt`, `ge`, `le`| +| target | Optional | The filtering criterion. | Required when type is `dict` | +| is_valid | Required | Whether a hit on action+target is considered valid or invalid. | `true` or `false` | +| arrow_file_keyword | Optional | Keywords in the Arrow file path. | - | + +### 1.4 Advanced Filtering + +`idk` also supports some more advanced filtering functions. + +#### 1.4.1 Filtering Criteria Applied to Part of Arrow Files + +Using the `arrow_file_keyword` parameter, filtering criteria can be applied only to part of the Arrow files. + +For example: + +* The filtering criterion `height>=0` applies only to arrows whose path includes `human`. +* The filtering criterion “keep samples in goodcase.txt” applies only to arrows whose path includes `human`. + +```yaml +filter: + column: + - name: height + type: int + action: ge + target: 512 + default: 1024 + arrow_file_keyword: + - human + + md5: + - name: goodcase + path: goodcase.txt + type: list + action: in + is_valid: true + arrow_file_keyword: + - human +``` + +#### 1.4.2 The “or” Logic in Filtering Criteria + +By default, filtering criteria follow an “and” logic. If you want two or more filtering criteria to follow an “or” logic, you should use the `logical_or` field. The column filtering criteria listed under this field will be combined using an “or” logic. + +```yaml +filter: + column: + - logical_or: + - name: md5 + type: str + action: lower_last_in + target: '02468ace' + default: '' + - name: text_zh + type: str + action: contains + target: 'turtle|rabbit|duck|swan|peacock|hedgehog|wolf|fox|seal|crow|deer' + default: '' +``` + +Special Note: The `logical_or` field is applicable only to the `column` filtering criteria within `filter`. + +1.4.3 Excluding Certain Arrows from the Source + +While wildcards can be used to fetch multiple arrows at once, there might be instances where we want to exclude some of them. This can be achieved through the `exclude` field. Keywords listed under `exclude`, if found in the path of the current group of arrows, will result in those arrows being excluded. + +```yaml +source: + - /HunYuanDiT/dataset/porcelain/arrows/*.arrow: + exclude: + - arrow1 + - arrow2 +``` + +### 1.5 Repeating Samples + +`idk` offers the capability to repeat either all samples or specific samples during dataset creation. There are three types of repeaters: +* Directly repeating the source +* Based on keywords in the Arrow file name (enable repeat conditions by adding the `repeater` field in the configuration file) +* Based on an MD5 file (enable repeat conditions by adding the `repeater` field in the configuration file) + +**Special Note:** The above three conditions can be used simultaneously. If a sample meets multiple repeat conditions, the highest number of repeats will be taken. + +#### 1.5.1 Repeating the Source + +In the source, for the Arrow(s) you want to repeat (which can include wildcards like `*`, `?`), add `repeat: n` to mark the number of times to repeat. + +```yaml +source: + - /HunYuanDiT/dataset/porcelain/arrows/*.arrow: + repeat: 10 +``` + +**Special Note: Add a colon at the end of the Arrow path** + +#### 1.5.2 Arrow Keywords + +Add an `arrow_file_keyword` field under the `repeater` section. + +```yaml +repeater: + arrow_file_keyword: + - repeat: 8 + keyword: + - Lolita anime style + - Minimalist style + - repeat: 5 + keyword: + - Magical Barbie style + - Disney style +``` + +Each repeat condition includes two fields: + + +| Field Name | Type | Description | Value Range | +|:-------:|:---:|:---------------:|:----:| +| repeat | Required | The number of times to repeat | Number | +| keyword | Required | Keywords in the Arrow file path | - | + + +#### 1.5.3 MD5 File + +Add an `md5` field under the `repeater` section. + +```yaml +repeater: + md5: + - name: goodcase1 + path: /HunYuanDiT/dataset/porcelain/md5_repeat_1.json + type: dict + plus: 3 + - name: goodcase2 + path: /HunYuanDiT/dataset/porcelain/md5_repeat_2.json + type: list + repeat: 6 +``` + +Each repeat condition includes the following fields: + + +| Field Name | Type | Description | Value Range | +|:------:|:---:|:----------------------------------------------:|:---------------------------:| +| name | Required | Custom name for the repeat condition | - | +| path | Required | Path to the file containing MD5s | `.txt`, `.json`, or `.pkl` formats | +| type | Required | Type of the MD5 file | `list` or `dict` | +| repeat | Optional | Number of times to repeat, this will override plus | Integer | +| plus | Optional | Adds this value to the value obtained from the MD5 file as the number of repeats | Integer | + +### 1.6 Deduplication + +To deduplicate, add `remove_md5_dup` and set it to `true`. For example: + +```yaml +remove_md5_dup: true +``` + +**Special Note:** Deduplication is performed after repeat conditions, so using them together will make repeat ineffective. + +### 1.7 Creating a Base Index V2 Dataset with Python Code + +```python +from index_kits import IndexV2Builder + +builder = IndexV2Builder(arrow_files) +builder.save('data_v2.json') +``` + + + +## 2. Creating a Multireso Index V2 Dataset + +### 2.1 Creating a Multireso Index V2 Dataset with `idk` +To create a multi-resolution dataset, you can do so through a configuration file: + +```shell +src: + - /HunYuanDiT/dataset/porcelain/jsons/a.json + - /HunYuanDiT/dataset/porcelain/jsons/b.json + - /HunYuanDiT/dataset/porcelain/jsons/c.json +base_size: 512 +reso_step: 32 +min_size: 512 +``` + +The fields are as follows: + +| Field Name | Type | Description | Value Range | +|:-------------:|:---:|:------------------------------------------------------:|:-------------------------------:| +| src | Required | Path(s) to the Base Index V2 file, can be single or multiple | - | +| base_size | Required | The base resolution (n, n) from which to create multiple resolutions | Recommended values: 256/512/1024 | +| reso_step | Optional | The step size for traversing multiple resolutions. Choose either this or target_ratios | Recommended values: 16/32/64 | +| target_ratios | Optional | Target aspect ratios, a list. Choose either this or reso_step | Recommended values: 1:1, 4:3, 3:4, 16:9, 9:16 | +| align | Optional | When using target_ratios, the multiple to align the target resolution to | Recommended value: 16 (2x patchify, 8x VAE) | +| min_size | Optional | The minimum resolution filter for samples when creating from Base to Multireso Index V2 | Recommended values: 256/512/1024 | +| md5_file | Optional | A pre-calculated dictionary of image sizes in pkl format, key is MD5, value is (h, w) | - | + + +### 2.2 Creating a Multireso Index V2 Dataset with Python Code + +First, import the necessary function + +```python +from index_kits import build_multi_resolution_bucket + +md5_hw = None +# If many arrows in your Index V2 lack height and width, you can pre-calculate them and pass through the md5_hw parameter. +# md5_hw = {'c67be1d8f30fd0edcff6ac99b703879f': (720, 1280), ...} +# with open('md5_hw.pkl', 'rb') as f: +# md5_hw = pickle.load(f) + +index_v2_file_path = 'data_v2.json' +index_v2_save_path = 'data_multireso.json' + +# Method 1: Given base_size and reso_step, automatically calculate all resolution buckets. +build_multi_resolution_bucket(base_size=1024, + reso_step=64, + min_size=1024, + src_index_files=index_v2_file_path, + save_file=index_v2_save_path, + md5_hw=md5_hw) + +# Method 2: Given a series of target aspect ratios, automatically calculate all resolution buckets. +build_multi_resolution_bucket(base_size=1024, + target_ratios=["1:1", "3:4", "4:3", "16:9", "9:16"], + align=16, + min_size=1024, + src_index_files=index_v2_file_path, + save_file=index_v2_save_path, + md5_hw=md5_hw) +``` + +**Note：** If both reso_step and target_ratios are provided, target_ratios will be prioritized. + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/full_config.yaml b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/full_config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5a3a6cf91cad244c21da176bb8de8944db4e2d4 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/docs/full_config.yaml @@ -0,0 +1,36 @@ +source: + - /HunyuanDiT/dataset/task1/*.arrow + - /HunyuanDiT/dataset/task2/*.arrow: + repeat: 2 + +remove_md5_dup: false + +filter: + column: + - name: height + type: int + action: ge + target: 512 + default: 1024 + - name: width + type: int + action: ge + target: 512 + default: 1024 + md5: + - name: Badcase + path: /path/to/bad_case_md5.txt + type: list + action: in + is_valid: false + +repeater: + md5: + - name: Hardcase Repeat + path: /path/to/hard_case_md5.json + type: dict + plus: 3 + - name: Goodcase Repeat + path: /path/to/good_case_md5.txt + type: list + repeat: 6 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..248f506991c8f9c8c0de36a7b0030909e338480f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/__init__.py @@ -0,0 +1,11 @@ +from .bucket import ( + MultiIndexV2, + MultiResolutionBucketIndexV2, MultiMultiResolutionBucketIndexV2, + build_multi_resolution_bucket +) +from .bucket import Resolution, ResolutionGroup +from .indexer import IndexV2Builder, ArrowIndexV2 +from .common import load_index, show_index_info + +__version__ = "0.3.5" + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/bucket.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/bucket.py new file mode 100644 index 0000000000000000000000000000000000000000..72b63e6c35ebf9b37ca9dcbed3d90b096e038c5d --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/bucket.py @@ -0,0 +1,888 @@ +import bisect +import json +import random +from pathlib import Path +from typing import Optional, Dict, List, Callable, Union + +import numpy as np +from tqdm import tqdm +from PIL import Image + +from .indexer import ArrowIndexV2, IndexV2Builder + + +class Resolution(object): + def __init__(self, size, *args): + if isinstance(size, str): + if 'x' in size: + size = size.split('x') + size = (int(size[0]), int(size[1])) + else: + size = int(size) + if len(args) > 0: + size = (size, args[0]) + if isinstance(size, int): + size = (size, size) + + self.h = self.height = size[0] + self.w = self.width = size[1] + self.r = self.ratio = self.height / self.width + + def __getitem__(self, idx): + if idx == 0: + return self.h + elif idx == 1: + return self.w + else: + raise IndexError(f'Index {idx} out of range') + + def __str__(self): + return f'{self.h}x{self.w}' + + +class ResolutionGroup(object): + def __init__(self, base_size=None, step=None, align=1, target_ratios=None, enlarge=1, data=None): + self.enlarge = enlarge + + if data is not None: + self.data = data + mid = len(self.data) // 2 + self.base_size = self.data[mid].h + self.step = self.data[mid].h - self.data[mid - 1].h + else: + self.align = align + self.base_size = base_size + assert base_size % align == 0, f'base_size {base_size} is not divisible by align {align}' + if base_size is not None and not isinstance(base_size, int): + raise ValueError(f'base_size must be None or int, but got {type(base_size)}') + if step is None and target_ratios is None: + raise ValueError(f'Either step or target_ratios must be provided') + if step is not None and step > base_size // 2: + raise ValueError(f'step must be smaller than base_size // 2, but got {step} > {base_size // 2}') + + self.step = step + self.data = self.calc(target_ratios) + + self.ratio = np.array([x.ratio for x in self.data]) + self.attr = ['' for _ in range(len(self.data))] + self.prefix_space = 0 + + def __len__(self): + return len(self.data) + + def __getitem__(self, idx): + return self.data[idx] + + def __repr__(self): + prefix = self.prefix_space * ' ' + prefix_close = (self.prefix_space - 4) * ' ' + res_str = f'ResolutionGroup(base_size={self.base_size}, step={self.step}, data=' + attr_maxlen = max([len(x) for x in self.attr] + [5]) + res_str += f'\n{prefix}ID: height width ratio {" " * max(0, attr_maxlen - 4)}count h/16 w/16 tokens\n{prefix}' + res_str += ('\n' + prefix).join([f'{i:2d}: ({x.h:4d}, {x.w:4d}) {self.ratio[i]:.4f} {self.attr[i]:>{attr_maxlen}s} ' + f'({x.h // 16:3d}, {x.w // 16:3d}) {x.h // 16 * x.w // 16:6d}' + for i, x in enumerate(self.data)]) + res_str += f'\n{prefix_close})' + return res_str + + @staticmethod + def from_list_of_hxw(hxw_list): + data = [Resolution(x) for x in hxw_list] + data = sorted(data, key=lambda x: x.ratio) + return ResolutionGroup(None, data=data) + + def calc(self, target_ratios=None): + if target_ratios is None: + return self._calc_by_step() + else: + return self._calc_by_ratio(target_ratios) + + def _calc_by_ratio(self, target_ratios): + resolutions = [] + for ratio in target_ratios: + if ratio == '1:1': + reso = Resolution(self.base_size, self.base_size) + else: + hr, wr = map(int, ratio.split(':')) + x = int((self.base_size ** 2 * self.enlarge // self.align // self.align / (hr * wr)) ** 0.5) + height = x * hr * self.align + width = x * wr * self.align + reso = Resolution(height, width) + resolutions.append(reso) + + resolutions = sorted(resolutions, key=lambda x_: x_.ratio) + + return resolutions + + def _calc_by_step(self): + min_height = self.base_size // 2 + min_width = self.base_size // 2 + max_height = self.base_size * 2 + max_width = self.base_size * 2 + + resolutions = [Resolution(self.base_size, self.base_size)] + + cur_height, cur_width = self.base_size, self.base_size + while True: + if cur_height >= max_height and cur_width <= min_width: + break + + cur_height = min(cur_height + self.step, max_height) + cur_width = max(cur_width - self.step, min_width) + resolutions.append(Resolution(cur_height, cur_width)) + + cur_height, cur_width = self.base_size, self.base_size + while True: + if cur_height <= min_height and cur_width >= max_width: + break + + cur_height = max(cur_height - self.step, min_height) + cur_width = min(cur_width + self.step, max_width) + resolutions.append(Resolution(cur_height, cur_width)) + + resolutions = sorted(resolutions, key=lambda x: x.ratio) + + return resolutions + + +class Bucket(ArrowIndexV2): + def __init__(self, height, width, *args, **kwargs): + super().__init__(*args, **kwargs) + self.height = height + self.width = width + self.ratio = height / width + self.scale_dist = [] + + def get_scale_by_index(self, index, size_col='hw', shadow=None): + """ + Calculate the scale to resize the image to fit the bucket. + + Parameters + ---------- + index: int + An in-json index. + size_col: str + How to get the size of the image. 'hw' for height and width column, + while 'image' for decoding image binary and get the PIL Image size. + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + scale: float + """ + if size_col == 'hw': + w = int(self.get_attribute_by_index(index, 'width', shadow=shadow)) + h = int(self.get_attribute_by_index(index, 'height', shadow=shadow)) + else: + w, h = self.get_image_by_index(index, shadow=shadow).size + tw, th = self.width, self.height + + tr = th / tw + r = h / w + + scale = th / h if r < tr else tw / w + return scale + + @staticmethod + def from_bucket_index(index_file, align=1, shadow_file_fn=None): + with open(index_file, 'r') as f: + res_dict = json.load(f) + + if not isinstance(res_dict['group_length'], dict): + error_msg = f'`group_length` must be a dict, but got {type(res_dict["group_length"])}' + if isinstance(res_dict['group_length'], list): + raise ValueError(f'{error_msg}\nYou may using a vanilla Index V2 file. Try `ArrowIndexV2` instead.') + else: + raise ValueError(error_msg) + + assert 'indices_file' in res_dict, f'indices_file not found in {index_file}' + assert res_dict['indices_file'] != '', f'indices_file is empty in {index_file}' + + indices_file = Path(index_file).parent / res_dict['indices_file'] + assert Path(indices_file).exists(), f'indices_file {indices_file} not found' + + # Loading indices data + indices_data = np.load(indices_file) + + # Build buckets + buckets = [] + keys = [] + for k, v in indices_data.items(): + data = { + 'data_type': res_dict['data_type'], + 'arrow_files': res_dict['arrow_files'], + 'cum_length': res_dict['cum_length'], + } + + data['indices_file'] = '' + data['indices'] = v + data['group_length'] = res_dict['group_length'][k] + + height, width = map(int, k.split('x')) + bucket = Bucket(height, width, res_dict=data, align=align, shadow_file_fn=shadow_file_fn) + + if len(bucket) > 0: + buckets.append(bucket) + keys.append(k) + + resolutions = ResolutionGroup.from_list_of_hxw(keys) + resolutions.attr = [f'{len(bucket):,d}' for bucket in buckets] + + return buckets, resolutions + + +class MultiIndexV2(object): + """ + Multi-bucket index. Support multi-GPU (either single node or multi-node distributed) training. + + Parameters + ---------- + index_files: list + The index files. + batch_size: int + The batch size of each GPU. Required when using MultiResolutionBucketIndexV2 as base index class. + world_size: int + The number of GPUs. Required when using MultiResolutionBucketIndexV2 as base index class. + sample_strategy: str + The sample strategy. Can be 'uniform' or 'probability'. Default to 'uniform'. + If set to probability, a list of probability must be provided. The length of the list must be the same + as the number of buckets. Each probability value means the sample rate of the corresponding bucket. + probability: list + A list of probability. Only used when sample_strategy=='probability'. + shadow_file_fn: callable or dict + A callable function to map shadow file path to a new path. If None, the shadow file path will not be + changed. If a dict is provided, the keys are the shadow names to call the function, and the values are the + callable functions to map the shadow file path to a new path. If a callable function is provided, the key + is 'default'. + seed: int + Only used when sample_strategy=='probability'. The seed to sample the indices. + """ + buckets: List[ArrowIndexV2] + + def __init__(self, + index_files: List[str], + batch_size: Optional[int] = None, + world_size: Optional[int] = None, + sample_strategy: str = 'uniform', + probability: Optional[List[float]] = None, + shadow_file_fn: Optional[Union[Callable, Dict[str, Callable]]] = None, + seed: Optional[int] = None, + ): + self.buckets = self.load_buckets(index_files, + batch_size=batch_size, world_size=world_size, shadow_file_fn=shadow_file_fn) + + self.sample_strategy = sample_strategy + self.probability = probability + self.check_sample_strategy(sample_strategy, probability) + + self.cum_length = self.calc_cum_length() + + self.sampler = np.random.RandomState(seed) + if sample_strategy == 'uniform': + self.total_length = sum([len(bucket) for bucket in self.buckets]) + self.ind_mapper = np.arange(self.total_length) + elif sample_strategy == 'probability': + self.ind_mapper = self.sample_indices_with_probability() + self.total_length = len(self.ind_mapper) + else: + raise ValueError(f"Not supported sample_strategy {sample_strategy}.") + + def load_buckets(self, index_files, **kwargs): + buckets = [ArrowIndexV2(index_file, **kwargs) for index_file in index_files] + return buckets + + def __len__(self): + return self.total_length + + def check_sample_strategy(self, sample_strategy, probability): + if sample_strategy == 'uniform': + pass + elif sample_strategy == 'probability': + if probability is None: + raise ValueError(f"probability must be provided when sample_strategy is 'probability'.") + assert isinstance(probability, (list, tuple)), \ + f"probability must be a list, but got {type(probability)}" + assert len(self.buckets) == len(probability), \ + f"Length of index_files {len(self.buckets)} != Length of probability {len(probability)}" + else: + raise ValueError(f"Not supported sample_strategy {sample_strategy}.") + + def sample_indices_with_probability(self): + ind_mapper_list = [] + accu = 0 + for bucket, p in zip(self.buckets, self.probability): + if p == 1: + # Just use all indices + indices = np.arange(len(bucket)) + accu + else: + # Use all indices multiple times, and then sample some indices without replacement + repeat_times = int(p) + indices_part1 = np.arange(len(bucket)).repeat(repeat_times) + indices_part2 = self.sampler.choice(len(bucket), int(len(bucket) * (p - repeat_times)), replace=False) + indices = np.sort(np.concatenate([indices_part1, indices_part2])) + accu + ind_mapper_list.append(indices) + accu += len(bucket) + ind_mapper = np.concatenate(ind_mapper_list) + return ind_mapper + + def calc_cum_length(self): + cum_length = [] + length = 0 + for bucket in self.buckets: + length += len(bucket) + cum_length.append(length) + return cum_length + + def shuffle(self, seed=None, fast=False): + if self.sample_strategy == 'probability': + # Notice: In order to resample indices when shuffling, shuffle will not preserve the + # initial sampled indices when loading the index. + pass + + # Shuffle indexes + if seed is not None: + state = random.getstate() + random.seed(seed) + random.shuffle(self.buckets) + random.setstate(state) + else: + random.shuffle(self.buckets) + + self.cum_length = self.calc_cum_length() + + # Shuffle indices in each index + for i, bucket in enumerate(self.buckets): + bucket.shuffle(seed + i, fast=fast) + + # Shuffle ind_mapper + if self.sample_strategy == 'uniform': + self.ind_mapper = np.arange(self.total_length) + elif self.sample_strategy == 'probability': + self.ind_mapper = self.sample_indices_with_probability() + else: + raise ValueError(f"Not supported sample_strategy {self.sample_strategy}.") + if seed is not None: + sampler = np.random.RandomState(seed) + sampler.shuffle(self.ind_mapper) + else: + np.random.shuffle(self.ind_mapper) + + def get_arrow_file(self, ind, **kwargs): + """ + Get arrow file by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + kwargs: dict + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + arrow_file: str + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_arrow_file(ind - bias, **kwargs) + + def get_data(self, ind, columns=None, allow_missing=False, return_meta=True, **kwargs): + """ + Get data by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + columns: str or list + The columns to be returned. If None, return all columns. + allow_missing: bool + If True, omit missing columns. If False, raise an error if the column is missing. + return_meta: bool + If True, the resulting dict will contain some meta information: + in-json index, in-arrow index, and arrow_name. + kwargs: dict + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + data: dict + A dict containing the data. + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_data(ind - bias, columns=columns, allow_missing=allow_missing, + return_meta=return_meta, **kwargs) + + def get_attribute(self, ind, column, **kwargs): + """ + Get single attribute by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + column: str + The column name. + kwargs: dict + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + attribute: Any + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_attribute(ind - bias, column, **kwargs) + + def get_image(self, ind, column='image', ret_type='pil', max_size=-1, **kwargs): + """ + Get image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + column: str + [Deprecated] The column name of the image. Default to 'image'. + ret_type: str + The return type. Can be 'pil' or 'numpy'. Default to 'pil'. + max_size: int + If not -1, resize the image to max_size. max_size is the size of long edge. + kwargs: dict + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + image: PIL.Image.Image or np.ndarray + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_image(ind - bias, column, ret_type, max_size, **kwargs) + + def get_md5(self, ind, **kwargs): + """ Get md5 by in-dataset index. """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_md5(ind - bias, **kwargs) + + def get_columns(self, ind, **kwargs): + """ Get columns by in-dataset index. """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_columns(ind - bias, **kwargs) + + @staticmethod + def resize_and_crop(image, target_size, resample=Image.LANCZOS, crop_type='random'): + """ + Resize image without changing aspect ratio, then crop the center/random part. + + Parameters + ---------- + image: PIL.Image.Image + The input image to be resized and cropped. + target_size: tuple + The target size of the image. + resample: + The resample method. See PIL.Image.Image.resize for details. Default to Image.LANCZOS. + crop_type: str + 'center' or 'random'. If 'center', crop the center part of the image. If 'random', + crop a random part of the image. Default to 'random'. + + Returns + ------- + image: PIL.Image.Image + The resized and cropped image. + crop_pos: tuple + The position of the cropped part. (crop_left, crop_top) + """ + return ArrowIndexV2.resize_and_crop(image, target_size, resample, crop_type) + + +class MultiResolutionBucketIndexV2(MultiIndexV2): + """ + Multi-resolution bucket index. Support multi-GPU (either single node or multi-node distributed) training. + + Parameters + ---------- + index_file: str + The index file of the bucket index. + batch_size: int + The batch size of each GPU. + world_size: int + The number of GPUs. + shadow_file_fn: callable or dict + A callable function to map shadow file path to a new path. If None, the shadow file path will not be + changed. If a dict is provided, the keys are the shadow names to call the function, and the values are the + callable functions to map the shadow file path to a new path. If a callable function is provided, the key + is 'default'. + """ + buckets: List[Bucket] + + def __init__(self, + index_file: str, + batch_size: int, + world_size: int, + shadow_file_fn: Optional[Union[Callable, Dict[str, Callable]]] = None, + ): + align = batch_size * world_size + if align <= 0: + raise ValueError(f'Align size must be positive, but got {align} = {batch_size} x {world_size}') + + self.buckets, self._resolutions = Bucket.from_bucket_index(index_file, + align=align, + shadow_file_fn=shadow_file_fn, + ) + self.arrow_files = self.buckets[0].arrow_files + self._base_size = self._resolutions.base_size + self._step = self._resolutions.step + + self.buckets = sorted(self.buckets, key=lambda x: x.ratio) + self.cum_length = self.calc_cum_length() + + self.total_length = sum([len(bucket) for bucket in self.buckets]) + assert self.total_length % align == 0, f'Total length {self.total_length} is not divisible by align size {align}' + + self.align_size = align + self.batch_size = batch_size + self.world_size = world_size + self.ind_mapper = np.arange(self.total_length) + + @property + def step(self): + return self._step + + @property + def base_size(self): + return self._base_size + + @property + def resolutions(self): + return self._resolutions + + def shuffle(self, seed=None, fast=False): + # Shuffle indexes + if seed is not None: + state = random.getstate() + random.seed(seed) + random.shuffle(self.buckets) + random.setstate(state) + else: + random.shuffle(self.buckets) + + self.cum_length = self.calc_cum_length() + + # Shuffle indices in each index + for i, bucket in enumerate(self.buckets): + bucket.shuffle(seed + i, fast=fast) + + # Shuffle ind_mapper + batch_ind_mapper = np.arange(self.total_length // self.batch_size) * self.batch_size + if seed is not None: + sampler = np.random.RandomState(seed) + sampler.shuffle(batch_ind_mapper) + else: + np.random.shuffle(batch_ind_mapper) + ind_mapper = np.stack([batch_ind_mapper + i for i in range(self.batch_size)], axis=1).reshape(-1) + self.ind_mapper = ind_mapper + + def get_ratio(self, ind, **kwargs): + """ + Get the ratio of the image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + + Returns + ------- + width, height, ratio + """ + ind = self.ind_mapper[ind] + width, height = self.get_image(ind, **kwargs).size + return width, height, height / width + + def get_target_size(self, ind): + """ + Get the target size of the image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + + Returns + ------- + target_width, target_height + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + return self.buckets[i].width, self.buckets[i].height + + def scale_distribution(self, save_file=None): + if save_file is not None: + scale_dict = np.load(save_file) + for bucket in self.buckets: + bucket.scale_dist = scale_dict[f'{bucket.height}x{bucket.width}'] + else: + for bucket in tqdm(self.buckets): + for index in tqdm(bucket.indices, leave=False): + scale = bucket.get_scale_by_index(index) + bucket.scale_dist.append(scale) + scale_dict = {f'{bucket.height}x{bucket.width}': bucket.scale_dist for bucket in self.buckets} + + if save_file is not None: + save_file = Path(save_file) + save_file.parent.mkdir(exist_ok=True, parents=True) + np.savez_compressed(save_file, **scale_dict) + + return self + + +class MultiMultiResolutionBucketIndexV2(MultiIndexV2): + buckets: List[MultiResolutionBucketIndexV2] + + @property + def step(self): + return [b.step for b in self.buckets] + + @property + def base_size(self): + return [b.base_size for b in self.buckets] + + @property + def resolutions(self): + return [b.resolutions for b in self.buckets] + + def load_buckets(self, index_files, **kwargs): + self.batch_size = kwargs.get('batch_size', None) + self.world_size = kwargs.get('world_size', None) + if self.batch_size is None or self.world_size is None: + raise ValueError("`batch_size` and `world_size` must be provided when using " + "`MultiMultiResolutionBucketIndexV2`.") + buckets = [ + MultiResolutionBucketIndexV2(index_file, + self.batch_size, + self.world_size, + shadow_file_fn=kwargs.get('shadow_file_fn', None), + ) + for index_file in index_files + ] + return buckets + + def sample_indices_with_probability(self, return_batch_indices=False): + bs = self.batch_size + ind_mapper_list = [] + accu = 0 + for bucket, p in zip(self.buckets, self.probability): + if p == 1: + # Just use all indices + batch_indices = np.arange(len(bucket) // bs) * bs + accu + else: + # Use all indices multiple times, and then sample some indices without replacement + repeat_times = int(p) + indices_part1 = np.arange(len(bucket) // bs).repeat(repeat_times) * bs + indices_part2 = self.sampler.choice(len(bucket) // bs, int(len(bucket) * (p / bs - repeat_times)), + replace=False) * bs + batch_indices = np.sort(np.concatenate([indices_part1, indices_part2])) + accu + + if return_batch_indices: + indices = batch_indices + else: + indices = np.stack([batch_indices + i for i in range(bs)], axis=1).reshape(-1) + ind_mapper_list.append(indices) + accu += len(bucket) + ind_mapper = np.concatenate(ind_mapper_list) + return ind_mapper + + def shuffle(self, seed=None, fast=False): + if self.sample_strategy == 'probability': + # Notice: In order to resample indices when shuffling, shuffle will not preserve the + # initial sampled indices when loading the index. + pass + + # Shuffle indexes + if seed is not None: + state = random.getstate() + random.seed(seed) + random.shuffle(self.buckets) + random.setstate(state) + else: + random.shuffle(self.buckets) + + self.cum_length = self.calc_cum_length() + + # Shuffle indices in each index + for i, bucket in enumerate(self.buckets): + bucket.shuffle(seed + i, fast=fast) + + # Shuffle ind_mapper in batch level + if self.sample_strategy == 'uniform': + batch_ind_mapper = np.arange(self.total_length // self.batch_size) * self.batch_size + elif self.sample_strategy == 'probability': + batch_ind_mapper = self.sample_indices_with_probability(return_batch_indices=True) + else: + raise ValueError(f"Not supported sample_strategy {self.sample_strategy}.") + if seed is not None: + sampler = np.random.RandomState(seed) + sampler.shuffle(batch_ind_mapper) + else: + np.random.shuffle(batch_ind_mapper) + self.ind_mapper = np.stack([batch_ind_mapper + i for i in range(self.batch_size)], axis=1).reshape(-1) + + def get_ratio(self, ind, **kwargs): + """ + Get the ratio of the image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + + Returns + ------- + width, height, ratio + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_ratio(ind - bias, **kwargs) + + def get_target_size(self, ind): + """ + Get the target size of the image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + + Returns + ------- + target_width, target_height + """ + ind = self.ind_mapper[ind] + i = bisect.bisect_right(self.cum_length, ind) + bias = self.cum_length[i - 1] if i > 0 else 0 + return self.buckets[i].get_target_size(ind - bias) + + +def build_multi_resolution_bucket(config_file, + base_size, + src_index_files, + save_file, + reso_step=64, + target_ratios=None, + align=1, + min_size=0, + md5_hw=None, + ): + # Compute base size + resolutions = ResolutionGroup(base_size, step=reso_step, target_ratios=target_ratios, align=align) + print(resolutions) + + save_file = Path(save_file) + save_file.parent.mkdir(exist_ok=True, parents=True) + + if isinstance(src_index_files, str): + src_index_files = [src_index_files] + src_indexes = [] + print(f'Loading indexes:') + for src_index_file in src_index_files: + src_indexes.append(ArrowIndexV2(src_index_file)) + print(f' {src_index_file} | cum_length: {src_indexes[-1].cum_length[-1]} | indices: {len(src_indexes[-1])}') + + if md5_hw is None: + md5_hw = {} + + arrow_files = src_indexes[0].arrow_files[:] # !!!important!!!, copy the list + for src_index in src_indexes[1:]: + arrow_files.extend(src_index.arrow_files[:]) + + cum_length = src_indexes[0].cum_length[:] + for src_index in src_indexes[1:]: + cum_length.extend([x + cum_length[-1] for x in src_index.cum_length]) + print(f'cum_length: {cum_length[-1]}') + + group_length_list = src_indexes[0].group_length[:] + for src_index in src_indexes[1:]: + group_length_list.extend(src_index.group_length[:]) + + total_indices = sum([len(src_index) for src_index in src_indexes]) + total_group_length = sum(group_length_list) + assert total_indices == total_group_length, f'Total indices {total_indices} != Total group length {total_group_length}' + + buckets = [[] for _ in range(len(resolutions))] + cum_length_tmp = 0 + total_index_count = 0 + for src_index, src_index_file in zip(src_indexes, src_index_files): + index_count = 0 + pbar = tqdm(src_index.indices.tolist()) + for i in pbar: + try: + height = int(src_index.get_attribute_by_index(i, 'height')) + width = int(src_index.get_attribute_by_index(i, 'width')) + except Exception as e1: + try: + md5 = src_index.get_attribute_by_index(i, 'md5') + height, width = md5_hw[md5] + except Exception as e2: + try: + width, height = src_index.get_image_by_index(i).size + except Exception as e3: + print(f'Error: {e1} --> {e2} --> {e3}. We will skip this image.') + continue + + if height < min_size or width < min_size: + continue + + ratio = height / width + idx = np.argmin(np.abs(resolutions.ratio - ratio)) + buckets[idx].append(i + cum_length_tmp) + index_count += 1 + print(f"Valid indices {index_count} in {src_index_file}.") + cum_length_tmp += src_index.cum_length[-1] + total_index_count += index_count + print(f'Total indices: {total_index_count}') + + print(f'Making bucket index.') + indices = {} + for i, bucket in tqdm(enumerate(buckets)): + if len(bucket) == 0: + continue + reso = f'{resolutions[i]}' + resolutions.attr[i] = f'{len(bucket):>6d}' + indices[reso] = bucket + + builder = IndexV2Builder(data_type=['multi-resolution-bucket-v2', + f'base_size={base_size}', + f'reso_step={reso_step}', + f'target_ratios={target_ratios}', + f'align={align}', + f'min_size={min_size}', + f'src_files='] + + [f'{src_index_file}' for src_index_file in src_index_files], + arrow_files=arrow_files, + cum_length=cum_length, + indices=indices, + config_file=config_file, + ) + builder.build(save_file) + print(resolutions) + print(f'Build index finished!\n\n' + f' Save path: {Path(save_file).absolute()}\n' + f' Number of indices: {sum([len(v) for k, v in indices.items()])}\n' + f'Number of arrow files: {len(arrow_files)}\n' + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/common.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/common.py new file mode 100644 index 0000000000000000000000000000000000000000..998184f54fefe941c4671dfe27d8fb26514a3a8d --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/common.py @@ -0,0 +1,262 @@ +import json +import numpy as np +from pathlib import Path +from functools import partial + +from .indexer import ArrowIndexV2 +from .bucket import ( + ResolutionGroup, + MultiIndexV2, MultiResolutionBucketIndexV2, MultiMultiResolutionBucketIndexV2, IndexV2Builder +) + + +def load_index(src, + multireso=False, + batch_size=1, + world_size=1, + sample_strategy='uniform', + probability=None, + shadow_file_fn=None, + seed=None, + ): + if isinstance(src, str): + src = [src] + if src[0].endswith('.arrow'): + if multireso: + raise ValueError('Arrow file does not support multiresolution. Please make base index V2 first and then' + 'build multiresolution index.') + idx = IndexV2Builder(src).to_index_v2() + elif src[0].endswith('.json'): + if multireso: + if len(src) == 1: + idx = MultiResolutionBucketIndexV2(src[0], batch_size=batch_size, + world_size=world_size, + shadow_file_fn=shadow_file_fn, + ) + else: + idx = MultiMultiResolutionBucketIndexV2(src, batch_size=batch_size, + world_size=world_size, + sample_strategy=sample_strategy, probability=probability, + shadow_file_fn=shadow_file_fn, seed=seed, + ) + else: + if len(src) == 1: + idx = ArrowIndexV2(src[0], + shadow_file_fn=shadow_file_fn, + ) + else: + idx = MultiIndexV2(src, + sample_strategy=sample_strategy, probability=probability, + shadow_file_fn=shadow_file_fn, seed=seed, + ) + else: + raise ValueError(f'Unknown file type: {src[0]}') + return idx + + +def get_attribute(data, attr_list): + ret_data = {} + for attr in attr_list: + ret_data[attr] = data.get(attr, None) + if ret_data[attr] is None: + raise ValueError(f'Missing {attr} in {data}') + return ret_data + + +def get_optional_attribute(data, attr_list): + ret_data = {} + for attr in attr_list: + ret_data[attr] = data.get(attr, None) + return ret_data + + +def detect_index_type(data): + if isinstance(data['group_length'], dict): + return 'multireso' + else: + return 'base' + +def show_index_info(src, only_arrow_files=False, depth=1): + """ + Show base/multireso index information. + """ + if not Path(src).exists(): + raise ValueError(f'{src} does not exist.') + print(f"Loading index file {src} ...") + with open(src, 'r') as f: + src_data = json.load(f) + print(f"Loaded.") + data = get_attribute(src_data, ['data_type', 'indices_file', 'arrow_files', 'cum_length', + 'group_length', 'indices', 'example_indices']) + opt_data = get_optional_attribute(src_data, ['config_file']) + + # Format arrow_files examples + arrow_files = data['arrow_files'] + if only_arrow_files: + existed = set() + arrow_files_output_list = [] + for arrow_file in arrow_files: + if depth == 0: + if arrow_file not in existed: + arrow_files_output_list.append(arrow_file) + existed.add(arrow_file) + elif depth > 0: + parts = Path(arrow_file).parts + if depth >= len(parts): + continue + else: + arrow_file_part = '/'.join(parts[:-depth]) + if arrow_file_part not in existed: + arrow_files_output_list.append(arrow_file_part) + existed.add(arrow_file_part) + else: + raise ValueError(f'Depth {depth} has exceeded the limit of arrow file {arrow_file}.') + arrow_files_repr = '\n'.join(arrow_files_output_list) + print(arrow_files_repr) + return None + + return_space = '\n' + ' ' * 21 + + if len(arrow_files) <= 4: + arrow_files_repr = return_space.join([arrow_file for arrow_file in arrow_files]) + else: + arrow_files_repr = return_space.join([_ for _ in arrow_files[:2]] + ['...'] + + [_ for _ in arrow_files[-2:]]) + arrow_files_length = len(arrow_files) + + # Format data_type + data_type = data['data_type'] + if isinstance(data_type, str): + data_type = [data_type] + data_type_common = [] + src_files = [] + found_src_files = False + for data_type_item in data_type: + if not found_src_files and data_type_item.strip() != 'src_files=': + data_type_common.append(data_type_item.strip()) + continue + found_src_files = True + if data_type_item.endswith('.json'): + src_files.append(data_type_item.strip()) + else: + data_type_common.append(data_type_item.strip()) + data_type_part2_with_ids = [] + max_id_len = len(str(len(src_files))) + for sid, data_type_item in enumerate(src_files, start=1): + data_type_part2_with_ids.append(f'{str(sid).rjust(max_id_len)}. {data_type_item}') + data_type = data_type_common + data_type_part2_with_ids + data_repr = return_space.join(data_type) + + # Format cum_length examples + cum_length = data['cum_length'] + if len(cum_length) <= 8: + cum_length_repr = ', '.join([str(i) for i in cum_length]) + else: + cum_length_repr = ', '.join([str(i) for i in cum_length[:4]] + ['...'] + [str(i) for i in cum_length[-4:]]) + cum_length_length = len(cum_length) + + if detect_index_type(data) == 'base': + # Format group_length examples + group_length = data['group_length'] + if len(group_length) <= 8: + group_length_repr = ', '.join([str(i) for i in group_length]) + else: + group_length_repr = ', '.join([str(i) for i in group_length[:4]] + ['...'] + [str(i) for i in group_length[-4:]]) + group_length_length = len(group_length) + + # Format indices examples + indices = data['indices'] + if len(indices) == 0 and data['indices_file'] != '': + indices_file = Path(src).parent / data['indices_file'] + if Path(indices_file).exists(): + print(f"Loading indices from {indices_file} ...") + indices = np.load(indices_file)['x'] + print(f"Loaded.") + else: + raise ValueError(f'This Index file contains an extra file {indices_file} which is missed.') + if len(indices) <= 8: + indices_repr = ', '.join([str(i) for i in indices]) + else: + indices_repr = ', '.join([str(i) for i in indices[:4]] + ['...'] + [str(i) for i in indices[-4:]]) + + # Calculate indices total length + indices_length = len(indices) + + print_str = f"""File: {Path(src).absolute()} + +ArrowIndexV2( + \033[4mdata_type:\033[0m {data_repr}""" + + # Process optional data + if opt_data['config_file'] is not None: + print_str += f""" + \033[4mconfig_file:\033[0m {opt_data['config_file']}""" + + # Add common data + print_str += f""" + \033[4mindices_file:\033[0m {data['indices_file']} + \033[4marrow_files: Count = {arrow_files_length:,}\033[0m + Examples: {arrow_files_repr} + \033[4mcum_length: Count = {cum_length_length:,}\033[0m + Examples: {cum_length_repr} + \033[4mgroup_length: Count = {group_length_length:,}\033[0m + Examples: {group_length_repr} + \033[4mindices: Count = {indices_length:,}\033[0m + Examples: {indices_repr}""" + + else: + group_length = data['group_length'] + + indices_file = Path(src).parent / data['indices_file'] + assert Path(indices_file).exists(), f'indices_file {indices_file} not found' + print(f"Loading indices from {indices_file} ...") + indices_data = np.load(indices_file) + print(f"Loaded.") + indices_length = sum([len(indices) for key, indices in indices_data.items()]) + keys = [k for k in group_length.keys() if len(indices_data[k]) > 0] + + resolutions = ResolutionGroup.from_list_of_hxw(keys) + resolutions.attr = [f'{len(indices):>,d}' for k, indices in indices_data.items()] + resolutions.prefix_space = 25 + + print_str = f"""File: {Path(src).absolute()} + +MultiResolutionBucketIndexV2( + \033[4mdata_type:\033[0m {data_repr}""" + + # Process optional data + if opt_data['config_file'] is not None: + print_str += f""" + \033[4mconfig_file:\033[0m {opt_data['config_file']}""" + + # Process config files of base index files + config_files = [] + for src_file in src_files: + src_file = Path(src_file) + if src_file.exists(): + with src_file.open() as f: + base_data = json.load(f) + if 'config_file' in base_data: + config_files.append(base_data['config_file']) + else: + config_files.append('Unknown') + else: + config_files.append('Missing the src file') + if config_files: + config_file_str = return_space.join([f'{str(sid).rjust(max_id_len)}. {config_file}' + for sid, config_file in enumerate(config_files, start=1)]) + print_str += f""" + \033[4mbase config files:\033[0m {config_file_str}""" + + # Add common data + print_str += f""" + \033[4mindices_file:\033[0m {data['indices_file']} + \033[4marrow_files: Count = {arrow_files_length:,}\033[0m + Examples: {arrow_files_repr} + \033[4mcum_length: Count = {cum_length_length:,}\033[0m + Examples: {cum_length_repr} + \033[4mindices: Count = {indices_length:,}\033[0m + \033[4mbuckets: Count = {len(keys)}\033[0m + {resolutions}""" + + print(print_str + '\n)\n') diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/config_parse.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/config_parse.py new file mode 100644 index 0000000000000000000000000000000000000000..11e345cefa9c493c2c17f4394b15d1076efa0335 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/config_parse.py @@ -0,0 +1,777 @@ +import yaml +from glob import glob +import pathlib +from pathlib import Path +import pickle +import json +from typing import List +from collections import defaultdict + +def source_names_to_names(source_names_): + """ + Get all arrow file names from a list of source names. + + Examples: + `test.arrow@@x2` means repeat the arrow file `test.arrow` 2 times. + """ + names = [] + exclude_count = 0 + for data_name in source_names_: + if isinstance(data_name, dict): + data_name, value = list(data_name.items())[0] + exclude = value.get('exclude', []) + repeat_times = value.get('repeat', 1) + else: + exclude = [] + repeat_times = 1 + + cur_count = 0 + for name in glob(data_name): + for e in exclude: + if e in name: + print(f"Excluding {name}") + exclude_count += 1 + break + else: + names.append((name, repeat_times)) + cur_count += 1 + print(f"Found {cur_count} arrow files in {data_name}.") + names = list(sorted(names, key=lambda x: x[0])) + return names, exclude_count + + +def get_or_create_arrow_name_list(source_names=None): + + if not isinstance(source_names, list): + raise ValueError(f"`source_names` should be of type list, got {type(source_names)}") + if len(source_names) == 0: + raise ValueError(f"`source_names` is an empty list.") + names, exclude_count = source_names_to_names(source_names) + + print(f"Found {len(names):,} arrow files.") + print(f"Excluded {exclude_count} arrow files.") + return names + + +def to_numeric(x, default_val=0.0): + if isinstance(x, (float, int)): + pass + elif isinstance(x, str): + try: + x = float(x) + except ValueError: + x = default_val + else: + x = default_val + return x + + +TYPE_MAPPER = { + "int": int, + "float": float, + "str": str, + "dict": dict, + "list": list, +} + + +class Operator(object): + def __init__(self, config): + self.config = config + self.arrow_file_keyword = None + + def applicable(self, arrow_file): + # If no keyword is provided, then it is applicable to all arrow files + if self.arrow_file_keyword is None: + return True + # If keyword is provided, then it is applicable to the arrow file if the keyword is in the arrow file name + for keyword in self.arrow_file_keyword: + if keyword in arrow_file: + return True + return False + + @staticmethod + def parse_path(paths): + if isinstance(paths, str): + paths = [paths] + if not isinstance(paths, list): + raise ValueError(f"Invalid type: {type(paths)}") + paths = [Path(p) for p in paths] + return paths + + @staticmethod + def load_md5s(paths: List[pathlib.Path], merge=True): + md5s_list = [] + for path in paths: + if not path.exists(): + raise ValueError(f"Path not found: {path}") + if path.suffix == '.pkl': + with path.open('rb') as f: + md5s = pickle.load(f) + assert isinstance(md5s, (set, dict)), f"Invalid type: {type(md5s)}" + elif path.suffix == '.json': + with path.open() as f: + md5s = json.load(f) + assert isinstance(md5s, dict), f"Invalid type: {type(md5s)}" + elif path.suffix == '.txt': + md5s = set(path.read_text().splitlines()) + else: + raise ValueError(f"Invalid file type: {path}") + md5s_list.append(md5s) + print(f" {path}: {len(md5s):,} md5s") + + if merge: + md5s = md5s_list[0] + for m in md5s_list[1:]: + md5s.update(m) + else: + md5s = md5s_list + return md5s + + +class ColumnFilter(Operator): + numeric_actions = {"eq", "ne", "gt", "lt", "ge", "le"} + str_actions = {"eq", "ne", "len_eq", "len_ne", "len_gt", "len_lt", "len_ge", "len_le", "contains", "not_contains", + "in", "not_in", "lower_last_in"} + int_target_str_actions = {"len_eq", "len_gt", "len_lt", "len_ge", "len_le"} + action_mapper_left = { + "eq": "==", "ne": "!=", "len_eq": ".len()==", + "gt": ">", "len_gt": ".len()>", + "lt": "<", "len_lt": ".len()<", + "ge": ">=", "len_ge": ".len()>=", + "le": "<=", "len_le": ".len()<=", + "contains": ".contains(", + "not_contains": ".not_contains(", + "in": ".isin(", "not_in": ".notin(", + "lower_last_in": "[-1].lower().isin(", + } + action_mapper_right = { + "eq": "", "ne": "", "len_eq": "", + "gt": "", "len_gt": "", + "lt": "", "len_lt": "", + "ge": "", "len_ge": "", + "le": "", "len_le": "", + "contains": ")", + "not_contains": ")", + "in": ")", "not_in": ")", + "lower_last_in": ")", + } + + def __init__(self, config): + super().__init__(config) + self.name = self.column_name = config['name'] + self.dtype = config['type'] + self.action = config['action'] + self.target = config['target'] + self.default = config['default'] + self.arrow_file_cond = config.get('arrow_file', None) + self.arrow_file_keyword = config.get('arrow_file_keyword', None) + if self.arrow_file_keyword is not None: + if isinstance(self.arrow_file_keyword, str): + self.arrow_file_keyword = [self.arrow_file_keyword] + + def check_exists(self, arrow_file, table): + status = True + if self.column_name not in table.column_names: + print(f"Warning: Column `{self.column_name}` not found in {arrow_file}.") + status = status and False + return status + + @property + def data_type(self): + if self.arrow_file_keyword is None or len(self.arrow_file_keyword) == 0: + arrow_file_keyword_repr = "" + else: + arrow_file_keyword_repr = f', arrow_file_keys={self.arrow_file_keyword}' + fmt_str = f"{self.column_name}" \ + f"{self.action_mapper_left[self.action]}{self.target}{self.action_mapper_right[self.action]}" \ + f" (default={self.default}{arrow_file_keyword_repr})" + return fmt_str + + +class NumericFilter(ColumnFilter): + def run_on_column(self, column): + if self.action == 'eq': + return column == self.target + elif self.action == 'ne': + return column != self.target + elif self.action == 'gt': + return column > self.target + elif self.action == 'lt': + return column < self.target + elif self.action == 'ge': + return column >= self.target + elif self.action == 'le': + return column <= self.target + else: + raise ValueError(f"Invalid action: {self.action}") + + +class IntFilter(NumericFilter): + def __call__(self, arrow_file, table): + success = self.check_exists(arrow_file, table) + if not success: + return None + column = table[self.column_name].to_pandas().apply(lambda x: to_numeric(x, self.default)) + return self.run_on_column(column) + + +class FloatFilter(NumericFilter): + def __call__(self, arrow_file, table): + success = self.check_exists(arrow_file, table) + if not success: + return None + column = table[self.column_name].to_pandas().apply(lambda x: to_numeric(x, self.default)) + return self.run_on_column(column) + + +class StrFilter(ColumnFilter): + def __call__(self, arrow_file, table): + success = self.check_exists(arrow_file, table) + if not success: + return None + column = table[self.column_name].to_pandas() + if self.action == 'eq': + return column.apply(lambda x: x == self.target) + elif self.action == 'ne': + return column.apply(lambda x: x != self.target) + elif self.action == 'len_eq': + return column.str.len() == self.target + elif self.action == 'len_ne': + return column.str.len() != self.target + elif self.action == 'len_gt': + return column.str.len() > self.target + elif self.action == 'len_lt': + return column.str.len() < self.target + elif self.action == 'len_ge': + return column.str.len() >= self.target + elif self.action == 'len_le': + return column.str.len() <= self.target + elif self.action == 'contains': + return column.str.contains(self.target) + elif self.action == 'not_contains': + return ~column.str.contains(self.target) + elif self.action == 'in': + return column.apply(lambda x: x in self.target) + elif self.action == 'not_in': + return column.apply(lambda x: x not in self.target) + elif self.action == 'lower_last_in': + return column.apply(lambda x: x[-1].lower() in self.target) + else: + raise ValueError(f"Invalid action: {self.action}") + + +def check_type(parent, config): + if 'type' not in config: + return False, f"Missing required argument: {parent}.type" + if not isinstance(config['type'], str): + return False, f"Argument {parent}.type should be of type str" + if config['type'] not in TYPE_MAPPER: + return False, f"Invalid type: {parent}.type: {config['type']}" + return True, "" + + +def check_keys(parent, config, required_args, types): + for arg, ts in zip(required_args, types): + # Check key exists + if arg not in config: + return False, f"Missing required argument: {parent}.{arg}" + # Check values' type + if ts is not None: + if not isinstance(ts, list): + ts = [ts] + for t in ts: + if not isinstance(config[arg], t): + return False, f"Argument {parent}.{arg} should be of type {t}, got {type(config[arg])}" + return True, "" + + +def get_column_filter(parent, config): + success, msg = check_type(parent, config) + if not success: + return False, msg, None + + if config['type'] == 'str' and config['action'] in ColumnFilter.int_target_str_actions: + dtype = TYPE_MAPPER['int'] + else: + dtype = TYPE_MAPPER[config['type']] + + # Check other keys + required_args = ['name', 'action', 'target', 'default'] + types = [str, str, dtype, dtype] + success, msg = check_keys(parent, config, required_args, types) + if not success: + return False, msg, None + + # Check action values + if config['type'] == 'str': + valid_actions = ColumnFilter.str_actions + else: + valid_actions = ColumnFilter.numeric_actions + if config['action'] not in valid_actions: + return False, f"Invalid action: {parent}.action: {config['action']}", None + + if config['type'] == 'int': + return True, "", IntFilter(config) + elif config['type'] == 'float': + return True, "", FloatFilter(config) + elif config['type'] == 'str': + return True, "", StrFilter(config) + else: + raise ValueError(f"Invalid type: {config['type']}") + + +class MD5Filter(Operator): + valid_actions = { + "list": {"in", "not_in"}, + "dict": {"eq", "ne", "gt", "lt", "ge", "le"}, + } + action_mapper_left = { + "eq": "==", "ne": "!=", + "gt": ">", "lt": "<", + "ge": ">=", "le": "<=", + "in": ".isin(", "not_in": ".isin(", + } + action_mapper_right = { + "eq": "", "ne": "", + "gt": "", "lt": "", + "ge": "", "le": "", + "in": ")", "not_in": ")", + } + + def __init__(self, config): + super().__init__(config) + self.name = config['name'] + self.paths = self.parse_path(config['path']) + self.dtype = config['type'] + self.action = config['action'] + self.target = config.get('target', None) # only for type=='dict' + self.is_valid = config.get('is_valid', None) # only for type=='dict' + self.arrow_file_keyword = config.get('arrow_file_keyword', None) + if self.arrow_file_keyword is not None: + if isinstance(self.arrow_file_keyword, str): + self.arrow_file_keyword = [self.arrow_file_keyword] + + self.md5_data = self.load_md5s(self.paths) + + @property + def data_type(self): + if self.arrow_file_keyword is None or len(self.arrow_file_keyword) == 0: + arrow_file_keyword_repr = "" + else: + arrow_file_keyword_repr = f'(arrow_file_keys={self.arrow_file_keyword})' + return_space = '\n' + ' ' * 22 + if self.dtype == 'list': + fmt_str = ("Good Cases" if self.is_valid else " Bad Cases") \ + + f" (md5): {return_space.join([str(p) for p in self.paths])} " \ + f"{arrow_file_keyword_repr} " \ + f"| {self.name}" + elif self.dtype == 'dict': + fmt_str = ("Good Cases" if self.is_valid else " Bad Cases") \ + + f" (md5): {return_space.join([str(p) for p in self.paths])}\n" \ + f" --> value" \ + f"{self.action_mapper_left[self.action]}{self.target}{self.action_mapper_right[self.action]} " \ + f"{arrow_file_keyword_repr} " \ + f"| {self.name}" + else: + raise ValueError(f"Invalid type: {self.dtype}") + return fmt_str + + +class ListFilter(MD5Filter): + def __call__(self, md5s): + if self.action == "in": + find_valid = self.is_valid + elif self.action == "not_in": + find_valid = not self.is_valid + else: + raise ValueError(f"Invalid action: {self.action}") + + if find_valid: + return md5s.apply(lambda x: x in self.md5_data) + else: + return md5s.apply(lambda x: x not in self.md5_data) + + +class DictFilter(MD5Filter): + def __call__(self, md5s): + if self.is_valid: + return md5s.apply(lambda x: x in self.md5_data and self.cond(self.md5_data[x])) + else: + return md5s.apply(lambda x: (x not in self.md5_data) or not self.cond(self.md5_data[x])) + + def cond(self, value): + if self.action == "eq": + return value == self.target + elif self.action == "ne": + return value != self.target + elif self.action == "gt": + return value > self.target + elif self.action == "lt": + return value < self.target + elif self.action == "ge": + return value >= self.target + elif self.action == "le": + return value <= self.target + else: + raise ValueError(f"Invalid action: {self.action}") + + +def get_md5_filter(parent, config): + success, msg = check_type(parent, config) + if not success: + return False, msg, None + + required_args = ['name', 'path', 'action'] + types = [str, (str, list), str] + if config['type'] == 'dict': + required_args.extend(['target', 'is_valid']) + types.extend([None, bool]) + success, msg = check_keys(parent, config, required_args, types) + if not success: + return False, msg, None + + if config['action'] not in MD5Filter.valid_actions[config['type']]: + return False, f"Invalid action: {parent}.action: {config['action']}", None + + if config['type'] == 'list': + return True, "", ListFilter(config) + elif config['type'] == 'dict': + return True, "", DictFilter(config) + else: + raise ValueError(f"Invalid type: {config['type']}") + + +class FilterCompose(object): + def __init__(self, column_filter_list, md5_filter_list): + self.column_filter_list = column_filter_list + self.md5_filter_list = md5_filter_list + + def __call__(self, arrow_file, table): + stats = {} + length = len(table) + assert length > 0, "Empty table" + + mask = None + for filter_ in self.column_filter_list: + if isinstance(filter_, tuple): + op, filter_list = filter_ + if op == 'logical_or': + sub_mask = None + for sub_filter in filter_list: + if not sub_filter.applicable(arrow_file): + continue + sub_current_mask = sub_filter(arrow_file, table) + if sub_current_mask is not None: + if sub_mask is None: + sub_mask = sub_current_mask + else: + sub_mask = sub_mask | sub_current_mask + if sub_mask is not None: + name = '|'.join([f.name for f in filter_list]) + stats.update({ + name: length - sum(sub_mask) + }) + if mask is None: + mask = sub_mask + else: + mask = mask & sub_mask + else: + raise ValueError(f"Invalid operation: {op}") + else: + if not filter_.applicable(arrow_file): + continue + current_mask = filter_(arrow_file, table) + if current_mask is not None: + stats.update({ + filter_.name: length - sum(current_mask) + }) + if mask is None: + mask = current_mask + else: + mask = mask & current_mask + + md5s = None + for filter_ in self.md5_filter_list: + if not filter_.applicable(arrow_file): + continue + if 'md5' not in table.column_names: + print(f"Warning: Column 'md5' not found in {arrow_file}.") + + md5s = table['md5'].to_pandas() + current_mask = filter_(md5s) + if current_mask is not None: + stats.update({ + filter_.name: length - sum(current_mask) + }) + if mask is None: + mask = current_mask + else: + mask = mask & current_mask + + return mask, stats, md5s + + @property + def data_type(self): + data_type_list = [] + for filter_ in self.column_filter_list + self.md5_filter_list: + if isinstance(filter_, tuple): + data_type_list.append(' || '.join([f.data_type for f in filter_[1]])) + else: + data_type_list.append(filter_.data_type) + return data_type_list + + +class MD5Repeater(Operator): + def __init__(self, config): + super().__init__(config) + self.name = config['name'] + self.paths = self.parse_path(config['path']) + self.dtype = config['type'] + self.plus = config.get('plus', 0) + self.repeat = config.get('repeat', None) + self.arrow_file_keyword = config.get('arrow_file_keyword', None) + if self.arrow_file_keyword is not None: + if isinstance(self.arrow_file_keyword, str): + self.arrow_file_keyword = [self.arrow_file_keyword] + + self.md5_data = self.load_md5s(self.paths) + if self.repeat is None: + # Check if md5_data.values() are integers + for v in self.md5_data.values(): + if not isinstance(v, int): + raise ValueError(f"Values from {self.paths} are not integers. For example: {v}") + # We only check the first value for performance + # We assume all values are the same type + break + + def __call__(self, arrow_file, index, md5): + if md5 in self.md5_data: + if self.repeat is None: + return self.md5_data[md5] + self.plus + else: + return self.repeat + else: + return 1 + + @property + def data_type(self): + path_repr = ', '.join([str(p) for p in self.paths]) + if self.dtype == 'list': + fmt_str = f"[MD5] Repeat {self.repeat} times: {path_repr}" + elif self.dtype == 'dict': + fmt_str = f"[MD5] Repeat multiple times: {path_repr}" + else: + raise ValueError(f"Invalid type: {self.dtype}") + return fmt_str + + +def get_md5_repeater(parent, config): + success, msg = check_type(parent, config) + if not success: + return False, msg, None + + required_args = ['name', 'path'] + types = [str, str] + if config['type'] == 'list': + required_args.extend(['repeat']) + types.extend([int]) + success, msg = check_keys(parent, config, required_args, types) + if not success: + return False, msg, None + + return True, "", MD5Repeater(config) + + +class KeywordRepeater(Operator): + def __init__(self, config): + super().__init__(config) + self.keywords = config['keyword'] + self.repeat = config['repeat'] + self.name = f"Repeat {self.repeat} times" + + def __call__(self, arrow_file, idx, md5): + for key in self.keywords: + if key in arrow_file: + return self.repeat + return 1 + + @property + def data_type(self): + fmt_str = f"[Keyword] Repeat {self.repeat} times: {self.keywords}" + return fmt_str + +def get_keyword_repeater(parent, config): + required_args = ["repeat", "keyword"] + types = [int, list] + success, msg = check_keys(parent, config, required_args, types) + if not success: + return False, msg, None + + return True, "", KeywordRepeater(config) + + +class RepeaterCompose(object): + def __init__(self, repeater_list): + self.repeater_list = repeater_list + + def __call__(self, arrow_file, table, indices, repeat_times=1, md5s=None): + stats = defaultdict(int) + length = len(table) + assert length > 0, "Empty table" + + if md5s is None: + if 'md5' not in table.column_names: + print(f"Warning: Column 'md5' not found in {arrow_file}.") + md5s = None + else: + md5s = table['md5'].to_pandas() + + repeated_indices = [] + for idx in indices: + md5 = md5s[idx] if md5s is not None else None + max_repeat = repeat_times + max_i = -1 + for i, repeater in enumerate(self.repeater_list): + if not repeater.applicable(arrow_file): + continue + repeat = repeater(arrow_file, idx, md5) + if repeat > max_repeat: + max_repeat = repeat + max_i = i + if max_i >= 0: + stats[self.repeater_list[max_i].name] += (max_repeat - 1) + repeated_indices.extend([idx] * max_repeat) + + return repeated_indices, stats + + @property + def data_type(self): + data_type_list = [] + for repeater in self.repeater_list: + data_type_list.append(repeater.data_type) + return data_type_list + + +class DatasetConfig(object): + def __init__(self, work_dir, config_file): + self.work_dir = work_dir + self.config_file = str(Path(config_file).absolute()) + + with open(self.config_file, 'r') as f: + self.data = yaml.safe_load(f) + + self.names = self.parse_names() # arrow names + arrow_max_repeat = max([x[1] for x in self.names]) + self.filter = self.parse_filter() + self.repeater = self.parse_repeater(enable_arrow_repeat=arrow_max_repeat > 1) + + # Extra arguments + self.remove_md5_dup = self.data.get('remove_md5_dup', False) + + def assert_unknown_args(self): + unknown_args = set(self.data.keys()) - { + 'source', 'filter', 'repeater', 'remove_md5_dup' + } + if len(unknown_args) > 0: + raise ValueError(f"Unknown arguments in config file ({self.config_file}): {unknown_args}") + + def parse_filter(self): + column_filter_list = [] + md5_filter_list = [] + if 'filter' in self.data: + filters = self.data['filter'] + if 'column' in filters: + column_filter_configs = filters['column'] + assert isinstance(column_filter_configs, list), "filter.column should be a list." + for i, config in enumerate(column_filter_configs): + if config.get('logical_or', None) is not None: + assert isinstance(config['logical_or'], list), \ + f"filter.column[{i}].logical_or should be a list, got {type(config['logical_or'])}" + sub_column_filter_list = [] + for j, sub_config in enumerate(config['logical_or']): + sub_success, sub_msg, sub_filter_ = get_column_filter(f'filter.column[{i}-logical_or].[{j}]', sub_config) + if not sub_success: + raise ValueError(sub_msg) + sub_column_filter_list.append(sub_filter_) + success = True + msg = '' + filter_ = ('logical_or', sub_column_filter_list) + else: + success, msg, filter_ = get_column_filter(f'filter.column[{i}]', config) + if not success: + raise ValueError(msg) + column_filter_list.append(filter_) + if 'md5' in filters: + md5_filter_configs = filters['md5'] + assert isinstance(md5_filter_configs, list), "filter.md5 should be a list." + for i, config in enumerate(md5_filter_configs): + if config.get('logical_or', None) is not None: + assert isinstance(config['logical_or'], list), \ + f"filter.md5[{i}].logical_or should be a list, got {type(config['logical_or'])}" + sub_md5_filter_list = [] + for j, sub_config in enumerate(config['logical_or']): + sub_success, sub_msg, sub_filter_ = get_md5_filter(f'filter.md5[{i}-logical_or].[{j}]', sub_config) + if not sub_success: + raise ValueError(sub_msg) + sub_md5_filter_list.append(sub_filter_) + success = True + msg = '' + filter_ = ('logical_or', sub_md5_filter_list) + else: + success, msg, filter_ = get_md5_filter(f'filter.md5[{i}]', config) + if not success: + raise ValueError(msg) + md5_filter_list.append(filter_) + + if column_filter_list or md5_filter_list: + composed_filter = FilterCompose(column_filter_list, md5_filter_list) + else: + composed_filter = None + return composed_filter + + def parse_repeater(self, enable_arrow_repeat=False): + repeater_list = [] + if 'repeater' in self.data: + repeaters = self.data['repeater'] + if 'md5' in repeaters: + md5_repeater_configs = repeaters['md5'] + assert isinstance(md5_repeater_configs, list), "repeater.md5 should be a list." + for i, config in enumerate(md5_repeater_configs): + success, msg, repeater = get_md5_repeater(f'repeater.md5[{i}]', config) + if not success: + raise ValueError(msg) + repeater_list.append(repeater) + + if 'arrow_file_keyword' in repeaters: + keyword_repeater_configs = repeaters['arrow_file_keyword'] + assert isinstance(keyword_repeater_configs, list), "repeater.arrow_file_keyword should be a list." + for i, config in enumerate(keyword_repeater_configs): + success, msg, repeater = get_keyword_repeater(f'repeater.arrow_file_keyword[{i}]', config) + if not success: + raise ValueError(msg) + repeater_list.append(repeater) + + if repeater_list or enable_arrow_repeat: + composed_repeater = RepeaterCompose(repeater_list) + else: + composed_repeater = None + return composed_repeater + + def parse_names(self): + if 'source' in self.data: + source = self.data['source'] + assert isinstance(source, list), "source should be a list." + else: + raise ValueError("In the YAML file, the ‘source’ field is filled in incorrectly, please check") + + names = get_or_create_arrow_name_list(source) + return names + + @property + def data_type(self): + data_types = { + 'Filters': [] if self.filter is None else self.filter.data_type, + 'Repeaters': [] if self.repeater is None else self.repeater.data_type, + } + return data_types diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/make_dataset_core.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/make_dataset_core.py new file mode 100644 index 0000000000000000000000000000000000000000..6f9f388b3ff08c7e7c94a20ed7a3b49140752dfd --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/dataset/make_dataset_core.py @@ -0,0 +1,320 @@ +import json +import pickle +from collections import defaultdict +from glob import glob +from multiprocessing import Pool +from pathlib import Path + +import pandas as pd +import yaml + +import numpy as np +import pyarrow as pa +from tqdm import tqdm + +from index_kits.indexer import IndexV2Builder +from index_kits.bucket import build_multi_resolution_bucket +from index_kits.dataset.config_parse import DatasetConfig + + +def get_table(arrow_file): + return pa.ipc.RecordBatchFileReader(pa.memory_map(arrow_file, 'r')).read_all() + + +def get_indices(arrow_file, repeat_times, filter_fn, repeat_fn, callback=None): + """ + Get valid indices from a single arrow_file. + + Parameters + ---------- + arrow_file: str + repeat_times: int + Repeat remain indices multiple times. + filter_fn + callback + + Returns + ------- + + """ + try: + table = pa.ipc.RecordBatchFileReader(pa.memory_map(arrow_file, 'r')).read_all() + except Exception as e: + print(arrow_file, e) + raise e + length = len(table) + + if len(table) == 0: + print(f"Warning: Empty table: {arrow_file}") + indices = [] + stats = {} + + else: + # Apply filter_fn if available + if filter_fn is not None: + mask, stats, md5s = filter_fn(arrow_file, table) + else: + mask = pd.Series([True] * length) + stats = {} + md5s = None + + # Apply callback function if available + if callback is not None: + mask, stats = callback(arrow_file, table, mask, stats, md5s) + + # Get indices + if mask is not None: + indices = np.where(mask)[0].tolist() + else: + indices = list(range(length)) + + # Apply indices repeat + if repeat_fn is not None: + indices, repeat_stats = repeat_fn(arrow_file, table, indices, repeat_times, md5s) + stats.update(repeat_stats) + + return arrow_file, length, indices, stats + + +def load_md5_files(files, name=None): + if isinstance(files, str): + files = [files] + md5s = set() + for file in files: + md5s.update(Path(file).read_text().splitlines()) + print(f" {name} md5s: {len(md5s):,}") + + return md5s + +def load_md52cls_files(files, name=None): + if isinstance(files, str): + files = [files] + md52cls = {} + for file in files: + with Path(file).open() as f: + md52cls.update(json.load(f)) + print(f" {name} md52cls: {len(md52cls):,}") + + return md52cls + + +def merge_and_build_index(data_type, src, dconfig, save_path): + if isinstance(src, str): + files = list(sorted(glob(src))) + else: + files = list(sorted(src)) + print(f"Found {len(files):,} temp pickle files.") + for fname in files: + print(f" {fname}") + arrow_files = [] + table_lengths = [] + indices_list = [] + bad_stats_total = defaultdict(int) + total_indices = 0 + total_processed_length = 0 + for file_name in tqdm(files): + with Path(file_name).open('rb') as f: + data = pickle.load(f) + for arrow_file, table_length, indices, *args in tqdm(data, leave=False): + arrow_files.append(arrow_file) + table_lengths.append(table_length) + total_processed_length += table_length + indices_list.append(indices) + total_indices += len(indices) + if len(args) > 0 and args[0]: + bad_stats = args[0] + for k, v in bad_stats.items(): + bad_stats_total[k] += v + + if len(bad_stats_total): + stats_save_dir = Path(save_path).parent + stats_save_dir.mkdir(parents=True, exist_ok=True) + stats_save_path = stats_save_dir / (Path(save_path).stem + '_stats.txt') + stats_save_path.write_text('\n'.join([f'{k:>50s} {v}' for k, v in bad_stats_total.items()]) + '\n') + print(f"Save stats to {stats_save_path}") + + print(f'Arrow files: {len(arrow_files):,}') + print(f'Processed indices: {total_processed_length:,}') + print(f'Valid indices: {total_indices:,}') + + cum_length = 0 + total_indices = [] + cum_lengths = [] + group_lengths = [] + existed = set() + print(f"Accumulating indices...") + pbar = tqdm(zip(arrow_files, table_lengths, indices_list), total=len(arrow_files), mininterval=1) + _count = 0 + for arrow_file, table_length, indices in pbar: + if len(indices) > 0 and dconfig.remove_md5_dup: + new_indices = [] + table = get_table(arrow_file) + if 'md5' not in table.column_names: + raise ValueError(f"Column 'md5' not found in {arrow_file}. " + f"When `remove_md5_dup: true` is set, md5 column is required.") + md5s = table['md5'].to_pandas() + for i in indices: + md5 = md5s[i] + if md5 in existed: + continue + existed.add(md5) + new_indices.append(i) + indices = new_indices + + total_indices.extend([int(i + cum_length) for i in indices]) + cum_length += table_length + cum_lengths.append(cum_length) + group_lengths.append(len(indices)) + + _count += 1 + + if _count % 100 == 0: + pbar.set_description(f'Indices: {len(total_indices):,}') + + builder = IndexV2Builder(data_type=data_type, + arrow_files=arrow_files, + cum_length=cum_lengths, + group_length=group_lengths, + indices=total_indices, + config_file=dconfig.config_file, + ) + builder.build(save_path) + print(f'Build index finished!\n\n' + f' Save path: {Path(save_path).absolute()}\n' + f' Number of indices: {len(total_indices)}\n' + f'Number of arrow files: {len(arrow_files)}\n' + ) + + +def worker_startup(rank, world_size, dconfig, prefix, work_dir, callback=None): + # Prepare names for this worker + num = (len(dconfig.names) + world_size - 1) // world_size + arrow_names = dconfig.names[rank * num:(rank + 1) * num] + print(f'Rank {rank} has {len(arrow_names):,} names.') + + # Run get indices + print(f"Start getting indices...") + indices = [] + for arrow_name, repeat_times in tqdm(arrow_names, position=rank, desc=f"#{rank}: ", leave=False): + indices.append(get_indices(arrow_name, repeat_times, dconfig.filter, dconfig.repeater, callback)) + + # Save to a temp file + temp_save_path = work_dir / f'data/temp_pickles/{prefix}-{rank + 1}_of_{world_size}.pkl' + temp_save_path.parent.mkdir(parents=True, exist_ok=True) + with temp_save_path.open('wb') as f: + pickle.dump(indices, f) + print(f'Rank {rank} finished. Write temporary data to {temp_save_path}') + + return temp_save_path + + +def startup(config_file, + save, + world_size=1, + work_dir='.', + callback=None, + use_cache=False, + ): + work_dir = Path(work_dir) + save_path = Path(save) + if save_path.suffix != '.json': + save_path = save_path.parent / (save_path.name + '.json') + print(f"Using save_path: {save_path}") + prefix = f"{save_path.stem}" + + # Parse dataset config and build the data_type list + dconfig = DatasetConfig(work_dir, config_file) + data_type = [] + for k, v in dconfig.data_type.items(): + data_type.extend(v) + print(f"{k}:") + for x in v: + print(f' {x}') + if dconfig.remove_md5_dup: + data_type.append('Remove md5 duplicates.') + else: + data_type.append('Keep md5 duplicates.') + + # Start processing + if not use_cache: + temp_pickles = [] + if world_size == 1: + print(f"\nRunning in single process mode...") + temp_pickles.append(worker_startup(rank=0, + world_size=1, + dconfig=dconfig, + prefix=prefix, + work_dir=work_dir, + callback=callback, + )) + else: + print(f"\nRunning in multi-process mode (world_size={world_size})...") + p = Pool(world_size) + temp_pickles_ = [] + for i in range(world_size): + temp_pickles_.append(p.apply_async(worker_startup, args=(i, world_size, dconfig, prefix, work_dir, callback))) + + for res in temp_pickles_: + temp_pickles.append(res.get()) + # close + p.close() + p.join() + else: + temp_pickles = glob(f'{work_dir}/data/temp_pickles/{prefix}-*_of_{world_size}.pkl') + + # Merge temp pickles and build index + merge_and_build_index(data_type, + temp_pickles, + dconfig, + save_path, + ) + + +def make_multireso(target, + config_file=None, + src=None, + base_size=None, + reso_step=None, + target_ratios=None, + align=None, + min_size=None, + md5_file=None, + ): + if config_file is not None: + with Path(config_file).open() as f: + config = yaml.safe_load(f) + else: + config = {} + src = config.get('src', src) + base_size = config.get('base_size', base_size) + reso_step = config.get('reso_step', reso_step) + target_ratios = config.get('target_ratios', target_ratios) + align = config.get('align', align) + min_size = config.get('min_size', min_size) + md5_file = config.get('md5_file', md5_file) + + if src is None: + raise ValueError('src must be provided in either config file or command line.') + if base_size is None: + raise ValueError('base_size must be provided.') + if reso_step is None and target_ratios is None: + raise ValueError('Either reso_step or target_ratios must be provided.') + + if md5_file is not None: + with open(md5_file, 'rb') as f: + md5_hw = pickle.load(f) + print(f'Md5 to height and width: {len(md5_hw):,}') + else: + md5_hw = None + + build_multi_resolution_bucket( + config_file=config_file, + base_size=base_size, + reso_step=reso_step, + target_ratios=target_ratios, + align=align, + min_size=min_size, + src_index_files=src, + save_file=target, + md5_hw=md5_hw, + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/indexer.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/indexer.py new file mode 100644 index 0000000000000000000000000000000000000000..4e27fdaa628a5ceaad01e648a7a4c0cfa2afb080 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/indexer.py @@ -0,0 +1,832 @@ +import bisect +import io +import json +import random +from pathlib import Path +import ast +from itertools import chain +from collections import defaultdict +from functools import partial +from glob import glob + +import numpy as np +import pyarrow as pa +from PIL import Image +from tqdm import tqdm + + +def get_table(arrow_file): + """ + Read an arrow file and return an arrow table. + """ + return pa.ipc.RecordBatchFileReader(pa.memory_map(f"{arrow_file}", "r")).read_all() + + +def assert_type(data, dtype, msg=''): + if not isinstance(data, dtype): + raise ValueError(f'Expected {msg} type {dtype}, got {type(data)}.') + + +def ndarray_to_list(data): + if isinstance(data, np.ndarray): + data = data.tolist() + elif isinstance(data, dict): + data = {k: ndarray_to_list(v) for k, v in data.items()} + elif isinstance(data, (list, tuple)): + # Assert that all elements in data are python integer, not numpy integer. + # Because numpy integer cannot be serialized to json. + data = [int(x) for x in data] + else: + raise ValueError(f'Expected data type list, tuple, dict or np.ndarray, got {type(data)}.') + return data + + +class ArrowIndexV2(object): + """ + ArrowIndexV2 is a new version of ArrowIndex. + + Parameters + ---------- + index_file: str or pathlib.Path + The path of index file. Either index_file or res_dict should be provided. + res_dict: dict + The index dict. Either index_file or res_dict should be provided. + align: int + Align the length of indices to be a multiple of align. Generally align should be the batch size * world_size. + shadow_file_fn: callable or dict + A callable function to map shadow file path to a new path. If None, the shadow file path will not be + changed. If a dict is provided, the keys are the shadow names to call the function, and the values are the + callable functions to map the shadow file path to a new path. If a callable function is provided, the key + is 'default'. + + Examples + -------- + >>> index_file = 'data.json' + >>> indexObj = ArrowIndexV2(index_file) + >>> pil_image = indexObj.get_image(0) + >>> text = indexObj.get_attribute(0, column='text_zh') + + """ + def __init__(self, index_file=None, res_dict=None, align=1, + shadow_file_fn=None, **kwargs): + if index_file is not None: + with open(index_file, 'r') as f: + res_dict = json.load(f) + elif res_dict is not None: + pass + else: + raise ValueError(f'Either index_file or res_dict should be provided.') + + self.shadow_file_fn = {} + if shadow_file_fn is not None: + if not callable(shadow_file_fn) and not isinstance(shadow_file_fn, dict): + raise ValueError('shadow_file_fn should be a callable function or a dict.') + if callable(shadow_file_fn): + self.shadow_file_fn['default'] = shadow_file_fn + else: + for k, v in shadow_file_fn.items(): + if not callable(v): + raise ValueError(f'{k} should be a callable function.') + self.shadow_file_fn[k] = v + + self._data = res_dict + self.data_type = res_dict['data_type'] + self.arrow_files = res_dict['arrow_files'] + self.cum_length = res_dict['cum_length'] + + self.group_length = res_dict['group_length'] + error_msg = f'Expected group_length type list, got {type(self.group_length)}.' + if isinstance(self.group_length, dict): + raise ValueError(f'{error_msg}\nNote: You may using a multi-resolution index file. ' + 'Try `MultiResolutionBucketIndexV2` instead.') + elif not isinstance(self.group_length, list): + raise ValueError(error_msg) + + self.indices = res_dict['indices'] + if 'indices_file' in res_dict: + self.indices_file = res_dict['indices_file'] + if self.indices_file != '': + indices_file = Path(index_file).parent / self.indices_file + if Path(indices_file).exists(): + self.indices = np.load(indices_file)['x'] + else: + raise ValueError(f'This Index file contains an extra file {indices_file} which is missed.') + else: + self.indices_file = '' + + if not isinstance(self.indices, list) and not isinstance(self.indices, np.ndarray): + raise ValueError(f'Expected indices type list or np.ndarray, got {type(self.indices)}.') + + if align > 1: + if isinstance(self.indices, np.ndarray): + self.indices = self.indices.tolist() + self.align(align) + + self.indices = np.asarray(self.indices, int) + + if len(self.arrow_files) != len(self.cum_length): + raise ValueError(f'Length of arrow_files and cum_length does not match. {len(self.arrow_files)} != {len(self.cum_length)}') + if len(self.arrow_files) != len(self.group_length): + raise ValueError(f'Length of arrow_files and group_length does not match. {len(self.arrow_files)} != {len(self.group_length)}') + if len(self.indices) == 0: + raise ValueError(f'No indices found in index_dict.') + if isinstance(self.indices, list) and self.indices[-1] > self.cum_length[-1] - 1: + raise ValueError(f'Indices exceed cum_length.') + + # Warning: + # Ensure that indices are an increasing array. Currently, + # no checks are performed due to the potential slowness when dealing with hundreds of millions of data points. + + self.bias = self.cum_length + + self._cur_arrow_file = None + self._cur_table_map = None + self._cur_table = None + self._index_bias = 0 + self.last_index = -1 + + self._shadow_cur_arrow_file = {} + self._shadow_cur_table_map = {} + self._shadow_cur_table = {} + self._shadow_index_bias = {} + self.shadow_last_index = {} + for k in self.shadow_file_fn.keys(): + self._shadow_cur_arrow_file[k] = None + self._shadow_cur_table_map[k] = None + self._shadow_cur_table[k] = None + self._shadow_index_bias[k] = 0 + self.shadow_last_index[k] = -1 + + def __len__(self): + return len(self.indices) + + def __repr__(self): + return f""" + ArrowIndexV2( + data_type {self.data_type} + indices_file {self.indices_file} + arrow_files Count={len(self.arrow_files):,} ({self.arrow_files[0]}, ...) + cum_length Count={len(self.cum_length):,} ({self.cum_length[0]}, ...) + group_length Count={len(self.group_length):,} ({self.group_length[0]}, ...) + indices Count={len(self.indices):,} + example_indices Count={len(self._data['example_indices']):,} + ) + """ + + def check_exists(self): + for arrow_file in tqdm(self.arrow_files): + if not Path(arrow_file).exists(): + print(arrow_file) + + def align(self, align): + """ + Repeat the index so that the length is a multiple of batch_size * world_size. + """ + if len(self) % align == 0: + return + + repeat_num = align - len(self) % align + if repeat_num >= len(self): + repeat_n = repeat_num // len(self) + repeat_times = [repeat_n + 1 for _ in self.indices] + group_length_new = [ll * (repeat_n + 1) for ll in self.group_length] + repeat_num -= repeat_n * len(self) + else: + repeat_times = [1 for _ in range(repeat_num)] + group_length_new = [ll for ll in self.group_length] + + for i in range(repeat_num): + repeat_times[-i - 1] += 1 + + repeat_start_idx = len(self) - len(repeat_times) + + group_id = -1 + while group_length_new[group_id] == 0: + group_id -= 1 + + # Allocate the remaining indices that need to be repeated, + # while also counting how many indices have been checked. + # If the count reaches the group_length, switch to the next group + + # The reason for paying attention to group_length is that when repeating indices, + # group_length also needs to be updated synchronously.. + group_acc = 0 + for i in range(repeat_num): + group_length_new[group_id] += 1 + group_acc += 1 + if group_acc == self.group_length[group_id]: + group_id -= 1 + while group_length_new[group_id] == 0: + group_id -= 1 + group_acc = 0 + + temp = [] + for i, value in enumerate(self.indices[repeat_start_idx:]): + temp.extend([value] * repeat_times[i]) + + self.indices = np.concatenate([self.indices[:repeat_start_idx], temp]) + + self.group_length = group_length_new + + def shuffle(self, seed=None, fast=False): + """ + It takes about 30 seconds for an index consisting of 100_000 arrows. + """ + if fast: + return self.shuffle_fast(seed) + + indices = self.indices.tolist() + + if seed is not None: + state = random.getstate() + random.seed(seed) + + indices_group_list = [] + group_cum_len = 0 + for group_len in self.group_length: + indices_group = indices[group_cum_len:group_cum_len + group_len] + random.shuffle(indices_group) + indices_group_list.append((indices_group, group_len)) + group_cum_len += group_len + random.shuffle(indices_group_list) + self.group_length = [x[1] for x in indices_group_list] + self.indices = np.asarray(list(chain.from_iterable([x[0] for x in indices_group_list]))) + + if seed is not None: + random.setstate(state) + + def shuffle_fast(self, seed=None): + if seed is not None: + sampler = np.random.RandomState(seed) + sampler.shuffle(self.indices) + else: + np.random.shuffle(self.indices) + + def get_table(self, arrow_file, shadow=None): + """ + Read an arrow file and return an arrow table. + """ + if shadow is None: + if self._cur_table is not None: + if self._cur_arrow_file == arrow_file: + # This is the same arrow file. Return the cached table. + return self._cur_table + else: + # This is a different arrow file. Clear the cache. + self._cur_table_map.close() + self._cur_table = None + + self._cur_arrow_file = arrow_file + self._cur_table_map = pa.memory_map(f"{arrow_file}", "r") + self._cur_table = pa.ipc.RecordBatchFileReader(self._cur_table_map).read_all() + return self._cur_table + else: + if self._shadow_cur_table[shadow] is not None: + if self._shadow_cur_arrow_file[shadow] == arrow_file: + return self._shadow_cur_table[shadow] + else: + self._shadow_cur_table_map[shadow].close() + self._shadow_cur_table[shadow] = None + + self._shadow_cur_arrow_file[shadow] = arrow_file + self._shadow_cur_table_map[shadow] = pa.memory_map(f"{arrow_file}", "r") + self._shadow_cur_table[shadow] = pa.ipc.RecordBatchFileReader(self._shadow_cur_table_map[shadow]).read_all() + return self._shadow_cur_table[shadow] + + def get_arrow_file_by_index(self, index, return_index_bias=False, shadow=None): + i = bisect.bisect_right(self.cum_length, index) + arrow_file = self.arrow_files[i] + + if return_index_bias: + if i == 0: + index_bias = 0 + else: + index_bias = self.cum_length[i - 1] + + return arrow_file, index_bias + + return arrow_file + + def get_arrow_file(self, ind, shadow=None): + """ + Get arrow file by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + shadow: str + The shadow name. If None, return the main arrow file. If not None, return the shadow arrow file. + + Returns + ------- + arrow_file: str + The arrow file path. + """ + index = self.indices[ind] + return self.get_arrow_file_by_index(index, shadow=shadow) + + def load_table_by_index(self, index, shadow=None): + if shadow is None: + if index == self.last_index: + return self._cur_table + arrow_file, self._index_bias = \ + self.get_arrow_file_by_index(index, return_index_bias=True) + self._cur_table = self.get_table(arrow_file) + self.last_index = index + return self._cur_table + else: + if index == self.shadow_last_index[shadow]: + return self._shadow_cur_table[shadow] + shadow_arrow_file, _shadow_index_bias = \ + self.get_arrow_file_by_index(index, return_index_bias=True, shadow=shadow) + self._shadow_index_bias[shadow] = _shadow_index_bias + self._shadow_cur_table[shadow] = self.get_table(shadow_arrow_file, shadow=shadow) + self.shadow_last_index[shadow] = index + return self._shadow_cur_table[shadow] + + def get_data_by_index(self, index, columns=None, allow_missing=False, return_meta=True, shadow=None): + table = self.load_table_by_index(index, shadow=shadow) + if isinstance(columns, str): + columns = [columns] + if columns is None: + columns = list(table.column_names) + + index_bias = self._index_bias if shadow is None else self._shadow_index_bias[shadow] + in_arrow_index = index - index_bias + if return_meta: + cur_arrow_file = self._cur_arrow_file if shadow is None else self._shadow_cur_arrow_file[shadow] + data = { + 'index': index, + 'in_arrow_index': in_arrow_index, + 'arrow_name': cur_arrow_file, + } + else: + data = {} + + if allow_missing: + for col in columns: + if col in table.column_names: + data[col] = table[col][in_arrow_index].as_py() + else: + for col in columns: + data[col] = table[col][in_arrow_index].as_py() + return data + + def get_data(self, ind, columns=None, allow_missing=False, return_meta=True, shadow=None): + """ + Get data by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + columns: str or list + The columns to be returned. If None, return all columns. + allow_missing: bool + If True, omit missing columns. If False, raise an error if the column is missing. + return_meta: bool + If True, the resulting dict will contain some meta information: + in-json index, in-arrow index, and arrow_name. + shadow: str + The shadow name. If None, return the main data. If not None, return the shadow data. + + Returns + ------- + data: dict + A dict containing the data. + """ + index = self.indices[ind] + return self.get_data_by_index(index, columns, allow_missing=allow_missing, return_meta=return_meta, + shadow=shadow) + + def get_attribute_by_index(self, index, column, shadow=None): + table = self.load_table_by_index(index, shadow=shadow) + index_bias = self._index_bias if shadow is None else self._shadow_index_bias[shadow] + return table[column][index - index_bias].as_py() + + def get_attribute(self, ind, column, shadow=None): + """ + Get single attribute by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + column: str + The column name. + shadow: str + The shadow name. If None, return the main data. If not None, return the shadow data. + + Returns + ------- + data: can be any type + """ + index = self.indices[ind] + return self.get_attribute_by_index(index, column, shadow=shadow) + + def get_image_by_index(self, index, column='image', ret_type='pil', max_size=-1, shadow=None): + table = self.load_table_by_index(index, shadow=shadow) + index_bias = self._index_bias if shadow is None else self._shadow_index_bias[shadow] + + col = 'image' if 'image' in table.column_names else 'binary' + temp = table[col][index - index_bias].as_py() + image_bytes = io.BytesIO(temp) + image_bytes.seek(0) + try: + # convert(RGB) has two purposes: + # 1. Convert the image to RGB mode. Some images are in grayscale/RGBA mode, which will cause channel + # inconsistency in following processing. + # 2. Convert the image to RGB mode. Some images are in P mode, which will be forced to use NEAREST resample + # method in resize (even if you specify LANCZOS), which will cause blurry images. + pil_image = Image.open(image_bytes).convert("RGB") + except Exception as e: + print(f'get_image_by_index | Error: {e} ({self.get_arrow_file_by_index(index), index - index_bias})') + pil_image = Image.new("RGB", (256, 256), (255, 255, 255)) + + if max_size > 0: + # Resize the image to max_size. max_size is the size of long edge + w, h = pil_image.size + if w > h: + new_w = max_size + new_h = int(h * max_size / w) + else: + new_h = max_size + new_w = int(w * max_size / h) + pil_image = pil_image.resize((new_w, new_h)) + + if ret_type == 'numpy': + return np.array(pil_image) + + return pil_image + + def get_image(self, ind, column='image', ret_type='pil', max_size=-1, shadow=None): + """ + Get image by in-dataset index. + + Parameters + ---------- + ind: int + The in-dataset index. + column: str + [Deprecated] The column name of the image. Default to 'image'. + ret_type: str + The return type. Can be 'pil' or 'numpy'. Default to 'pil'. + max_size: int + If not -1, resize the image to max_size. max_size is the size of long edge. + shadow: str + The shadow name. If None, return the main image. If not None, return the shadow image. + + Returns + ------- + image: PIL.Image.Image or np.ndarray + """ + index = self.indices[ind] + return self.get_image_by_index(index, column, ret_type, max_size, shadow=shadow) + + def get_md5_by_index(self, index, shadow=None): + table = self.load_table_by_index(index, shadow=shadow) + index_bias = self._index_bias if shadow is None else self._shadow_index_bias[shadow] + return table['md5'][index - index_bias].as_py() + + def get_md5(self, ind, shadow=None): + index = self.indices[ind] + return self.get_md5_by_index(index, shadow=shadow) + + def get_columns_by_index(self, index, shadow=None): + table = self.load_table_by_index(index, shadow=shadow) + return table.column_names + + def get_columns(self, ind, shadow=None): + index = self.indices[ind] + return self.get_columns_by_index(index, shadow=shadow) + + def source_distribution(self, save_path=None, shadow=None): + sources = defaultdict(int) + for index in tqdm(self.indices): + source = self.get_attribute_by_index(index, 'source', shadow=shadow) + sources[source] += 1 + + sources = sorted(sources.items(), key=lambda x: x[1], reverse=True) + for k, v in sources: + print(f'{k:20s} {v:10d}') + if save_path is not None: + Path(save_path).write_text( + '\n'.join([f'{k:20s} {v:10d}' for k, v in sources])) + + def save(self, save_path): + """ + Save the index to a json file. + + Parameters + ---------- + save_path: str or pathlib.Path + The path to save the index file. + """ + builder = IndexV2Builder(data_type=self.data_type, + arrow_files=self.arrow_files, + cum_length=self.cum_length, + indices=self.indices, + ) + builder.build(save_path) + + def sample_batch_indices(self, n): + return np.random.choice(self.indices, n) + + def sample_batch(self, n, columns, progress=True, shadow=None): + if isinstance(n, int): + indices = self.sample_batch_indices(n) + else: + indices = n + + if progress: + pbar = tqdm(indices) + else: + pbar = indices + + batch_data = [] + for i in pbar: + batch_data.append(self.get_data_by_index(i, columns, shadow=shadow)) + return batch_data + + @staticmethod + def resize_and_crop(image, target_size, resample=Image.LANCZOS, crop_type='random'): + """ + Resize image without changing aspect ratio, then crop the center/random part. + + Parameters + ---------- + image: PIL.Image.Image + The input image to be resized and cropped. + target_size: tuple + The target size of the image. + resample: + The resample method. See PIL.Image.Image.resize for details. Default to Image.LANCZOS. + crop_type: str + 'center' or 'random'. If 'center', crop the center part of the image. If 'random', + crop a random part of the image. Default to 'random'. + + Returns + ------- + image: PIL.Image.Image + The resized and cropped image. + crop_pos: tuple + The position of the cropped part. (crop_left, crop_top) + """ + tw, th = target_size + w, h = image.size + + tr = th / tw + r = h / w + + # resize + if r < tr: + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) + + image = image.resize((resize_width, resize_height), resample=resample) + + if crop_type == 'center': + crop_top = int(round((resize_height - th) / 2.0)) + crop_left = int(round((resize_width - tw) / 2.0)) + elif crop_type == 'random': + crop_top = random.randint(0, resize_height - th) + crop_left = random.randint(0, resize_width - tw) + else: + raise ValueError(f'crop_type must be center or random, but got {crop_type}') + + image = image.crop((crop_left, crop_top, crop_left + tw, crop_top + th)) + return image, (crop_left, crop_top) + + +class IndexV2Builder(object): + def __init__(self, + arrow_files, + indices=None, + cum_length=None, + group_length=None, + data_type=None, + max_indices=5_000_000, + example_num=1000, + config_file=None, + ): + """ + Build index v2 from an index dict. + + Parameters + ---------- + arrow_files: list + A list of arrow files. + indices: list or dict + A list of indices or a dict of indices. + If not provided, it will be specified as range(cum_length[-1]). + cum_length: list + A list of cumulative length of arrow files. + If not provided, it will be calculated from arrow files. + group_length: list + A list of group length or a dict of group length for each arrow file. + If not provided, it will be calculated. + data_type: str or list + Some custom information of this index. + max_indices: int + If the number of indices is larger than max_indices, the indices will be saved in a separate file. + Default to 5_000_000. + example_num: int + The number of examples to be saved in the index file. Default to 1000. + config_file: str + The path of config file. + + Examples + -------- + >>> builder = IndexV2Builder( + >>> data_type='gold', + >>> arrow_files=arrow_files, + >>> cum_length=cum_length, + >>> indices=indices, + >>> ) + >>> builder.build(save_path) + + """ + self.arrow_files = arrow_files + self.indices = indices + self.cum_length = cum_length + self.group_length = group_length + self.data_type = data_type + self.max_indices = max_indices + self.example_num = example_num + self.config_file = config_file + + if isinstance(arrow_files, str): + if '*' in arrow_files or '?' in arrow_files: + self.arrow_files = list(glob(arrow_files)) + else: + self.arrow_files = [arrow_files] + elif isinstance(self.arrow_files, tuple): + self.arrow_files = list(self.arrow_files) + if not isinstance(self.arrow_files, list): + raise ValueError(f'Expected arrow_files to be a list, got {type(self.arrow_files)}.') + + if self.cum_length is None: + continuous = False + if self.indices is None: + self.group_length = [] + continuous = True + + print(f"Calculating cum_length...") + self.cum_length = [] + cur_cum_length = 0 + pbar = tqdm(self.arrow_files) + for arrow_file in pbar: + table_length = len(get_table(arrow_file)) + cur_cum_length += table_length + self.cum_length.append(cur_cum_length) + pbar.set_description(f"{self.cum_length[-1]:>12d}") + + if continuous: + self.group_length.append(table_length) + + if self.indices is None: + self.indices = list(range(self.cum_length[-1])) + + if self.group_length is None: + self.group_length = [] + + if self.data_type is None: + self.data_type = ['Made by IndexV2Builder'] + elif isinstance(self.data_type, str): + self.data_type = [self.data_type] + + assert_type(self.data_type, list, 'data_type') + assert_type(self.cum_length, (list, np.ndarray), 'cum_length') + assert_type(self.group_length, (list, dict, np.ndarray), 'group_length') + assert_type(self.indices, (list, dict, np.ndarray), 'indices') + self.cum_length = ndarray_to_list(self.cum_length) + self.group_length = ndarray_to_list(self.group_length) + self.indices = ndarray_to_list(self.indices) + + if isinstance(self.indices, dict): + for k, v in self.indices.items(): + assert_type(v, list, f'indices[{k}]') + + if len(self.arrow_files) != len(self.cum_length): + raise ValueError(f'Length of arrow_files and cum_length does not match. {len(self.arrow_files)} != {len(self.cum_length)}') + if len(self.indices) == 0: + raise ValueError(f'No indices found in index_dict.') + if isinstance(self.indices, list) and self.indices[-1] > self.cum_length[-1] - 1: + raise ValueError(f'Indices exceed cum_length. {self.indices[-1]} > {self.cum_length[-1] - 1}') + if len(self.group_length) > 0: + if len(self.arrow_files) != len(self.group_length): + raise ValueError(f'Length of arrow_files and group_length does not match. {len(self.arrow_files)} != {len(self.group_length)}') + if sum(self.group_length) != len(self.indices): + raise ValueError(f'Sum of group_length does not match length of indices. {sum(self.group_length)} != {len(self.indices)}') + + def encode(self): + # Encode arrow files + print("Encoding arrow files...") + arrow_files = [] + for arrow_file in tqdm(self.arrow_files): + shortname = arrow_file + arrow_files.append(shortname) + self.arrow_files = arrow_files + + # Calculate group_length + print("Calculating group length...") + if isinstance(self.indices, list): + if len(self.group_length) == 0: + self.group_length = self.calc_group_length(self.indices, self.cum_length) + else: + print("Group length already calculated, skip.") + elif isinstance(self.indices, dict): + if not isinstance(self.group_length, dict): + self.group_length = {} + for k, v in self.indices.items(): + print(f"Calculating group length for {k}...") + if k not in self.group_length or len(self.group_length[k]) == 0: + self.group_length[k] = self.calc_group_length(v, self.cum_length) + else: + print("Group length already calculated, skip.") + else: + raise ValueError(f'Expected indices type list or dict, got {type(self.indices)}.') + + return { + 'data_type': self.data_type, + 'config_file': self.config_file if self.config_file is not None else '', + 'indices_file': '', + 'arrow_files': self.arrow_files, + 'cum_length': self.cum_length, + 'group_length': self.group_length, + 'indices': self.indices, + 'example_indices': [], + } + + def build(self, save_path): + return self.save(save_path) + + def save(self, save_path): + """ + Make index v2 from an index dict. + + Parameters + ---------- + save_path: str or pathlib.Path + The path to save the index file. + """ + index_dict = self.encode() + # Ensure the indices either a list or a dict. + + save_path = Path(save_path) + save_path.parent.mkdir(exist_ok=True, parents=True) + + if isinstance(index_dict['indices'], list) and len(index_dict['indices']) > self.max_indices: + self.example_indices = index_dict['indices'][:self.example_num] + indices_to_save = {'x': index_dict['indices']} + index_dict['indices'] = [] + elif isinstance(index_dict['indices'], dict): + indices_to_save = index_dict['indices'] + index_dict['indices'] = {} + num_keys = len(indices_to_save) + example_num_per_key = max(self.example_num // num_keys, 10) + index_dict['example_indices'] = {k: v[:example_num_per_key] for k, v in index_dict['indices'].items()} + else: + indices_to_save = None + + # save indices + if indices_to_save is not None: + indices_file = save_path.parent / f'{save_path.stem}.index' + indices_dict = {k: np.array(v) for k, v in indices_to_save.items()} + np.savez_compressed(indices_file, **indices_dict) + index_dict['indices_file'] = indices_file.name + '.npz' + + with save_path.open('w') as f: + json.dump(index_dict, f, indent=4, ensure_ascii=False) + + @staticmethod + def calc_group_length(indices, cum_length): + group_lengths = [] + cum_ind = 0 + count = 0 + for index in tqdm(indices): + if index < cum_length[cum_ind]: + # index is still in the current group + count += 1 + else: + # index has exceeded the current group, need to switch to the next group + group_lengths.append(count) + cum_ind += 1 + # if the index exceeds the next group, continue to switch to the next group + while index >= cum_length[cum_ind]: + group_lengths.append(0) + cum_ind += 1 + count = 1 + # The indices array is exhausted, and the last group containing the index should also be added. + group_lengths.append(count) + assert len(group_lengths) <= len(cum_length), (len(group_lengths), len(cum_length)) + # Check if the number of groups is less than the number of cum_length, + # then the last n groups are empty and need to be filled with zeros. + if len(group_lengths) < len(cum_length): + group_lengths.extend([0] * (len(cum_length) - len(group_lengths))) + + return group_lengths \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/sampler.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..ab34ffc1fa0e2c3c2f7ff9ec02d93b011f0ab5b8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/index_kits/sampler.py @@ -0,0 +1,138 @@ +import math + +import torch.distributed as dist +from torch.utils.data.distributed import DistributedSampler + + +class BlockDistributedSampler(DistributedSampler): + def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True, seed=0, drop_last=False, + batch_size=-1, start_index=0): + super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last) + if num_replicas is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + num_replicas = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + if rank >= num_replicas or rank < 0: + raise ValueError( + "Invalid rank {}, rank should be in the interval" + " [0, {}]".format(rank, num_replicas - 1)) + if batch_size == -1: + raise ValueError("batch_size should be specified") + self.dataset = dataset + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.drop_last = drop_last + self.shuffle = shuffle + self.seed = seed + self.batch_size = batch_size + self._start_index = start_index + self.recompute_sizes() + + @property + def start_index(self): + return self._start_index + + @start_index.setter + def start_index(self, value): + self._start_index = value + self.recompute_sizes() + + def recompute_sizes(self): + self.num_samples = len(self.dataset) // self.batch_size * self.batch_size // self.num_replicas \ + - self._start_index + self.total_size = self.num_samples * self.num_replicas + + def __iter__(self): + indices = list(range(len(self.dataset))) # type: ignore[arg-type] + raw_num_samples = len(indices) // self.batch_size * self.batch_size // self.num_replicas + raw_total_size = raw_num_samples * self.num_replicas + indices = indices[:raw_total_size] + + # We require that the dataset size is divisible by batch_size * num_replicas + # This is naturally satisfied when using index_kits. + # In future, we can remove this assertion. + assert len(indices) == raw_total_size, f"{len(indices)} vs {raw_total_size}" + + # subsample with start_index + indices = indices[self.rank * raw_num_samples + self.start_index:(self.rank + 1) * raw_num_samples] + assert len(indices) + self.start_index == raw_num_samples, \ + f"{len(indices) + self.start_index} vs {raw_num_samples}" + + # This is a sequential sampler. The shuffle operation is done by the dataset itself. + return iter(indices) + + +class DistributedSamplerWithStartIndex(DistributedSampler): + def __init__(self, dataset, num_replicas=None, rank=None, shuffle=True, seed=0, drop_last=False, + start_index=0): + super().__init__(dataset, num_replicas, rank, shuffle, seed, drop_last) + if num_replicas is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + num_replicas = dist.get_world_size() + if rank is None: + if not dist.is_available(): + raise RuntimeError("Requires distributed package to be available") + rank = dist.get_rank() + if rank >= num_replicas or rank < 0: + raise ValueError( + "Invalid rank {}, rank should be in the interval" + " [0, {}]".format(rank, num_replicas - 1)) + self.dataset = dataset + self.num_replicas = num_replicas + self.rank = rank + self.epoch = 0 + self.drop_last = drop_last + self._start_index = start_index + self.recompute_sizes() + self.shuffle = shuffle + self.seed = seed + + @property + def start_index(self): + return self._start_index + + @start_index.setter + def start_index(self, value): + self._start_index = value + self.recompute_sizes() + + def recompute_sizes(self): + # If the dataset length is evenly divisible by # of replicas, then there + # is no need to drop any data, since the dataset will be split equally. + if self.drop_last and (len(self.dataset) - self._start_index) % self.num_replicas != 0: # type: ignore[arg-type] + # Split to nearest available length that is evenly divisible. + # This is to ensure each rank receives the same amount of data when + # using this Sampler. + self.num_samples = math.ceil( + ((len(self.dataset) - self._start_index) - self.num_replicas) / self.num_replicas # type: ignore[arg-type] + ) + else: + self.num_samples = math.ceil((len(self.dataset) - self._start_index) / self.num_replicas) # type: ignore[arg-type] + self.total_size = self.num_samples * self.num_replicas + + def __iter__(self): + indices = list(range(self._start_index, len(self.dataset))) # type: ignore[arg-type] + + if not self.drop_last: + # add extra samples to make it evenly divisible + padding_size = self.total_size - len(indices) + if padding_size <= len(indices): + indices += indices[:padding_size] + else: + indices += (indices * math.ceil(padding_size / len(indices)))[:padding_size] + else: + # remove tail of data to make it evenly divisible. + indices = indices[:self.total_size] + assert len(indices) == self.total_size + + # subsample with start_index + indices = indices[self.rank:self.total_size:self.num_replicas] + assert len(indices) == self.num_samples + + return iter(indices) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/setup.py b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..57c9e3cb1c1fb5e25f384c21c6e545375ad384e8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/IndexKits/setup.py @@ -0,0 +1,28 @@ +import re + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +with open("index_kits/__init__.py", "r") as file: + regex_version = r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]' + version = re.search(regex_version, file.read(), re.MULTILINE).group(1) + + +setup( + name='index_kits', + version=version, + author='jarvizhang', + author_email='jarvizhang@tencent.com', + description='An index kits for streaming reading arrow data.', + packages=['index_kits', 'index_kits/dataset'], + scripts=['bin/idk'], + install_requires=[ + "pillow>=9.3.0", + "tqdm>=4.60.0", + "pyarrow>=10.0.1", + "torch>=1.9", + ], + python_requires=">=3.8.12", +) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/LICENSE.txt b/PyTorch/built-in/mlm/HunyuanDiT/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..61ea65d146896b65b1e843ba04c10315e820912c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/LICENSE.txt @@ -0,0 +1,74 @@ +TENCENT HUNYUAN COMMUNITY LICENSE AGREEMENT +Tencent Hunyuan Release Date: 2024/5/14 +By clicking to agree or by using, reproducing, modifying, distributing, performing or displaying any portion or element of the Tencent Hunyuan Works, including via any Hosted Service, You will be deemed to have recognized and accepted the content of this Agreement, which is effective immediately. +1. DEFINITIONS. +a. “Acceptable Use Policy” shall mean the policy made available by Tencent as set forth in the Exhibit A. +b. “Agreement” shall mean the terms and conditions for use, reproduction, distribution, modification, performance and displaying of the Hunyuan Works or any portion or element thereof set forth herein. +c. “Documentation” shall mean the specifications, manuals and documentation for Tencent Hunyuan made publicly available by Tencent. +d. “Hosted Service” shall mean a hosted service offered via an application programming interface (API), web access, or any other electronic or remote means. +e. “Licensee,” “You” or “Your” shall mean a natural person or legal entity exercising the rights granted by this Agreement and/or using the Tencent Hunyuan Works for any purpose and in any field of use. +f. “Materials” shall mean, collectively, Tencent’s proprietary Tencent Hunyuan and Documentation (and any portion thereof) as made available by Tencent under this Agreement. +g. “Model Derivatives” shall mean all: (i) modifications to Tencent Hunyuan or any Model Derivative of Tencent Hunyuan; (ii) works based on Tencent Hunyuan or any Model Derivative of Tencent Hunyuan; or (iii) any other machine learning model which is created by transfer of patterns of the weights, parameters, operations, or Output of Tencent Hunyuan or any Model Derivative of Tencent Hunyuan, to that model in order to cause that model to perform similarly to Tencent Hunyuan or a Model Derivative of Tencent Hunyuan, including distillation methods, methods that use intermediate data representations, or methods based on the generation of synthetic data Outputs by Tencent Hunyuan or a Model Derivative of Tencent Hunyuan for training that model. For clarity, Outputs by themselves are not deemed Model Derivatives. +h. “Output” shall mean the information and/or content output of Tencent Hunyuan or a Model Derivative that results from operating or otherwise using Tencent Hunyuan or a Model Derivative, including via a Hosted Service. +i. “Tencent,” “We” or “Us” shall mean THL A29 Limited. +j. “Tencent Hunyuan” shall mean the large language models, image/video/audio/3D generation models, and multimodal large language models and their software and algorithms, including trained model weights, parameters (including optimizer states), machine-learning model code, inference-enabling code, training-enabling code, fine-tuning enabling code and other elements of the foregoing made publicly available by Us at https://huggingface.co/Tencent-Hunyuan/HunyuanDiT and https://github.com/Tencent/HunyuanDiT . +k. “Tencent Hunyuan Works” shall mean: (i) the Materials; (ii) Model Derivatives; and (iii) all derivative works thereof. +l. “Third Party” or “Third Parties” shall mean individuals or legal entities that are not under common control with Us or You. +m. “including” shall mean including but not limited to. +2. GRANT OF RIGHTS. +We grant You a non-exclusive, worldwide, non-transferable and royalty-free limited license under Tencent’s intellectual property or other rights owned by Us embodied in or utilized by the Materials to use, reproduce, distribute, create derivative works of (including Model Derivatives), and make modifications to the Materials, only in accordance with the terms of this Agreement and the Acceptable Use Policy, and You must not violate (or encourage or permit anyone else to violate) any term of this Agreement or the Acceptable Use Policy. +3. DISTRIBUTION. +You may, subject to Your compliance with this Agreement, distribute or make available to Third Parties the Tencent Hunyuan Works, provided that You meet all of the following conditions: +a. You must provide all such Third Party recipients of the Tencent Hunyuan Works or products or services using them a copy of this Agreement; +b. You must cause any modified files to carry prominent notices stating that You changed the files; +c. You are encouraged to: (i) publish at least one technology introduction blogpost or one public statement expressing Your experience of using the Tencent Hunyuan Works; and (ii) mark the products or services developed by using the Tencent Hunyuan Works to indicate that the product/service is “Powered by Tencent Hunyuan”; and +d. All distributions to Third Parties (other than through a Hosted Service) must be accompanied by a “Notice” text file that contains the following notice: “Tencent Hunyuan is licensed under the Tencent Hunyuan Community License Agreement, Copyright © 2024 Tencent. All Rights Reserved. The trademark rights of “Tencent Hunyuan” are owned by Tencent or its affiliate.” +You may add Your own copyright statement to Your modifications and, except as set forth in this Section and in Section 5, may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Model Derivatives as a whole, provided Your use, reproduction, modification, distribution, performance and display of the work otherwise complies with the terms and conditions of this Agreement. If You receive Tencent Hunyuan Works from a Licensee as part of an integrated end user product, then this Section 3 of this Agreement will not apply to You. +4. ADDITIONAL COMMERCIAL TERMS. +If, on the Tencent Hunyuan version release date, the monthly active users of all products or services made available by or for Licensee is greater than 100 million monthly active users in the preceding calendar month, You must request a license from Tencent, which Tencent may grant to You in its sole discretion, and You are not authorized to exercise any of the rights under this Agreement unless or until Tencent otherwise expressly grants You such rights. +5. RULES OF USE. +a. Your use of the Tencent Hunyuan Works must comply with applicable laws and regulations (including trade compliance laws and regulations) and adhere to the Acceptable Use Policy for the Tencent Hunyuan Works, which is hereby incorporated by reference into this Agreement. You must include the use restrictions referenced in these Sections 5(a) and 5(b) as an enforceable provision in any agreement (e.g., license agreement, terms of use, etc.) governing the use and/or distribution of Tencent Hunyuan Works and You must provide notice to subsequent users to whom You distribute that Tencent Hunyuan Works are subject to the use restrictions in these Sections 5(a) and 5(b). +b. You must not use the Tencent Hunyuan Works or any Output or results of the Tencent Hunyuan Works to improve any other large language model (other than Tencent Hunyuan or Model Derivatives thereof). +6. INTELLECTUAL PROPERTY. +a. Subject to Tencent’s ownership of Tencent Hunyuan Works made by or for Tencent and intellectual property rights therein, conditioned upon Your compliance with the terms and conditions of this Agreement, as between You and Tencent, You will be the owner of any derivative works and modifications of the Materials and any Model Derivatives that are made by or for You. +b. No trademark licenses are granted under this Agreement, and in connection with the Tencent Hunyuan Works, Licensee may not use any name or mark owned by or associated with Tencent or any of its affiliates, except as required for reasonable and customary use in describing and distributing the Tencent Hunyuan Works. Tencent hereby grants You a license to use “Tencent Hunyuan” (the “Mark”) solely as required to comply with the provisions of Section 3(c), provided that You comply with any applicable laws related to trademark protection. All goodwill arising out of Your use of the Mark will inure to the benefit of Tencent. +c. If You commence a lawsuit or other proceedings (including a cross-claim or counterclaim in a lawsuit) against Us or any person or entity alleging that the Materials or any Output, or any portion of any of the foregoing, infringe any intellectual property or other right owned or licensable by You, then all licenses granted to You under this Agreement shall terminate as of the date such lawsuit or other proceeding is filed. You will defend, indemnify and hold harmless Us from and against any claim by any Third Party arising out of or related to Your or the Third Party’s use or distribution of the Tencent Hunyuan Works. +d. Tencent claims no rights in Outputs You generate. You and Your users are solely responsible for Outputs and their subsequent uses. +7. DISCLAIMERS OF WARRANTY AND LIMITATIONS OF LIABILITY. +a. We are not obligated to support, update, provide training for, or develop any further version of the Tencent Hunyuan Works or to grant any license thereto. +b. UNLESS AND ONLY TO THE EXTENT REQUIRED BY APPLICABLE LAW, THE TENCENT HUNYUAN WORKS AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED “AS IS” WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES OF ANY KIND INCLUDING ANY WARRANTIES OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, COURSE OF DEALING, USAGE OF TRADE, OR FITNESS FOR A PARTICULAR PURPOSE. YOU ARE SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING, REPRODUCING, MODIFYING, PERFORMING, DISPLAYING OR DISTRIBUTING ANY OF THE TENCENT HUNYUAN WORKS OR OUTPUTS AND ASSUME ANY AND ALL RISKS ASSOCIATED WITH YOUR OR A THIRD PARTY’S USE OR DISTRIBUTION OF ANY OF THE TENCENT HUNYUAN WORKS OR OUTPUTS AND YOUR EXERCISE OF RIGHTS AND PERMISSIONS UNDER THIS AGREEMENT. +c. TO THE FULLEST EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT SHALL TENCENT OR ITS AFFILIATES BE LIABLE UNDER ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, TORT, NEGLIGENCE, PRODUCTS LIABILITY, OR OTHERWISE, FOR ANY DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, EXEMPLARY, CONSEQUENTIAL OR PUNITIVE DAMAGES, OR LOST PROFITS OF ANY KIND ARISING FROM THIS AGREEMENT OR RELATED TO ANY OF THE TENCENT HUNYUAN WORKS OR OUTPUTS, EVEN IF TENCENT OR ITS AFFILIATES HAVE BEEN ADVISED OF THE POSSIBILITY OF ANY OF THE FOREGOING. +8. SURVIVAL AND TERMINATION. +a. The term of this Agreement shall commence upon Your acceptance of this Agreement or access to the Materials and will continue in full force and effect until terminated in accordance with the terms and conditions herein. +b. We may terminate this Agreement if You breach any of the terms or conditions of this Agreement. Upon termination of this Agreement, You must promptly delete and cease use of the Tencent Hunyuan Works. Sections 6(a), 6(c), 7 and 9 shall survive the termination of this Agreement. +9. GOVERNING LAW AND JURISDICTION. +a. This Agreement and any dispute arising out of or relating to it will be governed by the laws of the Hong Kong Special Administrative Region of the People’s Republic of China, without regard to conflict of law principles, and the UN Convention on Contracts for the International Sale of Goods does not apply to this Agreement. +b. Exclusive jurisdiction and venue for any dispute arising out of or relating to this Agreement will be a court of competent jurisdiction in the Hong Kong Special Administrative Region of the People’s Republic of China, and Tencent and Licensee consent to the exclusive jurisdiction of such court with respect to any such dispute. +  + +EXHIBIT A +ACCEPTABLE USE POLICY + +Tencent reserves the right to update this Acceptable Use Policy from time to time. +Last modified: 2024/5/14 + +Tencent endeavors to promote safe and fair use of its tools and features, including Tencent Hunyuan. You agree not to use Tencent Hunyuan or Model Derivatives: +1. In any way that violates any applicable national, federal, state, local, international or any other law or regulation; +2. To harm Yourself or others; +3. To repurpose or distribute output from Tencent Hunyuan or any Model Derivatives to harm Yourself or others; +4. To override or circumvent the safety guardrails and safeguards We have put in place; +5. For the purpose of exploiting, harming or attempting to exploit or harm minors in any way; +6. To generate or disseminate verifiably false information and/or content with the purpose of harming others or influencing elections; +7. To generate or facilitate false online engagement, including fake reviews and other means of fake online engagement; +8. To intentionally defame, disparage or otherwise harass others; +9. To generate and/or disseminate malware (including ransomware) or any other content to be used for the purpose of harming electronic systems; +10. To generate or disseminate personal identifiable information with the purpose of harming others; +11. To generate or disseminate information (including images, code, posts, articles), and place the information in any public context (including –through the use of bot generated tweets), without expressly and conspicuously identifying that the information and/or content is machine generated; +12. To impersonate another individual without consent, authorization, or legal right; +13. To make high-stakes automated decisions in domains that affect an individual’s safety, rights or wellbeing (e.g., law enforcement, migration, medicine/health, management of critical infrastructure, safety components of products, essential services, credit, employment, housing, education, social scoring, or insurance); +14. In a manner that violates or disrespects the social ethics and moral standards of other countries or regions; +15. To perform, facilitate, threaten, incite, plan, promote or encourage violent extremism or terrorism; +16. For any use intended to discriminate against or harm individuals or groups based on protected characteristics or categories, online or offline social behavior or known or predicted personal or personality characteristics; +17. To intentionally exploit any of the vulnerabilities of a specific group of persons based on their age, social, physical or mental characteristics, in order to materially distort the behavior of a person pertaining to that group in a manner that causes or is likely to cause that person or another person physical or psychological harm; +18. For military purposes; +19. To engage in the unauthorized or unlicensed practice of any profession including, but not limited to, financial, legal, medical/health, or other professional practices. diff --git a/PyTorch/built-in/mlm/HunyuanDiT/Notice b/PyTorch/built-in/mlm/HunyuanDiT/Notice new file mode 100644 index 0000000000000000000000000000000000000000..6f792e1ed0c2bcde188bd961ec5ae7ef5e64a1e5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/Notice @@ -0,0 +1,546 @@ +Usage and Legal Notices: + +Tencent is pleased to support the open source community by making Tencent Hunyuan available. + +Copyright (C) 2024 THL A29 Limited, a Tencent company. All rights reserved. The below software and/or models in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) THL A29 Limited. + +Tencent Hunyuan is licensed under the Tencent Hunyuan Community License Agreement except for the third-party components listed below. Tencent Hunyuan does not impose any additional limitations beyond what is outlined in the repsective licenses of these third-party components. Users must comply with all terms and conditions of original licenses of these third-party components and must ensure that the usage of the third party components adheres to all relevant laws and regulations. + +For avoidance of doubts, Tencent Hunyuan means the large language models and their software and algorithms, including trained model weights, parameters (including optimizer states), machine-learning model code, inference-enabling code, training-enabling code, fine-tuning enabling code and other elements of the foregoing made publicly available by Tencent in accordance with Tencent Hunyuan Community License Agreement. + + + +Other dependencies and licenses: + +Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. torch +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + + +Terms of the BSD 3-Clause: +-------------------------------------------------------------------- +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +For the license of other third party components, please refer to the following URL: +https://github.com/pytorch/pytorch/blob/v1.13.1/NOTICE + + + +Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. pandas +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2011-2023, Open source contributors. + + +A copy of the BSD 3-Clause is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/pandas-dev/pandas/tree/v2.0.3/LICENSES + + + +Open Source Software Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. numpy +Copyright (c) 2005-2022, NumPy Developers. +All rights reserved. + + +A copy of the BSD 3-Clause is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/numpy/numpy/blob/v1.24.4/LICENSES_bundled.txt + + + +Open Source Software/Model Licensed under the BSD 3-Clause License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. Megatron-LM +Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. + + +A copy of the BSD 3-Clause is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/NVIDIA/Megatron-LM/blob/main/LICENSE + + + +Open Source Software/Models Licensed under the Apache License Version 2.0: +The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) 2023 THL A29 Limited. +-------------------------------------------------------------------- +1. diffusers +Copyright (c) diffusers original author and authors +Please note this software has been modified by Tencent in this distribution. + +2. transformers +Copyright (c) transformers original author and authors + +3. timm +Copyright 2019 Ross Wightman + +4. text-to-text-transfer-transformer +Copyright (c) text-to-text-transfer-transformer original author and authors +Please note this software has been modified by Tencent in this distribution. + +5. pytorch-fid +Copyright (c) pytorch-fid original author and authors +Please note this software has been modified by Tencent in this distribution. + +6. Image-Quality-Assessment-Toolbox +Copyright 2021 Qunliang Xing + +7. accelerate +Copyright (c) accelerate original author and authors + +8. IP-Adapter +Copyright (c) IP-Adapter original author and authors +Please note this software has been modified by Tencent in this distribution. + +9. mT5 +Copyright (c) mT5 original author and authors + +10. Mistral-7B +Copyright (c) 2024 Mistral AI, All rights reserved + +11. peft +Copyright 2023 The HuggingFace Team. All rights reserved. + + +Terms of the Apache License Version 2.0: +-------------------------------------------------------------------- +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and + +You must cause any modified files to carry prominent notices stating that You changed the files; and + +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + + +Open Source Software/Model Licensed under the BSD 3-Clause License: +-------------------------------------------------------------------- +1. torchvision +Copyright (c) Soumith Chintala 2016, +All rights reserved. + +2. flash_attn +Copyright (c) 2022, the respective contributors, as shown by the AUTHORS file. +All rights reserved. + +3. apex +Copyright (c) apex original author and authors + + +A copy of the BSD 3-Clause is included in this file. + + + +Open Source Software Licensed under the HPND License: +-------------------------------------------------------------------- +1. Pillow +Copyright © 2010-2023 by Jeffrey A. Clark (Alex) and contributors. + + +Terms of the HPND License: +-------------------------------------------------------------------- +The Python Imaging Library (PIL) is + + Copyright © 1997-2011 by Secret Labs AB + Copyright © 1995-2011 by Fredrik Lundh + +Pillow is the friendly PIL fork. It is + + Copyright © 2010-2023 by Jeffrey A. Clark (Alex) and contributors. + +Like PIL, Pillow is licensed under the open source HPND License: + +By obtaining, using, and/or copying this software and/or its associated +documentation, you agree that you have read, understood, and will comply +with the following terms and conditions: + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appears in all copies, and that +both that copyright notice and this permission notice appear in supporting +documentation, and that the name of Secret Labs AB or the author not be +used in advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + + + +Open Source Software/Model Licensed under the MIT License: +The below software in this distribution may have been modified by Tencent. +-------------------------------------------------------------------- +1. einops +Copyright (c) 2018 Alex Rogozhnikov + +2. loguru +Copyright (c) 2017 + +3. Chinese-CLIP +Copyright (c) 2012-2022 OFA-Sys Team +Copyright (c) 2012-2022 Gabriel Ilharco, Mitchell Wortsman, Nicholas Carlini, Rohan Taori, Achal Dave, Vaishaal Shankar, John Miller, Hongseok Namkoong, Hannaneh Hajishirzi, Ali Farhadi, Ludwig Schmidt + +4. DeepSpeed +Copyright (c) Microsoft Corporation. + +5. glid-3-xl +Copyright (c) 2021 OpenAI + +6. lazysizes +Copyright (c) 2015 Alexander Farkas + +7. thingsvision +Copyright (c) 2021 Vision and Computational Cognition Group + +8. sd-vae-ft-ema +Copyright (c) sd-vae-ft-ema original author and authors + +9. ComfyUI-Diffusers +Copyright (c) 2023 Limitex + +10. glide-text2im +Copyright (c) 2021 OpenAI + +11. improved-diffusion +Copyright (c) 2021 OpenAI + + +Terms of the MIT License: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +Open Source Software Licensed under the MIT License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. tqdm +Copyright (c) 2013 noamraph + + +A copy of the MIT is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/tqdm/tqdm/blob/v4.66.1/LICENCE + + +Open Source Software/Model Licensed under the MIT License and Other Licenses of the Third-Party Components therein: +The below software in this distribution may have been modified by Tencent. +-------------------------------------------------------------------- +1. generative-models +Copyright (c) 2023 Stability AI + + +A copy of the MIT is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/Stability-AI/generative-models/blob/main/LICENSE-CODE +https://github.com/Stability-AI/generative-models/tree/main/model_licenses + + +Open Source Software/Model Licensed under the Apache License Version 2.0 and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. pyarrow +Copyright 2016-2024 The Apache Software Foundation + + +A copy of the Apache License Version 2.0 is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/apache/arrow/blob/main/NOTICE.txt + + +Open Source Software Licensed under the MIT License and Other Licenses of the Third-Party Components therein: +The below software in this distribution may have been modified by THL A29 Limited ("Tencent Modifications"). All Tencent Modifications are Copyright (C) 2023 THL A29 Limited. +-------------------------------------------------------------------- +1. opencv-python +Copyright (c) Olli-Pekka Heinisuo + + +Terms of the MIT: +-------------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +For the license of other third party components, please refer to the following URL: +https://github.com/opencv/opencv-python/blob/4.x/LICENSE-3RD-PARTY.txt + + +Open Source Software Licensed under the MIT License and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. onnxruntime +Copyright (c) Microsoft Corporation. + + +A copy of the MIT is included in this file. + +For the license of other third party components, please refer to the following URL: +https://github.com/microsoft/onnxruntime/blob/v1.16.3/ThirdPartyNotices.txt + + +Open Source Software/Model Licensed under the Apache License Version 2.0: +The below software in this distribution may have been modified by Tencent. +-------------------------------------------------------------------- +1. dwpose +Copyright 2018-2020 Open-MMLab. +Please note this software has been modified by Tencent in this distribution. + + +Terms of the Apache License Version 2.0: +-------------------------------------------------------------------- +Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and + +You must cause any modified files to carry prominent notices stating that You changed the files; and + +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. + +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + + +Open Source Software Licensed under the License agreement for matplotlib and later and Other Licenses of the Third-Party Components therein: +-------------------------------------------------------------------- +1. matplotlib +Copyright (c) 2012- Matplotlib Development Team; All Rights Reserved + + +Terms of the License agreement for matplotlib versions 1.3.0 and later: +-------------------------------------------------------------------- +License agreement for matplotlib versions 1.3.0 and later +========================================================= + +1. This LICENSE AGREEMENT is between the Matplotlib Development Team +("MDT"), and the Individual or Organization ("Licensee") accessing and +otherwise using matplotlib software in source or binary form and its +associated documentation. + +2. Subject to the terms and conditions of this License Agreement, MDT +hereby grants Licensee a nonexclusive, royalty-free, world-wide license +to reproduce, analyze, test, perform and/or display publicly, prepare +derivative works, distribute, and otherwise use matplotlib +alone or in any derivative version, provided, however, that MDT's +License Agreement and MDT's notice of copyright, i.e., "Copyright (c) +2012- Matplotlib Development Team; All Rights Reserved" are retained in +matplotlib alone or in any derivative version prepared by +Licensee. + +3. In the event Licensee prepares a derivative work that is based on or +incorporates matplotlib or any part thereof, and wants to +make the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to matplotlib . + +4. MDT is making matplotlib available to Licensee on an "AS +IS" basis. MDT MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, MDT MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB +WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. MDT SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR +LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING +MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF +THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between MDT and +Licensee. This License Agreement does not grant permission to use MDT +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using matplotlib , +Licensee agrees to be bound by the terms and conditions of this License +Agreement. + +License agreement for matplotlib versions prior to 1.3.0 +======================================================== + +1. This LICENSE AGREEMENT is between John D. Hunter ("JDH"), and the +Individual or Organization ("Licensee") accessing and otherwise using +matplotlib software in source or binary form and its associated +documentation. + +2. Subject to the terms and conditions of this License Agreement, JDH +hereby grants Licensee a nonexclusive, royalty-free, world-wide license +to reproduce, analyze, test, perform and/or display publicly, prepare +derivative works, distribute, and otherwise use matplotlib +alone or in any derivative version, provided, however, that JDH's +License Agreement and JDH's notice of copyright, i.e., "Copyright (c) +2002-2011 John D. Hunter; All Rights Reserved" are retained in +matplotlib alone or in any derivative version prepared by +Licensee. + +3. In the event Licensee prepares a derivative work that is based on or +incorporates matplotlib or any part thereof, and wants to +make the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to matplotlib. + +4. JDH is making matplotlib available to Licensee on an "AS +IS" basis. JDH MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, JDH MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF MATPLOTLIB +WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. + +5. JDH SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF MATPLOTLIB + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR +LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING +MATPLOTLIB , OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF +THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between JDH and +Licensee. This License Agreement does not grant permission to use JDH +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using matplotlib, +Licensee agrees to be bound by the terms and conditions of this License +Agreement. + + +For the license of other third party components, please refer to the following URL: +https://github.com/matplotlib/matplotlib/blob/v3.7.5/LICENSE diff --git a/PyTorch/built-in/mlm/HunyuanDiT/README.md b/PyTorch/built-in/mlm/HunyuanDiT/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b40bae9818a5e205aa43c2241adb577ff5b9345a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/README.md @@ -0,0 +1,237 @@ + +# HunyuanDiT for PyTorch +# 目录 + +- [HunyuanDiT for PyTorch](#opensora11-for-pytorch) +- [目录](#目录) +- [简介](#简介) + - [模型介绍](#模型介绍) + - [支持任务列表](#支持任务列表) + - [代码实现](#代码实现) +- [STDiT2（在研版本）](#stdit2在研版本) + - [准备训练环境](#准备训练环境) + - [安装模型环境](#安装模型环境) + - [安装昇腾环境](#安装昇腾环境) + - [准备数据集](#准备数据集) + - [训练数据集准备](#训练数据集准备) + - [获取预训练模型](#获取预训练模型) + - [快速开始](#快速开始) + - [训练任务](#训练任务) + - [开始训练](#开始训练) + - [推理任务](#推理任务) + - [开始推理](#开始推理) +- [公网地址说明](#公网地址说明) +- [变更说明](#变更说明) + - [变更](#变更) +- [FAQ](#faq) + +# 简介 +## 模型介绍 + +OpenSora是HPC AI Tech开发的开源高效复现类Sora视频生成方案。OpenSora不仅实现了先进视频生成技术的低成本普及，还提供了一个精简且用户友好的方案，简化了视频制作的复杂性。 +本仓库主要将OpenSora1.1的STDiT2模型的任务迁移到了昇腾NPU上，并进行极致性能优化。 + +## 支持任务列表 + +本仓已经支持以下模型任务类型 + +| 模型 | 任务列表 | 是否支持 | +|:-----------:|:----:|:-----:| +| DiT-g/2 | 在线训练 | ✔ | +| DiT-g/2 | 在线推理 | ✔ | + + + +## 代码实现 + +- 参考实现： + + ``` + url=https://github.com/Tencent/HunyuanDiT + commit_id=(待补充) + ``` + +- 适配昇腾 AI 处理器的实现： + + ``` + url=https://gitee.com/ascend/ModelZoo-PyTorch.git + code_path=PyTorch/built-in/mlm/ + ``` + + +# HunyuanDiT（在研版本） + +## 准备训练环境 + +### 安装模型环境 + + + **表 3** 三方库版本支持表 + + | 三方库 | 支持版本 | + |:-----------:|:------:| + | PyTorch | 2.1.0 | + | TorchVision | 0.16.0 | + + + 在模型根目录下执行以下命令，安装模型对应PyTorch版本需要的依赖。 + + + ```python + source ${cann_install_path}/ascend-toolkit/set_env.sh # 激活cann环境 + cd OpenSora1.1 + pip install -v -e . # 安装本地代码仓，同时自动安装依赖 + ``` + +### 安装昇腾环境 + + 请参考昇腾社区中《[Pytorch框架训练环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/ptes)》文档搭建昇腾环境，本仓已支持表4中软件版本。 + + + **表 4** 昇腾软件版本支持表 + + | 软件类型 | 支持版本 | + | :--------: |:--------:| + | FrameworkPTAdapter | 在研版本 | + | CANN | 在研版本 | + | 昇腾NPU固件 | 在研版本 | + | 昇腾NPU驱动 | 在研版本 | + + + +### 准备数据集 +#### 训练数据集准备 +数据集准备请参考官网，链接如下： +https://github.com/Tencent/HunyuanDiT + +### 获取预训练模型 + +1. 联网情况下，预训练模型会自动下载。 + +2. 无网络时，用户可访问huggingface官网自行下载(https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main) + + +3. 获取对应的预训练模型后，在以下配置文件中将`model`、`vae`的`from_pretrained`参数设置为本地预训练模型绝对路径。 + ```shell + configs/opensora-v1-1/inference/sample.py + configs/opensora-v1-1/train/stage1.py + configs/opensora-v1-1/train/stage2.py + configs/opensora-v1-1/train/stage3.py + ``` + +4. 将下载好的t5模型放在本工程目录下的`DeepFloyd`目录下，组织结构如下： + ``` + $OpenSora1.1 + ├── DeepFloyd + ├── ├── t5-v1_1-xxl + ├── ├── ├── config.json + ├── ├── ├── pytorch_model-00001-of-00002.bin + ├── ├── ├── ... + └── ... + ``` + +## 快速开始 +### 训练任务 +本任务主要以全参微调为主，展示训练任务，包含单机单卡和单机多卡的训练。 +#### 开始训练 +1. 进入解压后的源码包根目录。 + + ``` + cd /${模型文件夹名称} + ``` +2. 准备训练数据。 +按照官网流程，准备对应数据集，放在模型文件夹下，如图： + ``` + dataset + ├──porcelain + │ ├──images/ (image files) + │ │ ├──0.png + │ │ ├──1.png + │ │ ├──...... + │ ├──csvfile/ (csv files containing text-image pairs) + │ │ ├──image_text.csv + │ ├──arrows/ (arrow files containing all necessary training data) + │ │ ├──00000.arrow + │ │ ├──00001.arrow + │ │ ├──...... + │ ├──jsons/ (final training data index files which read data from arrow files during training) + │ │ ├──porcelain.json + │ │ ├──porcelain_mt.json + ``` + +2. 运行训练脚本（待补充）。 + + 用户可以按照自己训练需要进行参数配置，以下给出单卡和多卡的一种训练示例。 + ```shell + bash test/train_full_1p_opensorav1_1.sh --data_path=train_data.csv + # 混合精度BF16，单卡训练，stage1 + ``` + + ```shell + bash test/train_full_8p_opensorav1_1.sh --data_path=train_data.csv + # 混合精度BF16，八卡训练，stage1 + ``` + 对于本模型，可以采用绑核优化，以绑核方式启动。 + 绑核方法参考：https://gitee.com/ascend/att/tree/master/profiler/affinity_cpu_bind + 本模型使用示例如下： + ``` + python3 bind_core.py \ + -app="bash test/train_full_18p_opensorav1_1.sh --data_path=train_data.csv" + ``` + +### 推理任务 +本任务主要以全参微调为主，展示推理任务，包括单卡在线推理。 +#### 开始推理 +1. 进入解压后的源码包根目录。 + + ``` + cd /${模型文件夹名称} + ``` + + +2. 运行推理的脚本（待补充）。 + +- 单机单卡推理 + ```shell + bash test/infer_full_1p_opensorav1_1.sh --ckpt_path=/path/to/OpenSora-STDiT-v2-stage3/model.pth # 混精bf16 在线推理 + ``` +- 推理脚本参数说明如下 + ```shell + test/infer_full_1p_opensorav1_1.sh + --batch_size //设置batch_size + --ckpt_path //推理加载的模型地址 + --prompt //测试用的prompt + --num_frames //生成视频的总帧数 + --img_h //生成视频的宽 + --img_w //生成视频的高 + + scripts/inference.py + config //配置文件路径 + --seed //随机种子 + --ckpt-path //推理加载的模型文件路径 + --batch-size //设置batch_size + --prompt-path //推理使用的prompt文件路径 + --prompt //测试用的prompt + --num-frames //生成视频的总帧数 + --image-size //生成视频的分辨率 + --fps //生成视频的帧率 + --save-dir //输出视频的路径 + --num-sampling-steps //推理的采样步数 + --cfg-scale //无分类器引导的权重系数 + ``` + + + +# 公网地址说明 +代码涉及公网地址参考 public_address_statement.md + +# 变更说明 + +## 变更 + +2024.04.29：OpenSora1.1 STDiT2 bf16训练和推理任务首次发布。 + +# FAQ + + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/README_ORG.md b/PyTorch/built-in/mlm/HunyuanDiT/README_ORG.md new file mode 100644 index 0000000000000000000000000000000000000000..c4b98495b86d230cc3f3818a9726fd2911b8e1a8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/README_ORG.md @@ -0,0 +1,819 @@ + + +

+ +

+ +# Hunyuan-DiT : A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding + +
+   +   +   +   +   +   +   +
+ +----- + +This repo contains PyTorch model definitions, pre-trained weights and inference/sampling code for our paper exploring Hunyuan-DiT. You can find more visualizations on our [project page](https://dit.hunyuan.tencent.com/). + +> [**Hunyuan-DiT: A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding**](https://arxiv.org/abs/2405.08748)
+ +> [**DialogGen: Multi-modal Interactive Dialogue System for Multi-turn Text-to-Image Generation**](https://arxiv.org/abs/2403.08857)
+ +## 🔥🔥🔥 News!! +* Jun 27, 2024: :art: Hunyuan-Captioner is released, providing fine-grained caption for training data. See [mllm](./mllm) for details. +* Jun 27, 2024: :tada: Support LoRa and ControlNet in diffusers. See [diffusers](./diffusers) for details. +* Jun 27, 2024: :tada: 6GB GPU VRAM Inference scripts are released. See [lite](./lite) for details. +* Jun 19, 2024: :tada: ControlNet is released, supporting canny, pose and depth control. See [training/inference codes](#controlnet) for details. +* Jun 13, 2024: :zap: HYDiT-v1.1 version is released, which mitigates the issue of image oversaturation and alleviates the watermark issue. Please check [HunyuanDiT-v1.1 ](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1) and +[Distillation-v1.1](https://huggingface.co/Tencent-Hunyuan/Distillation-v1.1) for more details. +* Jun 13, 2024: :truck: The training code is released, offering [full-parameter training](#full-parameter-training) and [LoRA training](#lora). +* Jun 06, 2024: :tada: Hunyuan-DiT is now available in ComfyUI. Please check [ComfyUI](#using-comfyui) for more details. +* Jun 06, 2024: 🚀 We introduce Distillation version for Hunyuan-DiT acceleration, which achieves **50%** acceleration on NVIDIA GPUs. Please check [Distillation](https://huggingface.co/Tencent-Hunyuan/Distillation) for more details. +* Jun 05, 2024: 🤗 Hunyuan-DiT is now available in 🤗 Diffusers! Please check the [example](#using--diffusers) below. +* Jun 04, 2024: :globe_with_meridians: Support Tencent Cloud links to download the pretrained models! Please check the [links](#-download-pretrained-models) below. +* May 22, 2024: 🚀 We introduce TensorRT version for Hunyuan-DiT acceleration, which achieves **47%** acceleration on NVIDIA GPUs. Please check [TensorRT-libs](https://huggingface.co/Tencent-Hunyuan/TensorRT-libs) for instructions. +* May 22, 2024: 💬 We support demo running multi-turn text2image generation now. Please check the [script](#using-gradio) below. + +## 🤖 Try it on the web + +Welcome to our web-based [**Tencent Hunyuan Bot**](https://hunyuan.tencent.com/bot/chat), where you can explore our innovative products! Just input the suggested prompts below or any other **imaginative prompts containing drawing-related keywords** to activate the Hunyuan text-to-image generation feature. Unleash your creativity and create any picture you desire, **all for free!** + +You can use simple prompts similar to natural language text + +> 画一只穿着西装的猪 +> +> draw a pig in a suit +> +> 生成一幅画，赛博朋克风，跑车 +> +> generate a painting, cyberpunk style, sports car + +or multi-turn language interactions to create the picture. + +> 画一个木制的鸟 +> +> draw a wooden bird +> +> 变成玻璃的 +> +> turn into glass + +## 📑 Open-source Plan + +- Hunyuan-DiT (Text-to-Image Model) + - [x] Inference + - [x] Checkpoints + - [x] Distillation Version + - [x] TensorRT Version + - [x] Training + - [x] Lora + - [x] Controlnet (Pose, Canny, Depth) + - [x] Hunyuan-Captioner (Re-caption the raw image-text pairs) + - [x] 6GB GPU VRAM Inference + - [ ] IP-adapter + - [ ] Hunyuan-DiT-S checkpoints (0.7B model) +- Mllm + - Hunyuan-Captioner + - [x] Inference + - [Hunyuan-DialogGen](https://github.com/Centaurusalpha/DialogGen) (Prompt Enhancement Model) + - [x] Inference +- [X] Web Demo (Gradio) +- [x] Multi-turn T2I Demo (Gradio) +- [X] Cli Demo +- [X] ComfyUI +- [X] Diffusers +- [ ] Kohya +- [ ] WebUI + + +## Contents +- [Hunyuan-DiT](#hunyuan-dit--a-powerful-multi-resolution-diffusion-transformer-with-fine-grained-chinese-understanding) + - [Abstract](#abstract) + - [🎉 Hunyuan-DiT Key Features](#-hunyuan-dit-key-features) + - [Chinese-English Bilingual DiT Architecture](#chinese-english-bilingual-dit-architecture) + - [Multi-turn Text2Image Generation](#multi-turn-text2image-generation) + - [📈 Comparisons](#-comparisons) + - [🎥 Visualization](#-visualization) + - [📜 Requirements](#-requirements) + - [🛠 Dependencies and Installation](#%EF%B8%8F-dependencies-and-installation) + - [🧱 Download Pretrained Models](#-download-pretrained-models) + - [:truck: Training](#truck-training) + - [Data Preparation](#data-preparation) + - [Full Parameter Training](#full-parameter-training) + - [LoRA](#lora) + - [🔑 Inference](#-inference) + - [6GB GPU VRAM Inference](#6gb-gpu-vram-inference) + - [Using Gradio](#using-gradio) + - [Using Diffusers](#using--diffusers) + - [Using Command Line](#using-command-line) + - [More Configurations](#more-configurations) + - [Using ComfyUI](#using-comfyui) + - [:building_construction: Adatper](#building_construction-adapter) + - [ControlNet](#controlnet) + - [:art: Hunyuan-Captioner](#art-hunyuan-captioner) + - [🚀 Acceleration (for Linux)](#-acceleration-for-linux) + - [🔗 BibTeX](#-bibtex) + +## **Abstract** + +We present Hunyuan-DiT, a text-to-image diffusion transformer with fine-grained understanding of both English and Chinese. To construct Hunyuan-DiT, we carefully designed the transformer structure, text encoder, and positional encoding. We also build from scratch a whole data pipeline to update and evaluate data for iterative model optimization. For fine-grained language understanding, we train a Multimodal Large Language Model to refine the captions of the images. Finally, Hunyuan-DiT can perform multi-round multi-modal dialogue with users, generating and refining images according to the context. +Through our carefully designed holistic human evaluation protocol with more than 50 professional human evaluators, Hunyuan-DiT sets a new state-of-the-art in Chinese-to-image generation compared with other open-source models. + + +## 🎉 **Hunyuan-DiT Key Features** +### **Chinese-English Bilingual DiT Architecture** +Hunyuan-DiT is a diffusion model in the latent space, as depicted in figure below. Following the Latent Diffusion Model, we use a pre-trained Variational Autoencoder (VAE) to compress the images into low-dimensional latent spaces and train a diffusion model to learn the data distribution with diffusion models. Our diffusion model is parameterized with a transformer. To encode the text prompts, we leverage a combination of pre-trained bilingual (English and Chinese) CLIP and multilingual T5 encoder. +

+ +

+ +### Multi-turn Text2Image Generation +Understanding natural language instructions and performing multi-turn interaction with users are important for a +text-to-image system. It can help build a dynamic and iterative creation process that bring the user’s idea into reality +step by step. In this section, we will detail how we empower Hunyuan-DiT with the ability to perform multi-round +conversations and image generation. We train MLLM to understand the multi-round user dialogue +and output the new text prompt for image generation. +

+ +

+ +## 📈 Comparisons +In order to comprehensively compare the generation capabilities of HunyuanDiT and other models, we constructed a 4-dimensional test set, including Text-Image Consistency, Excluding AI Artifacts, Subject Clarity, Aesthetic. More than 50 professional evaluators performs the evaluation. + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Model Open Source Text-Image Consistency (%) Excluding AI Artifacts (%) Subject Clarity (%) Aesthetics (%) Overall (%)
SDXL ✔ 64.3 60.6 91.1 76.3 42.7
PixArt-α ✔ 68.3 60.9 93.2 77.5 45.5
Playground 2.5 ✔ 71.9 70.8 94.9 83.3 54.3
SD 3 ✘ 77.1 69.3 94.6 82.5 56.7
MidJourney v6✘ 73.5 80.2 93.5 87.2 63.3
DALL-E 3✘ 83.9 80.3 96.5 89.4 71.0
Hunyuan-DiT✔ 74.2 74.3 95.4 86.6 59.0
+

+ +## 🎥 Visualization + +* **Chinese Elements** +

+ +

+ +* **Long Text Input** + + +

+ +

+ +* **Multi-turn Text2Image Generation** + +https://github.com/Tencent/tencent.github.io/assets/27557933/94b4dcc3-104d-44e1-8bb2-dc55108763d1 + + + +--- + +## 📜 Requirements + +This repo consists of DialogGen (a prompt enhancement model) and Hunyuan-DiT (a text-to-image model). + +The following table shows the requirements for running the models (batch size = 1): + +| Model | --load-4bit (DialogGen) | GPU Peak Memory | GPU | +|:-----------------------:|:-----------------------:|:---------------:|:---------------:| +| DialogGen + Hunyuan-DiT | ✘ | 32G | A100 | +| DialogGen + Hunyuan-DiT | ✔ | 22G | A100 | +| Hunyuan-DiT | - | 11G | A100 | +| Hunyuan-DiT | - | 14G | RTX3090/RTX4090 | + +* An NVIDIA GPU with CUDA support is required. + * We have tested V100 and A100 GPUs. + * **Minimum**: The minimum GPU memory required is 11GB. + * **Recommended**: We recommend using a GPU with 32GB of memory for better generation quality. +* Tested operating system: Linux + +## 🛠️ Dependencies and Installation + +Begin by cloning the repository: +```shell +git clone https://github.com/tencent/HunyuanDiT +cd HunyuanDiT +``` + +### Installation Guide for Linux + +We provide an `environment.yml` file for setting up a Conda environment. +Conda's installation instructions are available [here](https://docs.anaconda.com/free/miniconda/index.html). + +We recommend CUDA versions 11.7 and 12.0+. + +```shell +# 1. Prepare conda environment +conda env create -f environment.yml + +# 2. Activate the environment +conda activate HunyuanDiT + +# 3. Install pip dependencies +python -m pip install -r requirements.txt + +# 4. (Optional) Install flash attention v2 for acceleration (requires CUDA 11.6 or above) +python -m pip install git+https://github.com/Dao-AILab/flash-attention.git@v2.1.2.post3 +``` + +## 🧱 Download Pretrained Models +To download the model, first install the huggingface-cli. (Detailed instructions are available [here](https://huggingface.co/docs/huggingface_hub/guides/cli).) + +```shell +python -m pip install "huggingface_hub[cli]" +``` + +Then download the model using the following commands: + +```shell +# Create a directory named 'ckpts' where the model will be saved, fulfilling the prerequisites for running the demo. +mkdir ckpts +# Use the huggingface-cli tool to download the model. +# The download time may vary from 10 minutes to 1 hour depending on network conditions. +huggingface-cli download Tencent-Hunyuan/HunyuanDiT --local-dir ./ckpts +``` + +
+💡Tips for using huggingface-cli (network problem) + +##### 1. Using HF-Mirror + +If you encounter slow download speeds in China, you can try a mirror to speed up the download process. For example, + +```shell +HF_ENDPOINT=https://hf-mirror.com huggingface-cli download Tencent-Hunyuan/HunyuanDiT --local-dir ./ckpts +``` + +##### 2. Resume Download + +`huggingface-cli` supports resuming downloads. If the download is interrupted, you can just rerun the download +command to resume the download process. + +Note: If an `No such file or directory: 'ckpts/.huggingface/.gitignore.lock'` like error occurs during the download +process, you can ignore the error and rerun the download command. + +
+ +--- + +All models will be automatically downloaded. For more information about the model, visit the Hugging Face repository [here](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT). + +| Model | #Params | Huggingface Download URL | Tencent Cloud Download URL | +|:------------------:|:-------:|:-------------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------:| +| mT5 | 1.6B | [mT5](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/t2i/mt5) | [mT5](https://dit.hunyuan.tencent.com/download/HunyuanDiT/mt5.zip) | +| CLIP | 350M | [CLIP](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/t2i/clip_text_encoder) | [CLIP](https://dit.hunyuan.tencent.com/download/HunyuanDiT/clip_text_encoder.zip) | +| Tokenizer | - | [Tokenizer](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/t2i/tokenizer) | [Tokenizer](https://dit.hunyuan.tencent.com/download/HunyuanDiT/tokenizer.zip) | +| DialogGen | 7.0B | [DialogGen](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/dialoggen) | [DialogGen](https://dit.hunyuan.tencent.com/download/HunyuanDiT/dialoggen.zip) | +| sdxl-vae-fp16-fix | 83M | [sdxl-vae-fp16-fix](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/t2i/sdxl-vae-fp16-fix) | [sdxl-vae-fp16-fix](https://dit.hunyuan.tencent.com/download/HunyuanDiT/sdxl-vae-fp16-fix.zip) | +| Hunyuan-DiT-v1.0 | 1.5B | [Hunyuan-DiT](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/tree/main/t2i/model) | [Hunyuan-DiT-v1.0](https://dit.hunyuan.tencent.com/download/HunyuanDiT/model.zip) | +| Hunyuan-DiT-v1.1 | 1.5B | [Hunyuan-DiT-v1.1](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1/tree/main/t2i/model) | [Hunyuan-DiT-v1.1](https://dit.hunyuan.tencent.com/download/HunyuanDiT/model-v1_1.zip) | +| Data demo | - | - | [Data demo](https://dit.hunyuan.tencent.com/download/HunyuanDiT/data_demo.zip) | + +## :truck: Training + +### Data Preparation + + Refer to the commands below to prepare the training data. + + 1. Install dependencies + + We offer an efficient data management library, named IndexKits, supporting the management of reading hundreds of millions of data during training, see more in [docs](./IndexKits/README.md). + ```shell + # 1 Install dependencies + cd HunyuanDiT + pip install -e ./IndexKits + ``` + 2. Data download + + Feel free to download the [data demo](https://dit.hunyuan.tencent.com/download/HunyuanDiT/data_demo.zip). + ```shell + # 2 Data download + wget -O ./dataset/data_demo.zip https://dit.hunyuan.tencent.com/download/HunyuanDiT/data_demo.zip + unzip ./dataset/data_demo.zip -d ./dataset + mkdir ./dataset/porcelain/arrows ./dataset/porcelain/jsons + ``` + 3. Data conversion + + Create a CSV file for training data with the fields listed in the table below. + + | Fields | Required | Description | Example | + |:---------------:| :------: |:----------------:|:-----------:| + | `image_path` | Required | image path | `./dataset/porcelain/images/0.png` | + | `text_zh` | Required | text | 青花瓷风格，一只蓝色的鸟儿站在蓝色的花瓶上，周围点缀着白色花朵，背景是白色 | + | `md5` | Optional | image md5 (Message Digest Algorithm 5) | `d41d8cd98f00b204e9800998ecf8427e` | + | `width` | Optional | image width | `1024 ` | + | `height` | Optional | image height | ` 1024 ` | + + > ⚠️ Optional fields like MD5, width, and height can be omitted. If omitted, the script below will automatically calculate them. This process can be time-consuming when dealing with large-scale training data. + + We utilize [Arrow](https://github.com/apache/arrow) for training data format, offering a standard and efficient in-memory data representation. A conversion script is provided to transform CSV files into Arrow format. + ```shell + # 3 Data conversion + python ./hydit/data_loader/csv2arrow.py ./dataset/porcelain/csvfile/image_text.csv ./dataset/porcelain/arrows + ``` + + 4. Data Selection and Configuration File Creation + + We configure the training data through YAML files. In these files, you can set up standard data processing strategies for filtering, copying, deduplicating, and more regarding the training data. For more details, see [./IndexKits](IndexKits/docs/MakeDataset.md). + + For a sample file, please refer to [file](./dataset/yamls/porcelain.yaml). For a full parameter configuration file, see [file](./IndexKits/docs/MakeDataset.md). + + + 5. Create training data index file using YAML file. + + ```shell + # Single Resolution Data Preparation + idk base -c dataset/yamls/porcelain.yaml -t dataset/porcelain/jsons/porcelain.json + + # Multi Resolution Data Preparation + idk multireso -c dataset/yamls/porcelain_mt.yaml -t dataset/porcelain/jsons/porcelain_mt.json + ``` + + The directory structure for `porcelain` dataset is: + + ```shell + cd ./dataset + + porcelain + ├──images/ (image files) + │ ├──0.png + │ ├──1.png + │ ├──...... + ├──csvfile/ (csv files containing text-image pairs) + │ ├──image_text.csv + ├──arrows/ (arrow files containing all necessary training data) + │ ├──00000.arrow + │ ├──00001.arrow + │ ├──...... + ├──jsons/ (final training data index files which read data from arrow files during training) + │ ├──porcelain.json + │ ├──porcelain_mt.json + ``` + +### Full-parameter Training + + To leverage DeepSpeed in training, you have the flexibility to control **single-node** / **multi-node** training by adjusting parameters such as `--hostfile` and `--master_addr`. For more details, see [link](https://www.deepspeed.ai/getting-started/#resource-configuration-multi-node). + + ```shell + # Single Resolution Training + PYTHONPATH=./ sh hydit/train.sh --index-file dataset/porcelain/jsons/porcelain.json + + # Multi Resolution Training + PYTHONPATH=./ sh hydit/train.sh --index-file dataset/porcelain/jsons/porcelain_mt.json --multireso --reso-step 64 + ``` + +### LoRA + + + +We provide training and inference scripts for LoRA, detailed in the [./lora](./lora/README.md). + + ```shell + # Training for porcelain LoRA. + PYTHONPATH=./ sh lora/train_lora.sh --index-file dataset/porcelain/jsons/porcelain.json + + # Inference using trained LORA weights. + python sample_t2i.py --prompt "青花瓷风格，一只小狗" --no-enhance --lora-ckpt log_EXP/001-lora_porcelain_ema_rank64/checkpoints/0001000.pt + ``` + We offer two types of trained LoRA weights for `porcelain` and `jade`, see details at [links](https://huggingface.co/Tencent-Hunyuan/HYDiT-LoRA) + ```shell + cd HunyuanDiT + # Use the huggingface-cli tool to download the model. + huggingface-cli download Tencent-Hunyuan/HYDiT-LoRA --local-dir ./ckpts/t2i/lora + + # Quick start + python sample_t2i.py --prompt "青花瓷风格，一只猫在追蝴蝶" --no-enhance --load-key ema --lora-ckpt ./ckpts/t2i/lora/porcelain + ``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Examples of training data
Image 0Image 1Image 2Image 3
青花瓷风格，一只蓝色的鸟儿站在蓝色的花瓶上，周围点缀着白色花朵，背景是白色 （Porcelain style, a blue bird stands on a blue vase, surrounded by white flowers, with a white background. +）青花瓷风格，这是一幅蓝白相间的陶瓷盘子，上面描绘着一只狐狸和它的幼崽在森林中漫步，背景是白色 （Porcelain style, this is a blue and white ceramic plate depicting a fox and its cubs strolling in the forest, with a white background.）青花瓷风格，在黑色背景上，一只蓝色的狼站在蓝白相间的盘子上，周围是树木和月亮 （Porcelain style, on a black background, a blue wolf stands on a blue and white plate, surrounded by trees and the moon.）青花瓷风格，在蓝色背景上，一只蓝色蝴蝶和白色花朵被放置在中央 （Porcelain style, on a blue background, a blue butterfly and white flowers are placed in the center.）
Examples of inference results
Image 4Image 5Image 6Image 7
青花瓷风格，苏州园林 （Porcelain style, Suzhou Gardens.）青花瓷风格，一朵荷花 （Porcelain style, a lotus flower.）青花瓷风格，一只羊（Porcelain style, a sheep.）青花瓷风格，一个女孩在雨中跳舞（Porcelain style, a girl dancing in the rain.）
+ + +## 🔑 Inference + +### 6GB GPU VRAM Inference +Running HunyuanDiT in under 6GB GPU VRAM is available now based on [diffusers](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hunyuandit). Here we provide instructions and demo for your quick start. + +> The 6GB version supports Nvidia Ampere architecture series graphics cards such as RTX 3070/3080/4080/4090, A100, and so on. + +The only thing you need do is to install the following library: + +```bash +pip install -U bitsandbytes +pip install git+https://github.com/huggingface/diffusers +pip install torch==2.0.0 +``` + +Then you can enjoy your HunyuanDiT text-to-image journey under 6GB GPU VRAM directly! + +Here is a demo for you. + +```bash +cd HunyuanDiT + +# Quick start +model_id=Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled +prompt=一个宇航员在骑马 +infer_steps=50 +guidance_scale=6 +python3 lite/inference.py ${model_id} ${prompt} ${infer_steps} ${guidance_scale} +``` + +More details can be found in [./lite](lite/README.md). + + +### Using Gradio + +Make sure the conda environment is activated before running the following command. + +```shell +# By default, we start a Chinese UI. +python app/hydit_app.py + +# Using Flash Attention for acceleration. +python app/hydit_app.py --infer-mode fa + +# You can disable the enhancement model if the GPU memory is insufficient. +# The enhancement will be unavailable until you restart the app without the `--no-enhance` flag. +python app/hydit_app.py --no-enhance + +# Start with English UI +python app/hydit_app.py --lang en + +# Start a multi-turn T2I generation UI. +# If your GPU memory is less than 32GB, use '--load-4bit' to enable 4-bit quantization, which requires at least 22GB of memory. +python app/multiTurnT2I_app.py +``` +Then the demo can be accessed through http://0.0.0.0:443. It should be noted that the 0.0.0.0 here needs to be X.X.X.X with your server IP. + +### Using 🤗 Diffusers + +Please install PyTorch version 2.0 or higher in advance to satisfy the requirements of the specified version of the diffusers library. + +Install 🤗 diffusers, ensuring that the version is at least 0.28.1: + +```shell +pip install git+https://github.com/huggingface/diffusers.git +``` +or +```shell +pip install diffusers +``` + +You can generate images with both Chinese and English prompts using the following Python script: +```py +import torch +from diffusers import HunyuanDiTPipeline + +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-Diffusers", torch_dtype=torch.float16) +pipe.to("cuda") + +# You may also use English prompt as HunyuanDiT supports both English and Chinese +# prompt = "An astronaut riding a horse" +prompt = "一个宇航员在骑马" +image = pipe(prompt).images[0] +``` +You can use our distilled model to generate images even faster: + +```py +import torch +from diffusers import HunyuanDiTPipeline + +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-Diffusers-Distilled", torch_dtype=torch.float16) +pipe.to("cuda") + +# You may also use English prompt as HunyuanDiT supports both English and Chinese +# prompt = "An astronaut riding a horse" +prompt = "一个宇航员在骑马" +image = pipe(prompt, num_inference_steps=25).images[0] +``` +More details can be found in [HunyuanDiT-Diffusers-Distilled](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-Diffusers-Distilled) + +**More functions:** For other functions like LoRA and ControlNet, please have a look at the README of [./diffusers](diffusers). + +### Using Command Line + +We provide several commands to quick start: + +```shell +# Prompt Enhancement + Text-to-Image. Torch mode +python sample_t2i.py --prompt "渔舟唱晚" + +# Only Text-to-Image. Torch mode +python sample_t2i.py --prompt "渔舟唱晚" --no-enhance + +# Only Text-to-Image. Flash Attention mode +python sample_t2i.py --infer-mode fa --prompt "渔舟唱晚" + +# Generate an image with other image sizes. +python sample_t2i.py --prompt "渔舟唱晚" --image-size 1280 768 + +# Prompt Enhancement + Text-to-Image. DialogGen loads with 4-bit quantization, but it may loss performance. +python sample_t2i.py --prompt "渔舟唱晚" --load-4bit + +``` + +More example prompts can be found in [example_prompts.txt](example_prompts.txt) + +### More Configurations + +We list some more useful configurations for easy usage: + +| Argument | Default | Description | +|:---------------:|:---------:|:---------------------------------------------------:| +| `--prompt` | None | The text prompt for image generation | +| `--image-size` | 1024 1024 | The size of the generated image | +| `--seed` | 42 | The random seed for generating images | +| `--infer-steps` | 100 | The number of steps for sampling | +| `--negative` | - | The negative prompt for image generation | +| `--infer-mode` | torch | The inference mode (torch, fa, or trt) | +| `--sampler` | ddpm | The diffusion sampler (ddpm, ddim, or dpmms) | +| `--no-enhance` | False | Disable the prompt enhancement model | +| `--model-root` | ckpts | The root directory of the model checkpoints | +| `--load-key` | ema | Load the student model or EMA model (ema or module) | +| `--load-4bit` | Fasle | Load DialogGen model with 4bit quantization | + +### Using ComfyUI + +We provide several commands to quick start: + +```shell +# Download comfyui code +git clone https://github.com/comfyanonymous/ComfyUI.git + +# Install torch, torchvision, torchaudio +pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu117 + +# Install Comfyui essential python package. +cd ComfyUI +pip install -r requirements.txt + +# ComfyUI has been successfully installed! + +# Download model weight as before or link the existing model folder to ComfyUI. +python -m pip install "huggingface_hub[cli]" +mkdir models/hunyuan +huggingface-cli download Tencent-Hunyuan/HunyuanDiT --local-dir ./models/hunyuan/ckpts + +# Move to the ComfyUI custom_nodes folder and copy comfyui-hydit folder from HunyuanDiT Repo. +cd custom_nodes +cp -r ${HunyuanDiT}/comfyui-hydit ./ +cd comfyui-hydit + +# Install some essential python Package. +pip install -r requirements.txt + +# Our tool has been successfully installed! + +# Go to ComfyUI main folder +cd ../.. +# Run the ComfyUI Lauch command +python main.py --listen --port 80 + +# Running ComfyUI successfully! +``` +More details can be found in [./comfyui-hydit](comfyui-hydit/README.md) + +## :building_construction: Adapter + +### ControlNet + +We provide training scripts for ControlNet, detailed in the [./controlnet](./controlnet/README.md). + + ```shell + # Training for canny ControlNet. + PYTHONPATH=./ sh hydit/train_controlnet.sh + ``` + We offer three types of trained ControlNet weights for `canny` `depth` and `pose`, see details at [links](https://huggingface.co/Tencent-Hunyuan/HYDiT-ControlNet) + ```shell + cd HunyuanDiT + # Use the huggingface-cli tool to download the model. + # We recommend using distilled weights as the base model for ControlNet inference, as our provided pretrained weights are trained on them. + huggingface-cli download Tencent-Hunyuan/HYDiT-ControlNet --local-dir ./ckpts/t2i/controlnet + huggingface-cli download Tencent-Hunyuan/Distillation-v1.1 ./pytorch_model_distill.pt --local-dir ./ckpts/t2i/model + + # Quick start + python3 sample_controlnet.py --no-enhance --load-key distill --infer-steps 50 --control-type canny --prompt "在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围" --condition-image-path controlnet/asset/input/canny.jpg --control-weight 1.0 + ``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Condition Input
Canny ControlNet Depth ControlNet Pose ControlNet
在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围
（At night, an ancient Chinese-style lion statue stands in front of the hotel, its eyes gleaming as if guarding the building. The background is the hotel entrance at night, with a close-up, eye-level, and centered composition. This photo presents a realistic photographic style, embodies Chinese sculpture culture, and reveals a mysterious atmosphere.） 		在茂密的森林中，一只黑白相间的熊猫静静地坐在绿树红花中，周围是山川和海洋。背景是白天的森林，光线充足
（In the dense forest, a black and white panda sits quietly in green trees and red flowers, surrounded by mountains, rivers, and the ocean. The background is the forest in a bright environment.） 		一位亚洲女性，身穿绿色上衣，戴着紫色头巾和紫色围巾，站在黑板前。背景是黑板。照片采用近景、平视和居中构图的方式呈现真实摄影风格
（An Asian woman, dressed in a green top, wearing a purple headscarf and a purple scarf, stands in front of a blackboard. The background is the blackboard. The photo is presented in a close-up, eye-level, and centered composition, adopting a realistic photographic style）
Image 0Image 1Image 2
ControlNet Output
Image 3Image 4Image 5
+ +## :art: Hunyuan-Captioner +Hunyuan-Captioner meets the need of text-to-image techniques by maintaining a high degree of image-text consistency. It can generate high-quality image descriptions from a variety of angles, including object description, objects relationships, background information, image style, etc. Our code is based on [LLaVA](https://github.com/haotian-liu/LLaVA) implementation. + +### Examples + +Image 3 + +### Instructions +a. Install dependencies + +The dependencies and installation are basically the same as the [**base model**](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1). + +b. Data download +```shell +cd HunyuanDiT +wget -O ./dataset/data_demo.zip https://dit.hunyuan.tencent.com/download/HunyuanDiT/data_demo.zip +unzip ./dataset/data_demo.zip -d ./dataset +mkdir ./dataset/porcelain/arrows ./dataset/porcelain/jsons +``` + +c. Model download +```shell +# Use the huggingface-cli tool to download the model. +huggingface-cli download Tencent-Hunyuan/HunyuanCaptioner --local-dir ./ckpts/captioner +``` + +### Inference + +Current supported prompt templates: + +|Mode | Prompt template |Description | +| --- | --- | --- | +|caption_zh | 描述这张图片 |Caption in Chinese | +|insert_content | 根据提示词“{}”,描述这张图片 |Insert specific knowledge into caption| +|caption_en | Please describe the content of this image |Caption in English | +| | | | + + +a. Single picture inference in Chinese + +```bash +python mllm/caption_demo.py --mode "caption_zh" --image_file "mllm/images/demo1.png" --model_path "./ckpts/captioner" +``` + +b. Insert specific knowledge into caption + +```bash +python mllm/caption_demo.py --mode "insert_content" --content "宫保鸡丁" --image_file "mllm/images/demo2.png" --model_path "./ckpts/captioner" +``` + +c. Single picture inference in English + +```bash +python mllm/caption_demo.py --mode "caption_en" --image_file "mllm/images/demo3.png" --model_path "./ckpts/captioner" +``` + +d. Multiple pictures inference in Chinese + +```bash +### Convert multiple pictures to csv file. +python mllm/make_csv.py --img_dir "mllm/images" --input_file "mllm/images/demo.csv" + +### Multiple pictures inference +python mllm/caption_demo.py --mode "caption_zh" --input_file "mllm/images/demo.csv" --output_file "mllm/images/demo_res.csv" --model_path "./ckpts/captioner" +``` + +(Optional) To convert the output csv file to Arrow format, please refer to [Data Preparation #3](#data-preparation) for detailed instructions. + + +### Gradio +To launch a Gradio demo locally, please run the following commands one by one. For more detailed instructions, please refer to [LLaVA](https://github.com/haotian-liu/LLaVA). +```bash +cd mllm +python -m llava.serve.controller --host 0.0.0.0 --port 10000 + +python -m llava.serve.gradio_web_server --controller http://0.0.0.0:10000 --model-list-mode reload --port 443 + +python -m llava.serve.model_worker --host 0.0.0.0 --controller http://0.0.0.0:10000 --port 40000 --worker http://0.0.0.0:40000 --model-path "./ckpts/captioner" --model-name LlavaMistral +``` +Then the demo can be accessed through http://0.0.0.0:443. It should be noted that the 0.0.0.0 here needs to be X.X.X.X with your server IP. + +## 🚀 Acceleration (for Linux) + +- We provide TensorRT version of HunyuanDiT for inference acceleration (faster than flash attention). +See [Tencent-Hunyuan/TensorRT-libs](https://huggingface.co/Tencent-Hunyuan/TensorRT-libs) for more details. + +- We provide Distillation version of HunyuanDiT for inference acceleration. +See [Tencent-Hunyuan/Distillation](https://huggingface.co/Tencent-Hunyuan/Distillation) for more details. + +## 🔗 BibTeX +If you find [Hunyuan-DiT](https://arxiv.org/abs/2405.08748) or [DialogGen](https://arxiv.org/abs/2403.08857) useful for your research and applications, please cite using this BibTeX: + +```BibTeX +@misc{li2024hunyuandit, + title={Hunyuan-DiT: A Powerful Multi-Resolution Diffusion Transformer with Fine-Grained Chinese Understanding}, + author={Zhimin Li and Jianwei Zhang and Qin Lin and Jiangfeng Xiong and Yanxin Long and Xinchi Deng and Yingfang Zhang and Xingchao Liu and Minbin Huang and Zedong Xiao and Dayou Chen and Jiajun He and Jiahao Li and Wenyue Li and Chen Zhang and Rongwei Quan and Jianxiang Lu and Jiabin Huang and Xiaoyan Yuan and Xiaoxiao Zheng and Yixuan Li and Jihong Zhang and Chao Zhang and Meng Chen and Jie Liu and Zheng Fang and Weiyan Wang and Jinbao Xue and Yangyu Tao and Jianchen Zhu and Kai Liu and Sihuan Lin and Yifu Sun and Yun Li and Dongdong Wang and Mingtao Chen and Zhichao Hu and Xiao Xiao and Yan Chen and Yuhong Liu and Wei Liu and Di Wang and Yong Yang and Jie Jiang and Qinglin Lu}, + year={2024}, + eprint={2405.08748}, + archivePrefix={arXiv}, + primaryClass={cs.CV} +} + +@article{huang2024dialoggen, + title={DialogGen: Multi-modal Interactive Dialogue System for Multi-turn Text-to-Image Generation}, + author={Huang, Minbin and Long, Yanxin and Deng, Xinchi and Chu, Ruihang and Xiong, Jiangfeng and Liang, Xiaodan and Cheng, Hong and Lu, Qinglin and Liu, Wei}, + journal={arXiv preprint arXiv:2403.08857}, + year={2024} +} +``` + +## Start History + + + + + + Star History Chart + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/app/hydit_app.py b/PyTorch/built-in/mlm/HunyuanDiT/app/hydit_app.py new file mode 100644 index 0000000000000000000000000000000000000000..695fcb3eea4dad50bdb28f6c553a4834c2499706 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/app/hydit_app.py @@ -0,0 +1,169 @@ +import gradio as gr +import pandas as pd +from pathlib import Path +from PIL import Image +import sys +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from hydit.constants import SAMPLER_FACTORY +from sample_t2i import inferencer + +ROOT = Path(__file__).parent.parent +SAMPLERS = list(SAMPLER_FACTORY.keys()) +SIZES = { + "square": (1024, 1024), + "landscape": (768, 1280), + "portrait": (1280, 768), +} + +def get_strings(lang): + lang_file = Path(f"app/lang/{lang}.csv") + strings = pd.read_csv(lang_file, header=0) + strings = strings.set_index("key")['value'].to_dict() + return strings + + +args, gen, enhancer = inferencer() +strings = get_strings(args.lang) + + +def infer( + prompt, + negative_prompt, + seed, + cfg_scale, + infer_steps, + oriW, oriH, + sampler, + size, + enhance +): + if enhance and enhancer is not None: + success, enhanced_prompt = enhancer(prompt) + if not success: + fail_image = Image.open(ROOT / 'app/fail.png') + return fail_image + else: + enhanced_prompt = None + + height, width = SIZES[size] + results = gen.predict(prompt, + height=height, + width=width, + seed=seed, + enhanced_prompt=enhanced_prompt, + negative_prompt=negative_prompt, + infer_steps=infer_steps, + guidance_scale=cfg_scale, + batch_size=1, + src_size_cond=(oriW, oriH), + sampler=sampler, + ) + image = results['images'][0] + return image + + +def ui(): + block = gr.Blocks() + + description = f""" + # {strings['title']} + + ## {strings['desc']} + + """ + + with block: + with gr.Row(): + gr.Markdown(description) + with gr.Row(): + with gr.Column(): + with gr.Row(): + size = gr.Radio( + label=strings['size'], choices=[ + (strings['square'], 'square'), + (strings['landscape'], 'landscape'), + (strings['portrait'], 'portrait'), + ], + value="square" + ) + prompt = gr.Textbox(label=strings['prompt'], value=strings['default prompt'], lines=3) + with gr.Row(): + infer_steps = gr.Slider( + label=strings['infer steps'], minimum=1, maximum=200, value=100, step=1, + ) + seed = gr.Number( + label=strings['seed'], minimum=-1, maximum=1_000_000_000, value=42, step=1, precision=0, + ) + enhance = gr.Checkbox( + label=strings['enhance'], value=enhancer is not None, interactive=True, + ) + + with gr.Accordion( + strings['accordion'], open=False + ): + with gr.Row(): + negative_prompt = gr.Textbox(label=strings['negative_prompt'], + value=gen.default_negative_prompt, + lines=2, + ) + with gr.Row(): + sampler = gr.Dropdown(SAMPLERS, label=strings['sampler'], value="ddpm") + cfg_scale = gr.Slider( + label=strings['cfg'], minimum=1.0, maximum=16.0, value=6.0, step=1 + ) + oriW = gr.Number( + label=strings['width cond'], minimum=1024, maximum=4096, value=1024, step=64, precision=0, + min_width=80, + ) + oriH = gr.Number( + label=strings['height cond'], minimum=1024, maximum=4096, value=1024, step=64, precision=0, + min_width=80, + ) + with gr.Row(): + advanced_button = gr.Button(strings['run']) + with gr.Column(): + default_img = Image.open(ROOT / 'app/default.png') + output_img = gr.Image( + label=strings['generated image'], + interactive=False, + format='png', + value=default_img, + ) + advanced_button.click( + fn=infer, + inputs=[ + prompt, negative_prompt, seed, cfg_scale, infer_steps, + oriW, oriH, sampler, size, enhance, + ], + outputs=output_img, + ) + + with gr.Row(): + gr.Examples([ + ['一只小猫'], + ['现实主义风格，画面主要描述一个巴洛克风格的花瓶，带有金色的装饰边框，花瓶上盛开着各种色彩鲜艳的花，白色背景'], + ['一只聪明的狐狸走在阔叶树林里, 旁边是一条小溪, 细节真实, 摄影'], + ['飞流直下三千尺，疑是银河落九天'], + ['一只长靴猫手持亮银色的宝剑，身着铠甲，眼神坚毅，站在一堆金币上，背景是暗色调的洞穴，图像上有金币的光影点缀。'], + ['麻婆豆腐'], + ['苏州园林'], + ['一颗新鲜的草莓特写，红色的外表，表面布满许多种子，背景是淡绿色的叶子'], + ['请将“杞人忧天”的样子画出来'], + ['枯藤老树昏鸦，小桥流水人家'], + ['湖水清澈，天空湛蓝，阳光灿烂。一只优雅的白天鹅在湖边游泳。它周围有几只小鸭子，看起来非常可爱，整个画面给人一种宁静祥和的感觉。'], + ['一朵鲜艳的红色玫瑰花，花瓣撒有一些水珠，晶莹剔透，特写镜头'], + ['臭豆腐'], + ['九寨沟'], + ['俗语“鲤鱼跃龙门”'], + ['风格是写实，画面主要描述一个亚洲戏曲艺术家正在表演，她穿着华丽的戏服，脸上戴着精致的面具，身姿优雅，背景是古色古香的舞台，镜头是近景'], + ], + [prompt], + label=strings['examples'] + ) + return block + + +if __name__ == "__main__": + interface = ui() + interface.launch(server_name="0.0.0.0", server_port=443, share=True) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/app/multiTurnT2I_app.py b/PyTorch/built-in/mlm/HunyuanDiT/app/multiTurnT2I_app.py new file mode 100644 index 0000000000000000000000000000000000000000..540fdde236638e132a8774e69684febd36b1bdb7 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/app/multiTurnT2I_app.py @@ -0,0 +1,270 @@ +# -- coding: utf-8 -- +#!/usr/bin/env python +import gradio as gr +from PIL import Image +import sys +import os +sys.path.append(os.getcwd()) +import json +import numpy as np +from pathlib import Path +import io +import hashlib +import requests +import base64 +import pandas as pd +from sample_t2i import inferencer +from mllm.dialoggen_demo import init_dialoggen_model, eval_model + +SIZES = { + "正方形(square, 1024x1024)": (1024, 1024), + "风景(landscape, 1280x768)": (768, 1280), + "人像(portrait, 768x1280)": (1280, 768), +} + +global_seed=np.random.randint(0, 10000) + +# Helper Functions +def image_to_base64(image_path): + with open(image_path, "rb") as image_file: + encoded_image = base64.b64encode(image_file.read()).decode() + return encoded_image + +def get_strings(lang): + lang_file = Path(f"app/lang/{lang}.csv") + strings = pd.read_csv(lang_file, header=0) + strings = strings.set_index("key")['value'].to_dict() + return strings + +def get_image_md5(image): + image_data = io.BytesIO() + image.save(image_data, format="PNG") + image_data = image_data.getvalue() + md5_hash = hashlib.md5(image_data).hexdigest() + return md5_hash + + +# mllm调用 +def request_dialogGen(server_url='http://0.0.0.0:8080',history_messages=[], question="画一个木制的鸟",image=""): + if image != "": + image = base64.b64encode(open(image, "rb").read()).decode() + print("history_messages before request",history_messages) + headers = { + 'accept': 'application/json', + 'Content-Type': 'application/json' + } + data = { + "text": question, + "image": image, # "image为空字符串，则进行文本对话" + "history": history_messages, + } + response = requests.post(server_url, headers=headers, json=data) + print("response",response) + response = response.json() + print(response) + response_text = response["result"] + history_messages = response["history"] + print("history_messages before request",history_messages) + return history_messages, response_text + + +# 画图 +def image_generation( + prompt, infer_steps, seed, image_size +): + print(f"prompt sent to T2I model: {prompt}, infer_steps: {infer_steps}, seed: {seed}, size: {image_size}") + height, width = SIZES[image_size] + results = gen.predict(prompt, + height=height, + width=width, + seed=seed, + infer_steps=infer_steps, + batch_size=1, + ) + image = results['images'][0] + file_name = get_image_md5(image) + # Save images + save_dir = Path('results') + save_dir.mkdir(exist_ok=True) + save_path = f'results/multiRound_{file_name}.png' + image.save(save_path) + encoded_image = image_to_base64(save_path) + + return encoded_image + +# 图文对话 +def chat(history_messages, input_text): + + history_messages, response_text = request_dialogGen(history_messages=history_messages, question=input_text) + return history_messages, response_text +# +def pipeline(input_text, state, infer_steps, seed, image_size): + + # 忽略空输入 + if len(input_text) == 0: + return state, state[0] + + conversation = state[0] + history_messages = state[1] + + system_prompt = '请先判断用户的意图，若为画图则在输出前加入<画图>:' + print(f"input history:{history_messages}") + if not isinstance(history_messages, list) and len(history_messages.messages) >= 2: + response, history_messages = enhancer(input_text, return_history=True, history=history_messages, skip_special=True) + else: + response, history_messages = enhancer(input_text, return_history=True, history=history_messages, skip_special=False) + + history_messages.messages[-1][-1] = response + + if '<画图>' in response: + intention_draw = True + else: + intention_draw = False + + print(f"response:{response}") + print("-" * 80) + print(f"history_messages:{history_messages}") + print(f"intention_draw:{intention_draw}") + if intention_draw: + prompt = response.split('<画图>')[-1] + # 画图 + image_url = image_generation(prompt, infer_steps, seed, image_size) + response = f'

{prompt}

' + conversation += [((input_text, response))] + return [conversation, history_messages], conversation + +# 页面设计 +def upload_image(state, image_input): + conversation = state[0] + history_messages = state[1] + input_image = Image.open(image_input.name).resize( + (224, 224)).convert('RGB') + input_image.save(image_input.name) # Overwrite with smaller image. + system_prompt = '请先判断用户的意图，若为画图则在输出前加入<画图>:' + history_messages, response = request_dialogGen(question="这张图描述了什么？",history_messages=history_messages, + image=image_input.name) + conversation += [(f'', response)] + print("conversation" , conversation) + print("history_messages after uploading image", history_messages) + return [conversation, history_messages], conversation + + +def reset(): + global global_seed + global_seed=np.random.randint(0, 10000) + return [[], []], [] + + +def reset_last(state): + conversation, history = state[0], state[1] + conversation = conversation[:-1] + history.messages = history.messages[:-2] + return [conversation, history], conversation + +if __name__ == '__main__': + + # Initialize dialoggen and HunyuanDiT model + args, gen, enhancer = inferencer() + strings = get_strings(args.lang) + + css = """ + #chatbot { min-height: 800px; } + #save-btn { + background-image: linear-gradient(to right bottom, rgba(130,217,244, 0.9), rgba(158,231,214, 1.0)); + } + #save-btn:hover { + background-image: linear-gradient(to right bottom, rgba(110,197,224, 0.9), rgba(138,211,194, 1.0)); + } + #share-btn { + background-image: linear-gradient(to right bottom, rgba(130,217,244, 0.9), rgba(158,231,214, 1.0)); + } + #share-btn:hover { + background-image: linear-gradient(to right bottom, rgba(110,197,224, 0.9), rgba(138,211,194, 1.0)); + } + #gallery { z-index: 999999; } + #gallery img:hover {transform: scale(2.3); z-index: 999999; position: relative; padding-right: 30%; padding-bottom: 30%;} + #gallery button img:hover {transform: none; z-index: 999999; position: relative; padding-right: 0; padding-bottom: 0;} + @media (hover: none) { + #gallery img:hover {transform: none; z-index: 999999; position: relative; padding-right: 0; 0;} + } + .html2canvas-container { width: 3000px !important; height: 3000px !important; } + """ + + + + with gr.Blocks(css=css) as demo: + DESCRIPTION = '''# 多轮对话绘图 Multi-turn Text2Image Generation + 你可以参照[DialogGen](https://arxiv.org/abs/2403.08857)，通过简单的交互式语句来进行历史图片的修改，例如：主体编辑、增加主体、删除主体、背景更换、风格转换、镜头转换、图像合并。 + + (You can modify historical images through simple interactive statements referred to [DialogGen](https://arxiv.org/abs/2403.08857), such as: enity edit, add object, remove object, change background, change style, change lens, and combine images. ) + + 例如，主体编辑 (For example, enity edit) : + ```none + Round1: 画一个木制的鸟 + (Round1: draw a wooden bird) + + Round2: 变成玻璃的 + (Round2: turn into glass) + ``` + ''' + + gr.Markdown(DESCRIPTION) + gr_state = gr.State([[], []]) # conversation, chat_history + + with gr.Row(): + with gr.Column(scale=1, min_width=1000): + with gr.Row(): + chatbot = gr.Chatbot(elem_id="chatbot", label="DialogGen&HunyuanDiT") + with gr.Row(): + infer_steps = gr.Slider( + label='采样步数(sampling steps)', minimum=1, maximum=200, value=100, step=1, + ) + seed = gr.Number( + label='种子(seed)', minimum=-1, maximum=1_000_000_000, value=666, step=1, precision=0, + ) + size_dropdown = gr.Dropdown(choices=["正方形(square, 1024x1024)", "风景(landscape, 1280x768)", "人像(portrait, 768x1280)"], value="正方形(square, 1024x1024)", label="图片尺寸(Image Size)") + + with gr.Row(): + # image_btn = gr.UploadButton("🖼️ Upload Image", file_types=["image"]) + text_input = gr.Textbox(label="提示词(prompt)", placeholder="输入提示词(Type a prompt)") + + with gr.Column(): + submit_btn = gr.Button( + "提交(Submit)", interactive=True, variant="primary") + clear_last_btn = gr.Button("回退(Undo)") + clear_btn = gr.Button("全部重置(Reset All)") + with gr.Row(): + gr.Examples([ + ['画一个木制的鸟'], + ['一只小猫'], + ['现实主义风格，画面主要描述一个巴洛克风格的花瓶，带有金色的装饰边框，花瓶上盛开着各种色彩鲜艳的花，白色背景'], + ['一只聪明的狐狸走在阔叶树林里, 旁边是一条小溪, 细节真实, 摄影'], + ['飞流直下三千尺，疑是银河落九天'], + ['一只长靴猫手持亮银色的宝剑，身着铠甲，眼神坚毅，站在一堆金币上，背景是暗色调的洞穴，图像上有金币的光影点缀。'], + ['麻婆豆腐'], + ['苏州园林'], + ['一颗新鲜的草莓特写，红色的外表，表面布满许多种子，背景是淡绿色的叶子'], + ['枯藤老树昏鸦，小桥流水人家'], + ['湖水清澈，天空湛蓝，阳光灿烂。一只优雅的白天鹅在湖边游泳。它周围有几只小鸭子，看起来非常可爱，整个画面给人一种宁静祥和的感觉。'], + ['一朵鲜艳的红色玫瑰花，花瓣撒有一些水珠，晶莹剔透，特写镜头'], + ['臭豆腐'], + ['九寨沟'], + ['俗语“鲤鱼跃龙门”'], + ['风格是写实，画面主要描述一个亚洲戏曲艺术家正在表演，她穿着华丽的戏服，脸上戴着精致的面具，身姿优雅，背景是古色古香的舞台，镜头是近景'], + ], [text_input], + label=strings['examples'] + ) + gr.Markdown('''

powered by DialogGen and HunyuanDiT

''') + + + text_input.submit(pipeline, [text_input, gr_state, infer_steps, seed, size_dropdown], [gr_state, chatbot]) + text_input.submit(lambda: "", None, text_input) # Reset chatbox. + submit_btn.click(pipeline, [text_input, gr_state, infer_steps, seed, size_dropdown], [gr_state, chatbot]) + submit_btn.click(lambda: "", None, text_input) # Reset chatbox. + + # image_btn.upload(upload_image, [gr_state, image_btn], [gr_state, chatbot]) + clear_last_btn.click(reset_last, [gr_state], [gr_state, chatbot]) + clear_btn.click(reset, [], [gr_state, chatbot]) + + interface = demo + interface.launch(server_name="0.0.0.0", server_port=443, share=False) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/LICENSE b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..150218d6ccaf298ca67d92545ff4d9e5b170ce30 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Limitex + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/README.md b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2a97025c55b765619f3394f6c35af8903938c35f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/README.md @@ -0,0 +1,133 @@ +# comfyui-hydit + +This repository houses a tailored node and workflow designed specifically for HunYuan DIT. The official tests conducted on DDPM, DDIM, and DPMMS have consistently yielded results that align with those obtained through the Diffusers library. However, it's important to note that we cannot assure the consistency of results from other ComfyUI native samplers with the Diffusers inference. We cordially invite users to explore our workflow and are open to receiving any inquiries or suggestions you may have. + +## Overview + + +### Workflow text2image + +![Workflow](img/work_diffusers.png) +[workflow_diffusers](workflow/workflow_diffusers.json) file for HunyuanDiT txt2image with diffusers backend. +![Workflow](img/workflow_ksampler.png) +[workflow_ksampler](workflow/workflow_ksampler.json) file for HunyuanDiT txt2image with ksampler backend. +![Workflow](img/workflow_lora_controlnet.png) +[workflow_lora_controlnet_diffusers](workflow/workflow_lora_controlnet.json) file for HunyuanDiT lora and controlnet model with diffusers backend. + + +## Usage + +Make sure you run the following command inside [ComfyUI](https://github.com/comfyanonymous/ComfyUI) project with our [comfyui-hydit](.) and have correct conda environment. + +```shell +# Please use python 3.10 version with cuda 11.7 +# Download comfyui code +git clone https://github.com/comfyanonymous/ComfyUI.git + +# Install torch, torchvision, torchaudio +pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu117 --default-timeout=100 future + +# Install Comfyui essential python package +cd ComfyUI +pip install -r requirements.txt + +# ComfyUI has been successfully installed! + + +# Move to the ComfyUI custom_nodes folder and copy comfyui-hydit folder from HunyuanDiT Repo. +cd custom_nodes +git clone https://github.com/Tencent/HunyuanDiT.git +cp -r HunyuanDiT/comfyui-hydit ./ +rm -rf HunyuanDiT +cd comfyui-hydit + +# !!! If using windows system !!! +cd custom_nodes +git clone https://github.com/Tencent/HunyuanDiT.git +xcopy /E /I HunyuanDiT\comfyui-hydit comfyui-hydit +rmdir /S /Q HunyuanDiT +cd comfyui-hydit + +# Install some essential python Package. +pip install -r requirements.txt + +# Our tool has been successfully installed! + +# Go to ComfyUI main folder +cd ../.. +# Run the ComfyUI Lauch command +python main.py --listen --port 80 + +# Running ComfyUI successfully! +``` + +## Download weights for diffusers mode + +```shell +python -m pip install "huggingface_hub[cli]" +mkdir models/hunyuan +huggingface-cli download Tencent-Hunyuan/HunyuanDiT-v1.1 --local-dir ./models/hunyuan/ckpts +huggingface-cli download Tencent-Hunyuan/Distillation-v1.1 pytorch_model_distill.pt --local-dir ./models/hunyuan/ckpts/t2i/model +``` + +## Download weights for ksampler mode +Download the [clip encoder](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/blob/main/t2i/clip_text_encoder/pytorch_model.bin) and place it in `ComfyUI/models/clip` +Download the [mt5](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/blob/main/t2i/mt5/pytorch_model.bin) and place it in `ComfyUI/models/t5` +Download the [base model](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/blob/main/t2i/model/pytorch_model_ema.pt) and place it in `ComfyUI/models/checkpoints` +Download the [sdxl vae](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT/blob/main/t2i/sdxl-vae-fp16-fix/diffusion_pytorch_model.bin) and place it in `ComfyUI/models/vae` + + +## Custom Node +Below I'm trying to document all the nodes, thanks for some good work[[1]](#1)[[2]](#2). +#### HunYuan Pipeline Loader +- Loads the full stack of models needed for HunYuanDiT. +- **pipeline_folder_name** is the official weight folder path for hunyuan dit including clip_text_encoder, model, mt5, sdxl-vae-fp16-fix and tokenizer. +- **lora** optional to load lora weight. + +#### HunYuan Checkpoint Loader +- Loads the base model for HunYuanDiT in ksampler backend. +- **model_name** is the weight list of comfyui checkpoint folder. + + +#### HunYuan CLIP Loader +- Loads the clip and mt5 model for HunYuanDiT in ksampler backend. +- **text_encoder_path** is the weight list of comfyui clip model folder. +- **t5_text_encoder_path** is the weight list of comfyui t5 model folder. + +#### HunYuan VAE Loader +- Loads the vae model for HunYuanDiT in ksampler backend. +- **model_name** is the weight list of comfyui vae model folder. + +#### HunYuan Scheduler Loader +- Loads the scheduler algorithm for HunYuanDiT. +- **Input** is the algorithm name including ddpm, ddim and dpmms. +- **Output** is the instance of diffusers.schedulers. + +#### HunYuan Model Makeup +- Assemble the models and scheduler module. +- **Input** is the instance of StableDiffusionPipeline and diffusers.schedulers. +- **Output** is the updated instance of StableDiffusionPipeline. + +#### HunYuan Clip Text Encode +- Assemble the models and scheduler module. +- **Input** is the string of positive and negative prompts. +- **Output** is the converted string for model. + +#### HunYuan Sampler +- Similar with KSampler in ComfyUI. +- **Input** is the instance of StableDiffusionPipeline and some hyper-parameters for sampling. +- **Output** is the generated image. + +#### HunYuan Lora Loader +- Loads the lora model for HunYuanDiT in diffusers backend. +- **lora_name** is the weight list of comfyui lora folder. + +#### HunYuan ControNet Loader +- Loads the controlnet model for HunYuanDiT in diffusers backend. +- **controlnet_path** is the weight list of comfyui controlnet folder. + +## Reference +[1] +https://github.com/Limitex/ComfyUI-Diffusers +[2] +https://github.com/Tencent/HunyuanDiT/pull/59 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ee645d67e7e7dcc87d5a66f104197569e88c2c89 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/__init__.py @@ -0,0 +1,4 @@ +from .nodes import * +#aa = DiffusersSampler() +#print(aa) +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/clip.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/clip.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd2690f8af7402e4897f8cfe7b9d46345469d3b --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/clip.py @@ -0,0 +1,150 @@ +import comfy.supported_models_base +import comfy.latent_formats +import comfy.model_patcher +import comfy.model_base +import comfy.utils +from .hydit.modules.text_encoder import MT5Embedder +from transformers import BertModel, BertTokenizer, AutoTokenizer +import torch +import os +from transformers import T5Config, T5EncoderModel, BertConfig, BertModel +from transformers import AutoTokenizer, modeling_utils +#import pdb + +class CLIP: + def __init__(self, root, text_encoder_path = None, t5_text_encoder_path = None): + self.device = "cuda" if torch.cuda.is_available() else "cpu" + if text_encoder_path == None: + text_encoder_path = os.path.join(root,"clip_text_encoder") + clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + else: + config = BertConfig.from_json_file(os.path.join(os.path.dirname(os.path.realpath(__file__)),"config_clip.json")) + with modeling_utils.no_init_weights(): + clip_text_encoder = BertModel(config) + sd = comfy.utils.load_torch_file(text_encoder_path) + prefix = "bert." + state_dict = {} + for key in sd: + nkey = key + if key.startswith(prefix): + nkey = key[len(prefix):] + state_dict[nkey] = sd[key] + + m, e = clip_text_encoder.load_state_dict(state_dict, strict=False) + if len(m) > 0 or len(e) > 0: + print(f"HYDiT: clip missing {len(m)} keys ({len(e)} extra)") + #clip_text_encoder.load_state_dict(sd, strict=False) + clip_text_encoder.eval().to(self.device) + + tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "tokenizer_clip") + self.tokenizer = HyBertTokenizer(tokenizer_path) + #assert(0) + + if t5_text_encoder_path == None: + t5_text_encoder_path = os.path.join(root,'mt5') + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.tokenizer_t5 = HyT5Tokenizer(embedder_t5.tokenizer, max_length=embedder_t5.max_length) + self.embedder_t5 = embedder_t5.model + else: + + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256, ksampler = True) + self.tokenizer_t5 = HyT5Tokenizer(embedder_t5.tokenizer, max_length=embedder_t5.max_length) + self.embedder_t5 = embedder_t5.model + + + self.cond_stage_model = clip_text_encoder + + + def tokenize(self, text): + tokens = self.tokenizer.tokenize(text) + t5_tokens = self.tokenizer_t5.tokenize(text) + tokens.update(t5_tokens) + return tokens + + def tokenize_t5(self, text): + return self.tokenizer_t5.tokenize(text) + + def encode_from_tokens(self, tokens, return_pooled=False): + attention_mask = tokens['attention_mask'].to(self.device) + with torch.no_grad(): + prompt_embeds = self.cond_stage_model( + tokens['text_input_ids'].to(self.device), + attention_mask=attention_mask + ) + prompt_embeds = prompt_embeds[0] + t5_attention_mask = tokens['t5_attention_mask'].to(self.device) + with torch.no_grad(): + t5_prompt_cond = self.embedder_t5( + tokens['t5_text_input_ids'].to(self.device), + attention_mask=t5_attention_mask + ) + t5_embeds = t5_prompt_cond[0] + + addit_embeds = { + "t5_embeds": t5_embeds, + "attention_mask": attention_mask.float(), + "t5_attention_mask": t5_attention_mask.float() + } + prompt_embeds.addit_embeds = addit_embeds + + if return_pooled: + return prompt_embeds, None + else: + return prompt_embeds + +class HyBertTokenizer: + def __init__(self, tokenizer_path=None, max_length=77, truncation=True, return_attention_mask=True, device='cpu'): + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + self.max_length = self.tokenizer.model_max_length or max_length + self.truncation = truncation + self.return_attention_mask = return_attention_mask + self.device = device + + def tokenize(self, text:str): + text_inputs = self.tokenizer( + text, + padding="max_length", + max_length=self.max_length, + truncation=self.truncation, + return_attention_mask=self.return_attention_mask, + add_special_tokens = True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + tokens = { + 'text_input_ids': text_input_ids, + 'attention_mask': attention_mask + } + return tokens + +class HyT5Tokenizer: + def __init__(self, tokenizer, tokenizer_path = None, max_length=77, truncation=True, return_attention_mask=True, device='cpu'): + if tokenizer_path: + self.tokenizer = AutoTokenizer.from_pretrained(str(tokenizer_path)) + else: + self.tokenizer = tokenizer + + self.max_length = max_length + self.truncation = truncation + self.return_attention_mask = return_attention_mask + self.device = device + + def tokenize(self, text:str): + text_inputs = self.tokenizer( + text, + padding="max_length", + max_length=self.max_length, + truncation=self.truncation, + return_attention_mask=self.return_attention_mask, + add_special_tokens = True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + tokens = { + 't5_text_input_ids': text_input_ids, + 't5_attention_mask': attention_mask + } + return tokens + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_clip.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_clip.json new file mode 100644 index 0000000000000000000000000000000000000000..f6298741b427e1045bae70d7f7c62b53641ad0a5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_clip.json @@ -0,0 +1,34 @@ +{ + "_name_or_path": "hfl/chinese-roberta-wwm-ext-large", + "architectures": [ + "BertModel" + ], + "attention_probs_dropout_prob": 0.1, + "bos_token_id": 0, + "classifier_dropout": null, + "directionality": "bidi", + "eos_token_id": 2, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.1, + "hidden_size": 1024, + "initializer_range": 0.02, + "intermediate_size": 4096, + "layer_norm_eps": 1e-12, + "max_position_embeddings": 512, + "model_type": "bert", + "num_attention_heads": 16, + "num_hidden_layers": 24, + "output_past": true, + "pad_token_id": 0, + "pooler_fc_size": 768, + "pooler_num_attention_heads": 12, + "pooler_num_fc_layers": 3, + "pooler_size_per_head": 128, + "pooler_type": "first_token_transform", + "position_embedding_type": "absolute", + "torch_dtype": "float32", + "transformers_version": "4.22.1", + "type_vocab_size": 2, + "use_cache": true, + "vocab_size": 47020 +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_mt5.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_mt5.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f8cc780dd2b8a4cd8af99b90263c3dc41a842a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/config_mt5.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "/home/patrick/t5/mt5-xl", + "architectures": [ + "MT5ForConditionalGeneration" + ], + "d_ff": 5120, + "d_kv": 64, + "d_model": 2048, + "decoder_start_token_id": 0, + "dropout_rate": 0.1, + "eos_token_id": 1, + "feed_forward_proj": "gated-gelu", + "initializer_factor": 1.0, + "is_encoder_decoder": true, + "layer_norm_epsilon": 1e-06, + "model_type": "mt5", + "num_decoder_layers": 24, + "num_heads": 32, + "num_layers": 24, + "output_past": true, + "pad_token_id": 0, + "relative_attention_num_buckets": 32, + "tie_word_embeddings": false, + "tokenizer_class": "T5Tokenizer", + "transformers_version": "4.10.0.dev0", + "use_cache": true, + "vocab_size": 250112 +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/constant.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..0937ffd874e02edaf318921e07c0ff952e3ee18b --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/constant.py @@ -0,0 +1,10 @@ +import os +from .hydit.constants import SAMPLER_FACTORY + +base_path = os.path.dirname(os.path.realpath(__file__)) +HUNYUAN_PATH = os.path.join(base_path, "..", "..", "models", "hunyuan") +SCHEDULERS_hunyuan = list(SAMPLER_FACTORY.keys()) + + +T5_PATH = os.path.join(base_path, "..", "..", "models", "t5") +LORA_PATH = os.path.join(base_path, "..", "..", "models", "loras") \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/dit.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/dit.py new file mode 100644 index 0000000000000000000000000000000000000000..237f542f0cb9c46c8c1dd46fd11ed04132aefcf0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/dit.py @@ -0,0 +1,145 @@ +import comfy.supported_models_base +import comfy.latent_formats +import comfy.model_patcher +import comfy.model_base +import comfy.utils +from comfy import model_management +from .supported_dit_models import HunYuan_DiT, HYDiT_Model, ModifiedHunYuanDiT +from .clip import CLIP +import os +import folder_paths +import torch + +sampling_settings = { + "beta_schedule" : "linear", + "linear_start" : 0.00085, + "linear_end" : 0.03, + "timesteps" : 1000, +} + +hydit_conf = { + "G/2": { # Seems to be the main one + "unet_config": { + "depth" : 40, + "num_heads" : 16, + "patch_size" : 2, + "hidden_size" : 1408, + "mlp_ratio" : 4.3637, + "input_size": (1024//8, 1024//8), + #"disable_unet_model_creation": True, + }, + "sampling_settings" : sampling_settings, + }, +} + +def load_dit(model_path, output_clip=True, output_model=True, output_vae=True, MODEL_PATH = None, VAE_PATH = None): + if MODEL_PATH: + state_dict = comfy.utils.load_torch_file(MODEL_PATH) + else: + state_dict = comfy.utils.load_torch_file(os.path.join(model_path, "t2i", "model", "pytorch_model_ema.pt")) + + + state_dict = state_dict.get("model", state_dict) + parameters = comfy.utils.calculate_parameters(state_dict) + unet_dtype = model_management.unet_dtype(model_params=parameters) + load_device = comfy.model_management.get_torch_device() + offload_device = comfy.model_management.unet_offload_device() + clip = None, + vae = None + model_patcher = None + + # ignore fp8/etc and use directly for now + manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device) + root = os.path.join(model_path, "t2i") + if manual_cast_dtype: + print(f"DiT: falling back to {manual_cast_dtype}") + unet_dtype = manual_cast_dtype + + #model_conf["unet_config"]["num_classes"] = state_dict["y_embedder.embedding_table.weight"].shape[0] - 1 # adj. for empty + + if output_model: + model_conf = HunYuan_DiT(hydit_conf["G/2"]) + model = HYDiT_Model( + model_conf, + model_type=comfy.model_base.ModelType.V_PREDICTION, + device=model_management.get_torch_device() + ) + + #print(model_conf.unet_config) + #assert(0) + + model.diffusion_model = ModifiedHunYuanDiT(model_conf.dit_conf, **model_conf.unet_config).half().to(load_device) + + model.diffusion_model.load_state_dict(state_dict) + model.diffusion_model.eval() + model.diffusion_model.to(unet_dtype) + + model_patcher = comfy.model_patcher.ModelPatcher( + model, + load_device = load_device, + offload_device = offload_device, + current_device = "cpu", + ) + if output_clip: + clip = CLIP(root) + + if output_vae: + if VAE_PATH: + vae_path = VAE_PATH + else: + vae_path = os.path.join(root, "sdxl-vae-fp16-fix", "diffusion_pytorch_model.safetensors") + #print(vae_path) + sd = comfy.utils.load_torch_file(vae_path) + vae = comfy.sd.VAE(sd=sd) + + return (model_patcher, clip, vae) + + +def load_checkpoint(model_path): + + state_dict = comfy.utils.load_torch_file(model_path) + + + state_dict = state_dict.get("model", state_dict) + parameters = comfy.utils.calculate_parameters(state_dict) + unet_dtype = model_management.unet_dtype(model_params=parameters) + load_device = comfy.model_management.get_torch_device() + offload_device = comfy.model_management.unet_offload_device() + model_patcher = None + + # ignore fp8/etc and use directly for now + manual_cast_dtype = model_management.unet_manual_cast(unet_dtype, load_device) + if manual_cast_dtype: + print(f"DiT: falling back to {manual_cast_dtype}") + unet_dtype = manual_cast_dtype + + + model_conf = HunYuan_DiT(hydit_conf["G/2"]) + model = HYDiT_Model( + model_conf, + model_type=comfy.model_base.ModelType.V_PREDICTION, + device=model_management.get_torch_device() + ) + + + model.diffusion_model = ModifiedHunYuanDiT(model_conf.dit_conf, **model_conf.unet_config).half().to(load_device) + + model.diffusion_model.load_state_dict(state_dict) + model.diffusion_model.eval() + model.diffusion_model.to(unet_dtype) + + model_patcher = comfy.model_patcher.ModelPatcher( + model, + load_device = load_device, + offload_device = offload_device, + current_device = "cpu",) + return (model_patcher,) + + + +def load_vae(vae_path): + + sd = comfy.utils.load_torch_file(vae_path) + vae = comfy.sd.VAE(sd=sd) + + return (vae,) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d17d0033b243cad48f26f5555d3dadac266a8f6 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/__init__.py @@ -0,0 +1,143 @@ +# Openpose +# Original from CMU https://github.com/CMU-Perceptual-Computing-Lab/openpose +# 2nd Edited by https://github.com/Hzzone/pytorch-openpose +# 3rd Edited by ControlNet +# 4th Edited by ControlNet (added face and correct hands) + +import os +import random + +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + +import torch +import numpy as np +from . import util +from .wholebody import Wholebody + + +def draw_pose(pose, H, W, draw_body=True): + bodies = pose['bodies'] + faces = pose['faces'] + hands = pose['hands'] + candidate = bodies['candidate'] + subset = bodies['subset'] + canvas = np.zeros(shape=(H, W, 3), dtype=np.uint8) + + if draw_body: + canvas = util.draw_bodypose(canvas, candidate, subset) + + canvas = util.draw_handpose(canvas, hands) + + canvas = util.draw_facepose(canvas, faces) + + return canvas + + +def keypoint2bbox(keypoints): + valid_keypoints = keypoints[keypoints[:, 0] >= 0] # Ignore keypoints with confidence 0 + if len(valid_keypoints) == 0: + return np.zeros(4) + x_min, y_min = np.min(valid_keypoints, axis=0) + x_max, y_max = np.max(valid_keypoints, axis=0) + + return np.array([x_min, y_min, x_max, y_max]) + +def expand_bboxes(bboxes, expansion_rate=0.5, image_shape=(0, 0)): + expanded_bboxes = [] + for bbox in bboxes: + x_min, y_min, x_max, y_max = map(int, bbox) + width = x_max - x_min + height = y_max - y_min + + # 扩展宽度和高度 + new_width = width * (1 + expansion_rate) + new_height = height * (1 + expansion_rate) + + # 计算新的边界框坐标 + x_min_new = max(0, x_min - (new_width - width) / 2) + x_max_new = min(image_shape[1], x_max + (new_width - width) / 2) + y_min_new = max(0, y_min - (new_height - height) / 2) + y_max_new = min(image_shape[0], y_max + (new_height - height) / 2) + + expanded_bboxes.append([x_min_new, y_min_new, x_max_new, y_max_new]) + + return expanded_bboxes + +def create_mask(image_width, image_height, bboxs): + mask = np.zeros((image_height, image_width), dtype=np.float32) + for bbox in bboxs: + x1, y1, x2, y2 = map(int, bbox) + mask[y1:y2+1, x1:x2+1] = 1.0 + return mask + +threshold = 0.4 +class DWposeDetector: + def __init__(self): + + self.pose_estimation = Wholebody() + + def __call__(self, oriImg, return_index=False, return_yolo=False, return_mask=False): + oriImg = oriImg.copy() + H, W, C = oriImg.shape + with torch.no_grad(): + candidate, subset = self.pose_estimation(oriImg) + candidate = np.zeros((1, 134, 2), dtype=np.float32) if candidate is None else candidate + subset = np.zeros((1, 134), dtype=np.float32) if subset is None else subset + nums, keys, locs = candidate.shape + candidate[..., 0] /= float(W) + candidate[..., 1] /= float(H) + # import pdb; pdb.set_trace() + if return_yolo: + candidate[subset < threshold] = -0.1 + subset = np.expand_dims(subset >= threshold, axis=-1) + keypoint = np.concatenate([candidate, subset], axis=-1) + + # return pose + hand + return np.concatenate([keypoint[:, :18], keypoint[:, 92:]], axis=1) + + body = candidate[:, :18].copy() + body = body.reshape(nums * 18, locs) + score = subset[:, :18] + for i in range(len(score)): + for j in range(len(score[i])): + if score[i][j] > threshold: + score[i][j] = int(18 * i + j) + else: + score[i][j] = -1 + + un_visible = subset < threshold + candidate[un_visible] = -1 + + foot = candidate[:, 18:24] + + faces = candidate[:, 24:92] + + hands1 = candidate[:, 92:113] + hands2 = candidate[:, 113:] + hands = np.vstack([hands1, hands2]) + + # import pdb; pdb.set_trace() + hands_ = hands[hands.max(axis=(1, 2)) > 0] + if len(hands_) == 0: + bbox = np.array([0, 0, 0, 0]).astype(int) + else: + hand_random = random.choice(hands_) + bbox = (keypoint2bbox(hand_random) * H).astype(int) # [0, 1] -> [h, w] + + + + bodies = dict(candidate=body, subset=score) + pose = dict(bodies=bodies, hands=hands, faces=faces) + + if return_mask: + bbox = [(keypoint2bbox(hand) * H).astype(int) for hand in hands_] + # bbox = expand_bboxes(bbox, expansion_rate=0.5, image_shape=(H, W)) + mask = create_mask(W, H, bbox) + return draw_pose(pose, H, W), mask + + if return_index: + return pose + else: + return draw_pose(pose, H, W), bbox + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxdet.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxdet.py new file mode 100644 index 0000000000000000000000000000000000000000..48056f54338de43cbd3430339b957922f37fc8c8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxdet.py @@ -0,0 +1,129 @@ +import cv2 +import numpy as np + +import onnxruntime + + +def nms(boxes, scores, nms_thr): + """Single class NMS implemented in Numpy.""" + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + + +def multiclass_nms(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-aware version.""" + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1 + ) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + + +def demo_postprocess(outputs, img_size, p6=False): + grids = [] + expanded_strides = [] + strides = [8, 16, 32] if not p6 else [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs + + +def preprocess(img, input_size, swap=(2, 0, 1)): + if len(img.shape) == 3: + padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(input_size, dtype=np.uint8) * 114 + + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + +def inference_detector(session, oriImg): + input_shape = (640, 640) + img, ratio = preprocess(oriImg, input_shape) + + ort_inputs = {session.get_inputs()[0].name: img[None, :, :, :]} + output = session.run(None, ort_inputs) + predictions = demo_postprocess(output[0], input_shape)[0] + + boxes = predictions[:, :4] + scores = predictions[:, 4:5] * predictions[:, 5:] + + boxes_xyxy = np.ones_like(boxes) + boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2. + boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2. + boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2. + boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2. + boxes_xyxy /= ratio + dets = multiclass_nms(boxes_xyxy, scores, nms_thr=0.45, score_thr=0.1) + if dets is not None: + final_boxes, final_scores, final_cls_inds = dets[:, :4], dets[:, 4], dets[:, 5] + isscore = final_scores > 0.3 + iscat = final_cls_inds == 0 + isbbox = [i and j for (i, j) in zip(isscore, iscat)] + final_boxes = final_boxes[isbbox] + return final_boxes + else: + return None diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxpose.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxpose.py new file mode 100644 index 0000000000000000000000000000000000000000..79cd4a06241123af81ea22446a4ca8816716443f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/onnxpose.py @@ -0,0 +1,360 @@ +from typing import List, Tuple + +import cv2 +import numpy as np +import onnxruntime as ort + +def preprocess( + img: np.ndarray, out_bbox, input_size: Tuple[int, int] = (192, 256) +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Do preprocessing for RTMPose model inference. + + Args: + img (np.ndarray): Input image in shape. + input_size (tuple): Input image size in shape (w, h). + + Returns: + tuple: + - resized_img (np.ndarray): Preprocessed image. + - center (np.ndarray): Center of image. + - scale (np.ndarray): Scale of image. + """ + # get shape of image + img_shape = img.shape[:2] + out_img, out_center, out_scale = [], [], [] + if len(out_bbox) == 0: + out_bbox = [[0, 0, img_shape[1], img_shape[0]]] + for i in range(len(out_bbox)): + x0 = out_bbox[i][0] + y0 = out_bbox[i][1] + x1 = out_bbox[i][2] + y1 = out_bbox[i][3] + bbox = np.array([x0, y0, x1, y1]) + + # get center and scale + center, scale = bbox_xyxy2cs(bbox, padding=1.25) + + # do affine transformation + resized_img, scale = top_down_affine(input_size, scale, center, img) + + # normalize image + mean = np.array([123.675, 116.28, 103.53]) + std = np.array([58.395, 57.12, 57.375]) + resized_img = (resized_img - mean) / std + + out_img.append(resized_img) + out_center.append(center) + out_scale.append(scale) + + return out_img, out_center, out_scale + + +def inference(sess: ort.InferenceSession, img: np.ndarray) -> np.ndarray: + """Inference RTMPose model. + + Args: + sess (ort.InferenceSession): ONNXRuntime session. + img (np.ndarray): Input image in shape. + + Returns: + outputs (np.ndarray): Output of RTMPose model. + """ + all_out = [] + # build input + for i in range(len(img)): + input = [img[i].transpose(2, 0, 1)] + + # build output + sess_input = {sess.get_inputs()[0].name: input} + sess_output = [] + for out in sess.get_outputs(): + sess_output.append(out.name) + + # run model + outputs = sess.run(sess_output, sess_input) + all_out.append(outputs) + + return all_out + + +def postprocess(outputs: List[np.ndarray], + model_input_size: Tuple[int, int], + center: Tuple[int, int], + scale: Tuple[int, int], + simcc_split_ratio: float = 2.0 + ) -> Tuple[np.ndarray, np.ndarray]: + """Postprocess for RTMPose model output. + + Args: + outputs (np.ndarray): Output of RTMPose model. + model_input_size (tuple): RTMPose model Input image size. + center (tuple): Center of bbox in shape (x, y). + scale (tuple): Scale of bbox in shape (w, h). + simcc_split_ratio (float): Split ratio of simcc. + + Returns: + tuple: + - keypoints (np.ndarray): Rescaled keypoints. + - scores (np.ndarray): Model predict scores. + """ + all_key = [] + all_score = [] + for i in range(len(outputs)): + # use simcc to decode + simcc_x, simcc_y = outputs[i] + keypoints, scores = decode(simcc_x, simcc_y, simcc_split_ratio) + + # rescale keypoints + keypoints = keypoints / model_input_size * scale[i] + center[i] - scale[i] / 2 + all_key.append(keypoints[0]) + all_score.append(scores[0]) + + return np.array(all_key), np.array(all_score) + + +def bbox_xyxy2cs(bbox: np.ndarray, + padding: float = 1.) -> Tuple[np.ndarray, np.ndarray]: + """Transform the bbox format from (x,y,w,h) into (center, scale) + + Args: + bbox (ndarray): Bounding box(es) in shape (4,) or (n, 4), formatted + as (left, top, right, bottom) + padding (float): BBox padding factor that will be multilied to scale. + Default: 1.0 + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: Center (x, y) of the bbox in shape (2,) or + (n, 2) + - np.ndarray[float32]: Scale (w, h) of the bbox in shape (2,) or + (n, 2) + """ + # convert single bbox from (4, ) to (1, 4) + dim = bbox.ndim + if dim == 1: + bbox = bbox[None, :] + + # get bbox center and scale + x1, y1, x2, y2 = np.hsplit(bbox, [1, 2, 3]) + center = np.hstack([x1 + x2, y1 + y2]) * 0.5 + scale = np.hstack([x2 - x1, y2 - y1]) * padding + + if dim == 1: + center = center[0] + scale = scale[0] + + return center, scale + + +def _fix_aspect_ratio(bbox_scale: np.ndarray, + aspect_ratio: float) -> np.ndarray: + """Extend the scale to match the given aspect ratio. + + Args: + scale (np.ndarray): The image scale (w, h) in shape (2, ) + aspect_ratio (float): The ratio of ``w/h`` + + Returns: + np.ndarray: The reshaped image scale in (2, ) + """ + w, h = np.hsplit(bbox_scale, [1]) + bbox_scale = np.where(w > h * aspect_ratio, + np.hstack([w, w / aspect_ratio]), + np.hstack([h * aspect_ratio, h])) + return bbox_scale + + +def _rotate_point(pt: np.ndarray, angle_rad: float) -> np.ndarray: + """Rotate a point by an angle. + + Args: + pt (np.ndarray): 2D point coordinates (x, y) in shape (2, ) + angle_rad (float): rotation angle in radian + + Returns: + np.ndarray: Rotated point in shape (2, ) + """ + sn, cs = np.sin(angle_rad), np.cos(angle_rad) + rot_mat = np.array([[cs, -sn], [sn, cs]]) + return rot_mat @ pt + + +def _get_3rd_point(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """To calculate the affine matrix, three pairs of points are required. This + function is used to get the 3rd point, given 2D points a & b. + + The 3rd point is defined by rotating vector `a - b` by 90 degrees + anticlockwise, using b as the rotation center. + + Args: + a (np.ndarray): The 1st point (x,y) in shape (2, ) + b (np.ndarray): The 2nd point (x,y) in shape (2, ) + + Returns: + np.ndarray: The 3rd point. + """ + direction = a - b + c = b + np.r_[-direction[1], direction[0]] + return c + + +def get_warp_matrix(center: np.ndarray, + scale: np.ndarray, + rot: float, + output_size: Tuple[int, int], + shift: Tuple[float, float] = (0., 0.), + inv: bool = False) -> np.ndarray: + """Calculate the affine transformation matrix that can warp the bbox area + in the input image to the output size. + + Args: + center (np.ndarray[2, ]): Center of the bounding box (x, y). + scale (np.ndarray[2, ]): Scale of the bounding box + wrt [width, height]. + rot (float): Rotation angle (degree). + output_size (np.ndarray[2, ] | list(2,)): Size of the + destination heatmaps. + shift (0-100%): Shift translation ratio wrt the width/height. + Default (0., 0.). + inv (bool): Option to inverse the affine transform direction. + (inv=False: src->dst or inv=True: dst->src) + + Returns: + np.ndarray: A 2x3 transformation matrix + """ + shift = np.array(shift) + src_w = scale[0] + dst_w = output_size[0] + dst_h = output_size[1] + + # compute transformation matrix + rot_rad = np.deg2rad(rot) + src_dir = _rotate_point(np.array([0., src_w * -0.5]), rot_rad) + dst_dir = np.array([0., dst_w * -0.5]) + + # get four corners of the src rectangle in the original image + src = np.zeros((3, 2), dtype=np.float32) + src[0, :] = center + scale * shift + src[1, :] = center + src_dir + scale * shift + src[2, :] = _get_3rd_point(src[0, :], src[1, :]) + + # get four corners of the dst rectangle in the input image + dst = np.zeros((3, 2), dtype=np.float32) + dst[0, :] = [dst_w * 0.5, dst_h * 0.5] + dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir + dst[2, :] = _get_3rd_point(dst[0, :], dst[1, :]) + + if inv: + warp_mat = cv2.getAffineTransform(np.float32(dst), np.float32(src)) + else: + warp_mat = cv2.getAffineTransform(np.float32(src), np.float32(dst)) + + return warp_mat + + +def top_down_affine(input_size: dict, bbox_scale: dict, bbox_center: dict, + img: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get the bbox image as the model input by affine transform. + + Args: + input_size (dict): The input size of the model. + bbox_scale (dict): The bbox scale of the img. + bbox_center (dict): The bbox center of the img. + img (np.ndarray): The original image. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: img after affine transform. + - np.ndarray[float32]: bbox scale after affine transform. + """ + w, h = input_size + warp_size = (int(w), int(h)) + + # reshape bbox to fixed aspect ratio + bbox_scale = _fix_aspect_ratio(bbox_scale, aspect_ratio=w / h) + + # get the affine matrix + center = bbox_center + scale = bbox_scale + rot = 0 + warp_mat = get_warp_matrix(center, scale, rot, output_size=(w, h)) + + # do affine transform + img = cv2.warpAffine(img, warp_mat, warp_size, flags=cv2.INTER_LINEAR) + + return img, bbox_scale + + +def get_simcc_maximum(simcc_x: np.ndarray, + simcc_y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get maximum response location and value from simcc representations. + + Note: + instance number: N + num_keypoints: K + heatmap height: H + heatmap width: W + + Args: + simcc_x (np.ndarray): x-axis SimCC in shape (K, Wx) or (N, K, Wx) + simcc_y (np.ndarray): y-axis SimCC in shape (K, Wy) or (N, K, Wy) + + Returns: + tuple: + - locs (np.ndarray): locations of maximum heatmap responses in shape + (K, 2) or (N, K, 2) + - vals (np.ndarray): values of maximum heatmap responses in shape + (K,) or (N, K) + """ + N, K, Wx = simcc_x.shape + simcc_x = simcc_x.reshape(N * K, -1) + simcc_y = simcc_y.reshape(N * K, -1) + + # get maximum value locations + x_locs = np.argmax(simcc_x, axis=1) + y_locs = np.argmax(simcc_y, axis=1) + locs = np.stack((x_locs, y_locs), axis=-1).astype(np.float32) + max_val_x = np.amax(simcc_x, axis=1) + max_val_y = np.amax(simcc_y, axis=1) + + # get maximum value across x and y axis + mask = max_val_x > max_val_y + max_val_x[mask] = max_val_y[mask] + vals = max_val_x + locs[vals <= 0.] = -1 + + # reshape + locs = locs.reshape(N, K, 2) + vals = vals.reshape(N, K) + + return locs, vals + + +def decode(simcc_x: np.ndarray, simcc_y: np.ndarray, + simcc_split_ratio) -> Tuple[np.ndarray, np.ndarray]: + """Modulate simcc distribution with Gaussian. + + Args: + simcc_x (np.ndarray[K, Wx]): model predicted simcc in x. + simcc_y (np.ndarray[K, Wy]): model predicted simcc in y. + simcc_split_ratio (int): The split ratio of simcc. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: keypoints in shape (K, 2) or (n, K, 2) + - np.ndarray[float32]: scores in shape (K,) or (n, K) + """ + keypoints, scores = get_simcc_maximum(simcc_x, simcc_y) + keypoints /= simcc_split_ratio + + return keypoints, scores + + +def inference_pose(session, out_bbox, oriImg): + h, w = session.get_inputs()[0].shape[2:] + model_input_size = (w, h) + resized_img, center, scale = preprocess(oriImg, out_bbox, model_input_size) + outputs = inference(session, resized_img) + keypoints, scores = postprocess(outputs, model_input_size, center, scale) + + return keypoints, scores \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/util.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/util.py new file mode 100644 index 0000000000000000000000000000000000000000..a1453ab66d3d7a2f161aa6fd45acf70729990863 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/util.py @@ -0,0 +1,298 @@ +import math +import numpy as np +import matplotlib +import cv2 + + +eps = 0.01 + + +def smart_resize(x, s): + Ht, Wt = s + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize(x[:, :, i], s) for i in range(Co)], axis=2) + + +def smart_resize_k(x, fx, fy): + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + Ht, Wt = Ho * fy, Wo * fx + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize_k(x[:, :, i], fx, fy) for i in range(Co)], axis=2) + + +def padRightDownCorner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join(weights_name.split('.')[1:])] + return transfered_model_weights + + +def draw_bodypose(canvas, candidate, subset): + H, W, C = canvas.shape + candidate = np.array(candidate) + subset = np.array(subset) + + stickwidth = 4 + + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], \ + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], \ + [1, 16], [16, 18], [3, 17], [6, 18]] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \ + [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \ + [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] + + for i in range(17): + for n in range(len(subset)): + index = subset[n][np.array(limbSeq[i]) - 1] + if -1 in index: + continue + Y = candidate[index.astype(int), 0] * float(W) + X = candidate[index.astype(int), 1] * float(H) + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) + # import pdb; pdb.set_trace() + cv2.fillConvexPoly(canvas, polygon, colors[i]) + + canvas = (canvas * 0.6).astype(np.uint8) + + for i in range(18): + for n in range(len(subset)): + index = int(subset[n][i]) + if index == -1: + continue + x, y = candidate[index][0:2] + x = int(x * W) + y = int(y * H) + cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1) + + return canvas + + +def draw_handpose(canvas, all_hand_peaks): + H, W, C = canvas.shape + + edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \ + [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] + + for peaks in all_hand_peaks: + peaks = np.array(peaks) + + for ie, e in enumerate(edges): + x1, y1 = peaks[e[0]] + x2, y2 = peaks[e[1]] + x1 = int(x1 * W) + y1 = int(y1 * H) + x2 = int(x2 * W) + y2 = int(y2 * H) + if x1 > eps and y1 > eps and x2 > eps and y2 > eps: + cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie / float(len(edges)), 1.0, 1.0]) * 255, thickness=2) + + for i, keyponit in enumerate(peaks): + x, y = keyponit + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1) + return canvas + + +def draw_facepose(canvas, all_lmks): + H, W, C = canvas.shape + for lmks in all_lmks: + lmks = np.array(lmks) + for lmk in lmks: + x, y = lmk + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 3, (255, 255, 255), thickness=-1) + return canvas + + +# detect hand according to body pose keypoints +# please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp +def handDetect(candidate, subset, oriImg): + # right hand: wrist 4, elbow 3, shoulder 2 + # left hand: wrist 7, elbow 6, shoulder 5 + ratioWristElbow = 0.33 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + # if any of three not detected + has_left = np.sum(person[[5, 6, 7]] == -1) == 0 + has_right = np.sum(person[[2, 3, 4]] == -1) == 0 + if not (has_left or has_right): + continue + hands = [] + #left hand + if has_left: + left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]] + x1, y1 = candidate[left_shoulder_index][:2] + x2, y2 = candidate[left_elbow_index][:2] + x3, y3 = candidate[left_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, True]) + # right hand + if has_right: + right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]] + x1, y1 = candidate[right_shoulder_index][:2] + x2, y2 = candidate[right_elbow_index][:2] + x3, y3 = candidate[right_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, False]) + + for x1, y1, x2, y2, x3, y3, is_left in hands: + # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox + # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]); + # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]); + # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow); + # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder); + # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder); + x = x3 + ratioWristElbow * (x3 - x2) + y = y3 + ratioWristElbow * (y3 - y2) + distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2) + distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) + width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder) + # x-y refers to the center --> offset to topLeft point + # handRectangle.x -= handRectangle.width / 2.f; + # handRectangle.y -= handRectangle.height / 2.f; + x -= width / 2 + y -= width / 2 # width = height + # overflow the image + if x < 0: x = 0 + if y < 0: y = 0 + width1 = width + width2 = width + if x + width > image_width: width1 = image_width - x + if y + width > image_height: width2 = image_height - y + width = min(width1, width2) + # the max hand box value is 20 pixels + if width >= 20: + detect_result.append([int(x), int(y), int(width), is_left]) + + ''' + return value: [[x, y, w, True if left hand else False]]. + width=height since the network require squared input. + x, y is the coordinate of top left + ''' + return detect_result + + +# Written by Lvmin +def faceDetect(candidate, subset, oriImg): + # left right eye ear 14 15 16 17 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + has_head = person[0] > -1 + if not has_head: + continue + + has_left_eye = person[14] > -1 + has_right_eye = person[15] > -1 + has_left_ear = person[16] > -1 + has_right_ear = person[17] > -1 + + if not (has_left_eye or has_right_eye or has_left_ear or has_right_ear): + continue + + head, left_eye, right_eye, left_ear, right_ear = person[[0, 14, 15, 16, 17]] + + width = 0.0 + x0, y0 = candidate[head][:2] + + if has_left_eye: + x1, y1 = candidate[left_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_right_eye: + x1, y1 = candidate[right_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_left_ear: + x1, y1 = candidate[left_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + if has_right_ear: + x1, y1 = candidate[right_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + x, y = x0, y0 + + x -= width + y -= width + + if x < 0: + x = 0 + + if y < 0: + y = 0 + + width1 = width * 2 + width2 = width * 2 + + if x + width > image_width: + width1 = image_width - x + + if y + width > image_height: + width2 = image_height - y + + width = min(width1, width2) + + if width >= 20: + detect_result.append([int(x), int(y), int(width)]) + + return detect_result + + +# get max index of 2d array +def npmax(array): + arrayindex = array.argmax(1) + arrayvalue = array.max(1) + i = arrayvalue.argmax() + j = arrayindex[i] + return i, j diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/wholebody.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/wholebody.py new file mode 100644 index 0000000000000000000000000000000000000000..108fa670ceaee0a263291b87674032ea3b663066 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/dwpose/wholebody.py @@ -0,0 +1,56 @@ +import os +import cv2 +import numpy as np + +import onnxruntime as ort +from .onnxdet import inference_detector +from .onnxpose import inference_pose + +class Wholebody: + def __init__(self): + rank = int(os.getenv('LOCAL_RANK', '0')) + device = f'cuda:{rank}' + providers = ['CPUExecutionProvider'] if device == 'cpu' else [("CUDAExecutionProvider", {"device_id": rank})] + onnx_det = 'hydit/annotator/ckpts/yolox_l.onnx' + onnx_pose = 'hydit/annotator/ckpts/dw-ll_ucoco_384.onnx' + + self.session_det = ort.InferenceSession(path_or_bytes=onnx_det, providers=providers) + self.session_pose = ort.InferenceSession(path_or_bytes=onnx_pose, providers=providers) + + def __call__(self, oriImg): + det_result = inference_detector(self.session_det, oriImg) + if det_result is None: + return None, None + keypoints, scores = inference_pose(self.session_pose, det_result, oriImg) + + keypoints_info = np.concatenate( + (keypoints, scores[..., None]), axis=-1) + # compute neck joint + neck = np.mean(keypoints_info[:, [5, 6]], axis=1) + # neck score when visualizing pred + neck[:, 2:4] = np.logical_and( + keypoints_info[:, 5, 2:4] > 0.3, + keypoints_info[:, 6, 2:4] > 0.3).astype(int) + new_keypoints_info = np.insert( + keypoints_info, 17, neck, axis=1) + mmpose_idx = [ + 17, 6, 8, 10, 7, 9, 12, 14, 16, 13, 15, 2, 1, 4, 3 + ] + openpose_idx = [ + 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17 + ] + new_keypoints_info[:, openpose_idx] = \ + new_keypoints_info[:, mmpose_idx] + keypoints_info = new_keypoints_info + + keypoints, scores = keypoints_info[ + ..., :2], keypoints_info[..., 2] + + return keypoints, scores + + def to(self, device): + self.session_det.set_providers([device]) + self.session_pose.set_providers([device]) + return self + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/glyph.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/glyph.py new file mode 100644 index 0000000000000000000000000000000000000000..8bb67813897e90e051a218fcf157ee7261dc59b5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/glyph.py @@ -0,0 +1,249 @@ +# MIT License +# Copyright (c) 2023 AIGText +# https://github.com/AIGText/GlyphControl-release + +from PIL import Image, ImageFont, ImageDraw +import random +import numpy as np +import cv2 + + +# resize height to image_height first, then shrink or pad to image_width +def resize_and_pad_image(pil_image, image_size): + if isinstance(image_size, (tuple, list)) and len(image_size) == 2: + image_width, image_height = image_size + elif isinstance(image_size, int): + image_width = image_height = image_size + else: + raise ValueError(f"Image size should be int or list/tuple of int not {image_size}") + + while pil_image.size[1] >= 2 * image_height: + pil_image = pil_image.resize( + tuple(x // 2 for x in pil_image.size), resample=Image.BOX + ) + + scale = image_height / pil_image.size[1] + pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC) + + # shrink + if pil_image.size[0] > image_width: + pil_image = pil_image.resize((image_width, image_height), resample=Image.BICUBIC) + + # padding + if pil_image.size[0] < image_width: + img = Image.new(mode="RGBA", size=(image_width, image_height), color=(255, 255, 255, 0)) + width, _ = pil_image.size + img.paste(pil_image, ((image_width - width) // 2, 0)) + pil_image = img + + return pil_image + + +def resize_and_pad_image2(pil_image, image_size): + if isinstance(image_size, (tuple, list)) and len(image_size) == 2: + image_width, image_height = image_size + elif isinstance(image_size, int): + image_width = image_height = image_size + else: + raise ValueError(f"Image size should be int or list/tuple of int not {image_size}") + + while pil_image.size[1] >= 2 * image_height: + pil_image = pil_image.resize( + tuple(x // 2 for x in pil_image.size), resample=Image.BOX + ) + + scale = image_height / pil_image.size[1] + pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC) + + # shrink + if pil_image.size[0] > image_width: + pil_image = pil_image.resize((image_width, image_height), resample=Image.BICUBIC) + + # padding + if pil_image.size[0] < image_width: + img = Image.new(mode="RGB", size=(image_width, image_height), color="white") + width, _ = pil_image.size + img.paste(pil_image, ((image_width - width) // 2, 0)) + pil_image = img + + return pil_image + + +def draw_visual_text(image_size, bboxes, rendered_txt_values, num_rows_values=None, align="center"): + # aligns = ["center", "left", "right"] + """Render text image based on the glyph instructions, i.e., the list of tuples (text, bbox, num_rows). + Currently we just use Calibri font to render glyph images. + """ + # print(image_size, bboxes, rendered_txt_values, num_rows_values, align) + background = Image.new("RGB", image_size, "white") + font = ImageFont.truetype("simfang.ttf", encoding='utf-8', size=512) + if num_rows_values is None: + num_rows_values = [1] * len(rendered_txt_values) + + text_list = [] + for text, bbox, num_rows in zip(rendered_txt_values, bboxes, num_rows_values): + + if len(text) == 0: + continue + + text = text.strip() + if num_rows != 1: + word_tokens = text.split() + num_tokens = len(word_tokens) + index_list = range(1, num_tokens + 1) + if num_tokens > num_rows: + index_list = random.sample(index_list, num_rows) + index_list.sort() + line_list = [] + start_idx = 0 + for index in index_list: + line_list.append( + " ".join(word_tokens + [start_idx: index] + ) + ) + start_idx = index + text = "\n".join(line_list) + + if 'ratio' not in bbox or bbox['ratio'] == 0 or bbox['ratio'] < 1e-4: + image4ratio = Image.new("RGB", (512, 512), "white") + draw = ImageDraw.Draw(image4ratio) + _, _, w, h = draw.textbbox(xy=(0, 0), text=text, font=font) + ratio = w / h + else: + ratio = bbox['ratio'] + + width = int(bbox['width'] * image_size[1]) + height = int(width / ratio) + top_left_x = int(bbox['top_left_x'] * image_size[0]) + top_left_y = int(bbox['top_left_y'] * image_size[1]) + yaw = bbox['yaw'] + + text_image = Image.new("RGB", (512, 512), "white") + draw = ImageDraw.Draw(text_image) + x, y, w, h = draw.textbbox(xy=(0, 0), text=text, font=font) + text_image = Image.new("RGBA", (w, h), (255, 255, 255, 0)) + draw = ImageDraw.Draw(text_image) + draw.text((-x / 2, -y / 2), text, (0, 0, 0, 255), font=font, align=align) + + text_image_ = resize_and_pad_image2(text_image.convert('RGB'), (288, 48)) + # import pdb; pdb.set_trace() + text_list.append(np.array(text_image_)) + + text_image = resize_and_pad_image(text_image, (width, height)) + text_image = text_image.rotate(angle=-yaw, expand=True, fillcolor=(255, 255, 255, 0)) + # image = Image.new("RGB", (w, h), "white") + # draw = ImageDraw.Draw(image) + background.paste(text_image, (top_left_x, top_left_y), mask=text_image) + + return background, text_list + + +# [{'width': 0.1601562201976776, 'ratio': 81.99999451637203, 'yaw': 0.0, 'top_left_x': 0.712890625, 'top_left_y': 0.0}, +# {'width': 0.134765625, 'ratio': 34.5, 'yaw': 0.0, 'top_left_x': 0.4453125, 'top_left_y': 0.0}, + + +def insert_spaces(string, nSpace): + if nSpace == 0: + return string + new_string = "" + for char in string: + new_string += char + " " * nSpace + return new_string[:-nSpace] + + +def draw_glyph(text, font='simfang.ttf'): + if isinstance(font, str): + font = ImageFont.truetype(font, encoding='utf-8', size=512) + g_size = 50 + W, H = (512, 80) + new_font = font.font_variant(size=g_size) + img = Image.new(mode='1', size=(W, H), color=0) + draw = ImageDraw.Draw(img) + left, top, right, bottom = new_font.getbbox(text) + text_width = max(right-left, 5) + text_height = max(bottom - top, 5) + ratio = min(W*0.9/text_width, H*0.9/text_height) + new_font = font.font_variant(size=int(g_size*ratio)) + + text_width, text_height = new_font.getsize(text) + offset_x, offset_y = new_font.getoffset(text) + x = (img.width - text_width) // 2 + y = (img.height - text_height) // 2 - offset_y//2 + draw.text((x, y), text, font=new_font, fill='white') + img = np.expand_dims(np.array(img), axis=2).astype(np.float64) + + return img + + +def draw_glyph2(text, polygon, font='simfang.ttf', vertAng=10, scale=1, width=1024, height=1024, add_space=True): + if isinstance(font, str): + font = ImageFont.truetype(font, encoding='utf-8', size=60) + enlarge_polygon = polygon*scale + rect = cv2.minAreaRect(enlarge_polygon) + box = cv2.boxPoints(rect) + box = np.int0(box) + w, h = rect[1] + angle = rect[2] + if angle < -45: + angle += 90 + angle = -angle + if w < h: + angle += 90 + + vert = False + if (abs(angle) % 90 < vertAng or abs(90-abs(angle) % 90) % 90 < vertAng): + _w = max(box[:, 0]) - min(box[:, 0]) + _h = max(box[:, 1]) - min(box[:, 1]) + if _h >= _w: + vert = True + angle = 0 + + img = np.zeros((height*scale, width*scale, 3), np.uint8) + img = Image.fromarray(img) + + # infer font size + image4ratio = Image.new("RGB", img.size, "white") + draw = ImageDraw.Draw(image4ratio) + _, _, _tw, _th = draw.textbbox(xy=(0, 0), text=text, font=font) + text_w = min(w, h) * (_tw / _th) + if text_w <= max(w, h): + # add space + if len(text) > 1 and not vert and add_space: + for i in range(1, 100): + text_space = insert_spaces(text, i) + _, _, _tw2, _th2 = draw.textbbox(xy=(0, 0), text=text_space, font=font) + if min(w, h) * (_tw2 / _th2) > max(w, h): + break + text = insert_spaces(text, i-1) + font_size = min(w, h)*0.80 + else: + # shrink = 0.75 if vert else 0.85 + shrink = 1.0 + font_size = min(w, h) / (text_w/max(w, h)) * shrink + new_font = font.font_variant(size=int(font_size)) + + left, top, right, bottom = new_font.getbbox(text) + text_width = right-left + text_height = bottom - top + + layer = Image.new('RGBA', img.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + if not vert: + draw.text((rect[0][0]-text_width//2, rect[0][1]-text_height//2-top), text, font=new_font, fill=(255, 255, 255, 255)) + else: + x_s = min(box[:, 0]) + _w//2 - text_height//2 + y_s = min(box[:, 1]) + for c in text: + draw.text((x_s, y_s), c, font=new_font, fill=(255, 255, 255, 255)) + _, _t, _, _b = new_font.getbbox(c) + y_s += _b + + rotated_layer = layer.rotate(angle, expand=1, center=(rect[0][0], rect[0][1])) + + x_offset = int((img.width - rotated_layer.width) / 2) + y_offset = int((img.height - rotated_layer.height) / 2) + img.paste(rotated_layer, (x_offset, y_offset), rotated_layer) + img = np.expand_dims(np.array(img.convert('1')), axis=2).astype(np.float64) + + return img diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/util.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/util.py new file mode 100644 index 0000000000000000000000000000000000000000..e0b217ef9adf92dd5b1fe0debcfb07d0f241a4cb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/annotator/util.py @@ -0,0 +1,98 @@ +import random + +import numpy as np +import cv2 +import os + + +annotator_ckpts_path = os.path.join(os.path.dirname(__file__), 'ckpts') + + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + + +def resize_image(input_image, resolution): + H, W, C = input_image.shape + H = float(H) + W = float(W) + k = float(resolution) / min(H, W) + H *= k + W *= k + H = int(np.round(H / 64.0)) * 64 + W = int(np.round(W / 64.0)) * 64 + img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA) + return img + + +def nms(x, t, s): + x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s) + + f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8) + f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8) + f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8) + + y = np.zeros_like(x) + + for f in [f1, f2, f3, f4]: + np.putmask(y, cv2.dilate(x, kernel=f) == x, x) + + z = np.zeros_like(y, dtype=np.uint8) + z[y > t] = 255 + return z + + +def make_noise_disk(H, W, C, F): + noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) + noise = cv2.resize(noise, (W + 2 * F, H + 2 * F), interpolation=cv2.INTER_CUBIC) + noise = noise[F: F + H, F: F + W] + noise -= np.min(noise) + noise /= np.max(noise) + if C == 1: + noise = noise[:, :, None] + return noise + + +def min_max_norm(x): + x -= np.min(x) + x /= np.maximum(np.max(x), 1e-5) + return x + + +def safe_step(x, step=2): + y = x.astype(np.float32) * float(step + 1) + y = y.astype(np.int32).astype(np.float32) / float(step) + return y + + +def img2mask(img, H, W, low=10, high=90): + assert img.ndim == 3 or img.ndim == 2 + assert img.dtype == np.uint8 + + if img.ndim == 3: + y = img[:, :, random.randrange(0, img.shape[2])] + else: + y = img + + y = cv2.resize(y, (W, H), interpolation=cv2.INTER_CUBIC) + + if random.uniform(0, 1) < 0.5: + y = 255 - y + + return y < np.percentile(y, random.randrange(low, high)) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config.py new file mode 100644 index 0000000000000000000000000000000000000000..07106b9dcd4377252c9427e8355fb1d635cded31 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config.py @@ -0,0 +1,200 @@ + +import argparse + +from .constants import * +from .modules.models import HUNYUAN_DIT_CONFIG, HUNYUAN_DIT_MODELS +from .diffusion.gaussian_diffusion import ModelVarType + +import deepspeed + +def model_var_type(value): + try: + return ModelVarType[value] + except KeyError: + raise ValueError(f"Invalid choice '{value}', valid choices are {[v.name for v in ModelVarType]}") + + +def get_args(default_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("--task-flag", type=str) + + # General Setting + parser.add_argument("--batch-size", type=int, default=1, help="Per-GPU batch size") + parser.add_argument('--seed', type=int, default=42, help="A seed for all the prompts.") + parser.add_argument("--use-fp16", action="store_true", help="Use FP16 precision.") + parser.add_argument("--no-fp16", dest="use_fp16", action="store_false") + parser.set_defaults(use_fp16=True) + parser.add_argument("--extra-fp16", action="store_true", help="Use extra fp16 for vae and text_encoder.") + + # HunYuan-DiT + parser.add_argument("--model", type=str, choices=list(HUNYUAN_DIT_CONFIG.keys()), default='DiT-g/2') + parser.add_argument("--image-size", type=int, nargs='+', default=[1024, 1024], + help='Image size (h, w). If a single value is provided, the image will be treated to ' + '(value, value).') + parser.add_argument("--qk-norm", action="store_true", help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") + parser.set_defaults(qk_norm=True) + parser.add_argument("--norm", type=str, choices=["rms", "laryer"], default="layer", help="Normalization layer type") + parser.add_argument("--text-states-dim", type=int, default=1024, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len", type=int, default=77, help="Token length of CLIP text encoder output.") + parser.add_argument("--text-states-dim-t5", type=int, default=2048, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len-t5", type=int, default=256, help="Token length of T5 text encoder output.") + + # LoRA config + parser.add_argument("--training-parts", type=str, default='all', choices=['all', 'lora'], help="Training parts") + parser.add_argument("--rank", type=int, default=64, help="Rank of LoRA") + parser.add_argument("--lora-ckpt", type=str, default=None, help="LoRA checkpoint") + parser.add_argument('--target-modules', type=str, nargs='+', default=['Wqkv', 'q_proj', 'kv_proj', 'out_proj'], + help="Target modules for LoRA fine tune") + parser.add_argument("--output-merge-path", type=str, default=None, help="Output path for merged model") + + # controlnet config + parser.add_argument("--control-type", type=str, default='canny', choices=['canny', 'depth', 'pose'], help="Controlnet condition type") + parser.add_argument("--control-weight", type=float, default=1.0, help="Controlnet weight") + parser.add_argument("--condition-image-path", type=str, default=None, help="Inference condition image path") + + # Diffusion + parser.add_argument("--learn-sigma", action="store_true", help="Learn extra channels for sigma.") + parser.add_argument("--no-learn-sigma", dest="learn_sigma", action="store_false") + parser.set_defaults(learn_sigma=True) + parser.add_argument("--predict-type", type=str, choices=list(PREDICT_TYPE), default="v_prediction", + help="Diffusion predict type") + parser.add_argument("--noise-schedule", type=str, choices=list(NOISE_SCHEDULES), default="scaled_linear", + help="Noise schedule") + parser.add_argument("--beta-start", type=float, default=0.00085, help="Beta start value") + parser.add_argument("--beta-end", type=float, default=0.03, help="Beta end value") + parser.add_argument("--sigma-small", action="store_true") + parser.add_argument("--mse-loss-weight-type", type=str, default="constant", + help="Min-SNR-gamma. Can be constant or min_snr_ where gamma is a integer. 5 is recommended in the paper.") + parser.add_argument("--model-var-type", type=model_var_type, default=None, help="Specify the model variable type.") + parser.add_argument("--noise-offset", type=float, default=0.0, help="Add extra noise to the input image.") + + # ======================================================================================================== + # Inference + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--prompt", type=str, default="一只小猫", help="The prompt for generating images.") + parser.add_argument("--model-root", type=str, default="ckpts", help="Model root path.") + + # Model setting + parser.add_argument("--load-key", type=str, choices=["ema", "module", "distill", 'merge'], default="ema", help="Load model key for HunYuanDiT checkpoint.") + parser.add_argument('--size-cond', type=int, nargs='+', default=[1024, 1024], + help="Size condition used in sampling. 2 values are required for height and width. " + "If a single value is provided, the image will be treated to (value, value).") + parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios for multi-resolution training.") + parser.add_argument("--cfg-scale", type=float, default=6.0, help="Guidance scale for classifier-free.") + parser.add_argument("--negative", type=str, default=None, help="Negative prompt.") + + # Acceleration + parser.add_argument("--infer-mode", type=str, choices=["fa", "torch", "trt"], default="torch", help="Inference mode") + parser.add_argument("--onnx-workdir", type=str, default="onnx_model", help="Path to save ONNX model") + + # Sampling + parser.add_argument("--sampler", type=str, choices=SAMPLER_FACTORY, default="ddpm", help="Diffusion sampler") + parser.add_argument("--infer-steps", type=int, default=100, help="Inference steps") + + # Prompt enhancement + parser.add_argument("--enhance", action="store_true", help="Enhance prompt with dialoggen.") + parser.add_argument("--no-enhance", dest="enhance", action="store_false") + parser.add_argument("--load-4bit", help="load DialogGen model with 4bit quantization.", action="store_true") + parser.set_defaults(enhance=True) + + # App + parser.add_argument("--lang", type=str, default="zh", choices=["zh", "en"], help="Language") + + + # ======================================================================================================== + # Training + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--epochs", type=int, default=1400) + parser.add_argument("--max-training-steps", type=int, default=10_000_000) + parser.add_argument("--gc-interval", type=int, default=40, + help='To address the memory bottleneck encountered during the preprocessing of the dataset,' + ' memory fragments are reclaimed here by invoking the gc.collect() function.') + parser.add_argument("--log-every", type=int, default=100) + parser.add_argument("--ckpt-every", type=int, default=100_000, help="Create a ckpt every a few steps.") + parser.add_argument("--ckpt-latest-every", type=int, default=10_000, help="Create a ckpt named `latest.pt` every a few steps.") + parser.add_argument("--num-workers", type=int, default=4) + parser.add_argument("--global-seed", type=int, default=1234) + parser.add_argument("--warmup-min-lr", type=float, default=1e-6) + parser.add_argument("--warmup-num-steps", type=float, default=0) + parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") + parser.add_argument("--rope-img", type=str, default=None, choices=['extend', 'base512', 'base1024'], + help="Extend or interpolate the positional embedding of the image.") + parser.add_argument("--rope-real", action="store_true", + help="Use real part and imaginary part separately for RoPE.") + + # Classifier-free + parser.add_argument("--uncond-p", type=float, default=0.2, + help="The probability of dropping training text used for CLIP feature extraction") + parser.add_argument("--uncond-p-t5", type=float, default=0.2, + help="The probability of dropping training text used for mT5 feature extraction") + + # Directory + parser.add_argument("--results-dir", type=str, default="results") + parser.add_argument("--resume", type=str, default=None, help="Resume experiment from a checkpoint") + parser.add_argument("--no-strict", dest="strict", action="store_false", help="Strict loading of checkpoint") + parser.set_defaults(strict=True) + parser.add_argument("--resume-deepspeed", action="store_true", + help="Resume model and ema states from a checkpoint saved by Deepspeed version of DIT.") + parser.add_argument("--resume-split", action="store_true", + help="Resume model and ema states from two checkpoint separated from DeepSpeed ckpt.") + parser.add_argument("--ema-to-module", action="store_true", + help="If true, initialize the module with EMA weights.") + parser.add_argument("--module-to-ema", action="store_true", + help="if true, initialize the ema with Module weights.") + + # Dataset + parser.add_argument("--index-file", type=str, nargs='+', help="During training, provide a JSON file with data indices.") + parser.add_argument("--random-flip", action="store_true", help="Random flip image") + parser.add_argument("--reset-loader", action="store_true", + help="Reset the data loader. It is useful when resuming from a checkpoint but switch to a new dataset.") + parser.add_argument("--multireso", action="store_true", help="Use multi-resolution training.") + parser.add_argument("--reso-step", type=int, default=None, help="Step size for multi-resolution training.") + + # Additional condition + parser.add_argument("--random-shrink-size-cond", action="store_true", + help="Randomly shrink the original size condition.") + parser.add_argument("--merge-src-cond", action="store_true", help="Merge the source condition into a single value.") + + # EMA Model + parser.add_argument("--use-ema", action="store_true", help="Use EMA model") + parser.add_argument("--ema-dtype", type=str, choices=['fp16', 'fp32', 'none'], default="none", + help="EMA data type. If none, use the same data type as the model.") + parser.add_argument("--ema-decay", type=float, default=None, + help="EMA decay rate. If None, use the default value of the model.") + parser.add_argument("--ema-warmup", action="store_true", + help="EMA warmup. If True, perform ema_decay warmup from 0 to ema_decay.") + parser.add_argument("--ema-warmup-power", type=float, default=None, + help="EMA power. If None, use the default value of the model.") + parser.add_argument("--ema-reset-decay", action="store_true", + help="Reset EMA decay to 0 and restart increasing the EMA decay." + "Only works when --ema-warmup is enabled.") + # Acceleration + parser.add_argument("--use-flash-attn", action="store_true", help="During training, " + "flash attention is used to accelerate training.") + parser.add_argument("--no-flash-attn", dest="use_flash_attn", + action="store_false", help="During training, flash attention is not used to accelerate training.") + parser.add_argument("--use-zero-stage", type=int, default=1, help="Use AngelPTM zero stage. Support 2 and 3") + parser.add_argument("--grad-accu-steps", type=int, default=1, help="Gradient accumulation steps.") + + # ======================================================================================================== + # Deepspeed config + # ======================================================================================================== + parser = deepspeed.add_config_arguments(parser) + parser.add_argument('--local_rank', type=int, default=None, + help='local rank passed from distributed launcher.') + parser.add_argument('--deepspeed-optimizer', action='store_true', + help='Switching to the optimizers in DeepSpeed') + parser.add_argument('--remote-device', type=str, default='none', choices=['none', 'cpu', 'nvme'], + help='Remote device for ZeRO-3 initialized parameters.') + parser.add_argument('--zero-stage', type=int, default=1) + parser.add_argument("--async-ema", action="store_true", help="Whether to use multi stream to excut EMA.") + + args = parser.parse_args(default_args) + + return args diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config_comfyui.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config_comfyui.py new file mode 100644 index 0000000000000000000000000000000000000000..b249dd66ed9bbb721998ae6acd1596809a03a281 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/config_comfyui.py @@ -0,0 +1,200 @@ + +import argparse + +from .constants import * +from .modules.models import HUNYUAN_DIT_CONFIG, HUNYUAN_DIT_MODELS +from .diffusion.gaussian_diffusion import ModelVarType + +#import deepspeed + +def model_var_type(value): + try: + return ModelVarType[value] + except KeyError: + raise ValueError(f"Invalid choice '{value}', valid choices are {[v.name for v in ModelVarType]}") + + +def get_args(default_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("--task-flag", type=str) + + # General Setting + parser.add_argument("--batch-size", type=int, default=1, help="Per-GPU batch size") + parser.add_argument('--seed', type=int, default=42, help="A seed for all the prompts.") + parser.add_argument("--use-fp16", action="store_true", help="Use FP16 precision.") + parser.add_argument("--no-fp16", dest="use_fp16", action="store_false") + parser.set_defaults(use_fp16=True) + parser.add_argument("--extra-fp16", action="store_true", help="Use extra fp16 for vae and text_encoder.") + + # HunYuan-DiT + parser.add_argument("--model", type=str, choices=list(HUNYUAN_DIT_CONFIG.keys()), default='DiT-g/2') + parser.add_argument("--image-size", type=int, nargs='+', default=[1024, 1024], + help='Image size (h, w). If a single value is provided, the image will be treated to ' + '(value, value).') + parser.add_argument("--qk-norm", action="store_true", help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") + parser.set_defaults(qk_norm=True) + parser.add_argument("--norm", type=str, choices=["rms", "laryer"], default="layer", help="Normalization layer type") + parser.add_argument("--text-states-dim", type=int, default=1024, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len", type=int, default=77, help="Token length of CLIP text encoder output.") + parser.add_argument("--text-states-dim-t5", type=int, default=2048, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len-t5", type=int, default=256, help="Token length of T5 text encoder output.") + + # LoRA config + parser.add_argument("--training-parts", type=str, default='all', choices=['all', 'lora'], help="Training parts") + parser.add_argument("--rank", type=int, default=64, help="Rank of LoRA") + parser.add_argument("--lora-ckpt", type=str, default=None, help="LoRA checkpoint") + parser.add_argument('--target-modules', type=str, nargs='+', default=['Wqkv', 'q_proj', 'kv_proj', 'out_proj'], + help="Target modules for LoRA fine tune") + parser.add_argument("--output-merge-path", type=str, default=None, help="Output path for merged model") + + # controlnet config + parser.add_argument("--control-type", type=str, default='canny', choices=['canny', 'depth', 'pose'], help="Controlnet condition type") + parser.add_argument("--control-weight", type=float, default=1.0, help="Controlnet weight") + parser.add_argument("--condition-image-path", type=str, default=None, help="Inference condition image path") + + # Diffusion + parser.add_argument("--learn-sigma", action="store_true", help="Learn extra channels for sigma.") + parser.add_argument("--no-learn-sigma", dest="learn_sigma", action="store_false") + parser.set_defaults(learn_sigma=True) + parser.add_argument("--predict-type", type=str, choices=list(PREDICT_TYPE), default="v_prediction", + help="Diffusion predict type") + parser.add_argument("--noise-schedule", type=str, choices=list(NOISE_SCHEDULES), default="scaled_linear", + help="Noise schedule") + parser.add_argument("--beta-start", type=float, default=0.00085, help="Beta start value") + parser.add_argument("--beta-end", type=float, default=0.03, help="Beta end value") + parser.add_argument("--sigma-small", action="store_true") + parser.add_argument("--mse-loss-weight-type", type=str, default="constant", + help="Min-SNR-gamma. Can be constant or min_snr_ where gamma is a integer. 5 is recommended in the paper.") + parser.add_argument("--model-var-type", type=model_var_type, default=None, help="Specify the model variable type.") + parser.add_argument("--noise-offset", type=float, default=0.0, help="Add extra noise to the input image.") + + # ======================================================================================================== + # Inference + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--prompt", type=str, default="一只小猫", help="The prompt for generating images.") + parser.add_argument("--model-root", type=str, default="ckpts", help="Model root path.") + + # Model setting + parser.add_argument("--load-key", type=str, choices=["ema", "module", "distill", 'merge'], default="distill", help="Load model key for HunYuanDiT checkpoint.") + parser.add_argument('--size-cond', type=int, nargs='+', default=[1024, 1024], + help="Size condition used in sampling. 2 values are required for height and width. " + "If a single value is provided, the image will be treated to (value, value).") + parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios for multi-resolution training.") + parser.add_argument("--cfg-scale", type=float, default=6.0, help="Guidance scale for classifier-free.") + parser.add_argument("--negative", type=str, default=None, help="Negative prompt.") + + # Acceleration + parser.add_argument("--infer-mode", type=str, choices=["fa", "torch", "trt"], default="torch", help="Inference mode") + parser.add_argument("--onnx-workdir", type=str, default="onnx_model", help="Path to save ONNX model") + + # Sampling + parser.add_argument("--sampler", type=str, choices=SAMPLER_FACTORY, default="ddpm", help="Diffusion sampler") + parser.add_argument("--infer-steps", type=int, default=100, help="Inference steps") + + # Prompt enhancement + parser.add_argument("--enhance", action="store_true", help="Enhance prompt with dialoggen.") + parser.add_argument("--no-enhance", dest="enhance", action="store_false") + parser.add_argument("--load-4bit", help="load DialogGen model with 4bit quantization.", action="store_true") + parser.set_defaults(enhance=True) + + # App + parser.add_argument("--lang", type=str, default="zh", choices=["zh", "en"], help="Language") + + + # ======================================================================================================== + # Training + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--epochs", type=int, default=1400) + parser.add_argument("--max-training-steps", type=int, default=10_000_000) + parser.add_argument("--gc-interval", type=int, default=40, + help='To address the memory bottleneck encountered during the preprocessing of the dataset,' + ' memory fragments are reclaimed here by invoking the gc.collect() function.') + parser.add_argument("--log-every", type=int, default=100) + parser.add_argument("--ckpt-every", type=int, default=100_000, help="Create a ckpt every a few steps.") + parser.add_argument("--ckpt-latest-every", type=int, default=10_000, help="Create a ckpt named `latest.pt` every a few steps.") + parser.add_argument("--num-workers", type=int, default=4) + parser.add_argument("--global-seed", type=int, default=1234) + parser.add_argument("--warmup-min-lr", type=float, default=1e-6) + parser.add_argument("--warmup-num-steps", type=float, default=0) + parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") + parser.add_argument("--rope-img", type=str, default=None, choices=['extend', 'base512', 'base1024'], + help="Extend or interpolate the positional embedding of the image.") + parser.add_argument("--rope-real", action="store_true", + help="Use real part and imaginary part separately for RoPE.") + + # Classifier-free + parser.add_argument("--uncond-p", type=float, default=0.2, + help="The probability of dropping training text used for CLIP feature extraction") + parser.add_argument("--uncond-p-t5", type=float, default=0.2, + help="The probability of dropping training text used for mT5 feature extraction") + + # Directory + parser.add_argument("--results-dir", type=str, default="results") + parser.add_argument("--resume", type=str, default=None, help="Resume experiment from a checkpoint") + parser.add_argument("--no-strict", dest="strict", action="store_false", help="Strict loading of checkpoint") + parser.set_defaults(strict=True) + parser.add_argument("--resume-deepspeed", action="store_true", + help="Resume model and ema states from a checkpoint saved by Deepspeed version of DIT.") + parser.add_argument("--resume-split", action="store_true", + help="Resume model and ema states from two checkpoint separated from DeepSpeed ckpt.") + parser.add_argument("--ema-to-module", action="store_true", + help="If true, initialize the module with EMA weights.") + parser.add_argument("--module-to-ema", action="store_true", + help="if true, initialize the ema with Module weights.") + + # Dataset + parser.add_argument("--index-file", type=str, nargs='+', help="During training, provide a JSON file with data indices.") + parser.add_argument("--random-flip", action="store_true", help="Random flip image") + parser.add_argument("--reset-loader", action="store_true", + help="Reset the data loader. It is useful when resuming from a checkpoint but switch to a new dataset.") + parser.add_argument("--multireso", action="store_true", help="Use multi-resolution training.") + parser.add_argument("--reso-step", type=int, default=None, help="Step size for multi-resolution training.") + + # Additional condition + parser.add_argument("--random-shrink-size-cond", action="store_true", + help="Randomly shrink the original size condition.") + parser.add_argument("--merge-src-cond", action="store_true", help="Merge the source condition into a single value.") + + # EMA Model + parser.add_argument("--use-ema", action="store_true", help="Use EMA model") + parser.add_argument("--ema-dtype", type=str, choices=['fp16', 'fp32', 'none'], default="none", + help="EMA data type. If none, use the same data type as the model.") + parser.add_argument("--ema-decay", type=float, default=None, + help="EMA decay rate. If None, use the default value of the model.") + parser.add_argument("--ema-warmup", action="store_true", + help="EMA warmup. If True, perform ema_decay warmup from 0 to ema_decay.") + parser.add_argument("--ema-warmup-power", type=float, default=None, + help="EMA power. If None, use the default value of the model.") + parser.add_argument("--ema-reset-decay", action="store_true", + help="Reset EMA decay to 0 and restart increasing the EMA decay." + "Only works when --ema-warmup is enabled.") + # Acceleration + parser.add_argument("--use-flash-attn", action="store_true", help="During training, " + "flash attention is used to accelerate training.") + parser.add_argument("--no-flash-attn", dest="use_flash_attn", + action="store_false", help="During training, flash attention is not used to accelerate training.") + parser.add_argument("--use-zero-stage", type=int, default=1, help="Use AngelPTM zero stage. Support 2 and 3") + parser.add_argument("--grad-accu-steps", type=int, default=1, help="Gradient accumulation steps.") + + # ======================================================================================================== + # Deepspeed config + # ======================================================================================================== + #parser = deepspeed.add_config_arguments(parser) + parser.add_argument('--local_rank', type=int, default=None, + help='local rank passed from distributed launcher.') + parser.add_argument('--deepspeed-optimizer', action='store_true', + help='Switching to the optimizers in DeepSpeed') + parser.add_argument('--remote-device', type=str, default='none', choices=['none', 'cpu', 'nvme'], + help='Remote device for ZeRO-3 initialized parameters.') + parser.add_argument('--zero-stage', type=int, default=1) + parser.add_argument("--async-ema", action="store_true", help="Whether to use multi stream to excut EMA.") + + args = parser.parse_known_args() + + return args diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/constants.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a46921578127b3c81ffea3d1bfb78d06a35421af --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/constants.py @@ -0,0 +1,82 @@ +import torch + +# ======================================================= + +NOISE_SCHEDULES = { + "linear", + "scaled_linear", + "squaredcos_cap_v2", +} + +PREDICT_TYPE = { + "epsilon", + "sample", + "v_prediction", +} + +# ======================================================= + +NEGATIVE_PROMPT = '错误的眼睛，糟糕的人脸，毁容，糟糕的艺术，变形，多余的肢体，模糊的颜色，模糊，重复，病态，残缺，' + +# ======================================================= +TRT_MAX_BATCH_SIZE = 1 +TRT_MAX_WIDTH = 1280 +TRT_MAX_HEIGHT = 1280 + +# ======================================================= +# Constants about models +# ======================================================= + +VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" +TOKENIZER = "ckpts/t2i/tokenizer" +TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' +T5_ENCODER = { + 'MT5': 'ckpts/t2i/mt5', + 'attention_mask': True, + 'layer_index': -1, + 'attention_pool': True, + 'torch_dtype': torch.float16, + 'learnable_replace': True +} + +SAMPLER_FACTORY = { + 'ddpm': { + 'scheduler': 'DDPMScheduler', + 'name': 'DDPM', + 'kwargs': { + 'steps_offset': 1, + 'clip_sample': False, + 'clip_sample_range': 1.0, + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + } + }, + 'ddim': { + 'scheduler': 'DDIMScheduler', + 'name': 'DDIM', + 'kwargs': { + 'steps_offset': 1, + 'clip_sample': False, + 'clip_sample_range': 1.0, + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + } + }, + 'dpmms': { + 'scheduler': 'DPMSolverMultistepScheduler', + 'name': 'DPMMS', + 'kwargs': { + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', + } + }, +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/arrow_load_stream.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/arrow_load_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2bd17ae04b0959ced080fae0c3345123bcc0e7 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/arrow_load_stream.py @@ -0,0 +1,255 @@ +import pickle +import random +from pathlib import Path +import ast +import numpy as np +import re +import json +import time +from functools import partial +from PIL import Image + +import torch +import torchvision.transforms as T +import torch.nn.functional as F +from torchvision.transforms import functional as TF +from torch.utils.data import Dataset + +from IndexKits.index_kits import ArrowIndexV2, MultiResolutionBucketIndexV2, MultiIndexV2 + + +class TextImageArrowStream(Dataset): + def __init__(self, + args, + resolution=512, + random_flip=None, + enable_CN=True, + log_fn=print, + index_file=None, + multireso=False, + batch_size=-1, + world_size=1, + random_shrink_size_cond=False, + merge_src_cond=False, + uncond_p=0.0, + text_ctx_len=77, + tokenizer=None, + uncond_p_t5=0.0, + text_ctx_len_t5=256, + tokenizer_t5=None, + ): + self.args = args + self.resolution = resolution + self.log_fn = lambda x: log_fn(f" {Path(__file__).stem} | " + x) + + self.random_flip = random_flip + # If true, the Chinese prompt from the `text_zh` column will be taken from the arrow file; + # otherwise, the English prompt from the `text_en` column will be taken, + # provided that `text_zh` or `text_en` exists in the arrow file. + self.enable_CN = enable_CN + self.index_file = index_file + self.multireso = multireso + self.batch_size = batch_size + self.world_size = world_size + self.index_manager = self.load_index() + + # clip params + self.uncond_p = uncond_p + self.text_ctx_len = text_ctx_len + self.tokenizer = tokenizer + + # t5 params + self.uncond_p_t5 = uncond_p_t5 + self.text_ctx_len_t5 = text_ctx_len_t5 + self.tokenizer_t5 = tokenizer_t5 + + # size condition + self.random_shrink_size_cond = random_shrink_size_cond + self.merge_src_cond = merge_src_cond + + assert isinstance(resolution, int), f"resolution must be an integer, got {resolution}" + self.flip_norm = T.Compose( + [ + T.RandomHorizontalFlip() if self.random_flip else T.Lambda(lambda x: x), + T.ToTensor(), + T.Normalize([0.5], [0.5]), + ] + ) + + # show info + if self.merge_src_cond: + self.log_fn("Enable merging src condition: (oriW, oriH) --> ((WH)**0.5, (WH)**0.5)") + + self.log_fn("Enable image_meta_size condition (original_size, target_size, crop_coords)") + self.log_fn(f"Image_transforms: {self.flip_norm}") + + def load_index(self): + multireso = self.multireso + index_file = self.index_file + batch_size = self.batch_size + world_size = self.world_size + + if multireso: + if isinstance(index_file, (list, tuple)): + if len(index_file) > 1: + raise ValueError(f"When enabling multireso, index_file should be a single file, but got {index_file}") + index_file = index_file[0] + index_manager = MultiResolutionBucketIndexV2(index_file, batch_size, world_size) + self.log_fn(f"Using MultiResolutionBucketIndexV2: {len(index_manager):,}") + else: + if isinstance(index_file, str): + index_file = [index_file] + if len(index_file) == 1: + index_manager = ArrowIndexV2(index_file[0]) + self.log_fn(f"Using ArrowIndexV2: {len(index_manager):,}") + else: + index_manager = MultiIndexV2(index_file) + self.log_fn(f"Using MultiIndexV2: {len(index_manager):,}") + + return index_manager + + def shuffle(self, seed, fast=False): + self.index_manager.shuffle(seed, fast=fast) + + def get_raw_image(self, index, image_key="image"): + try: + ret = self.index_manager.get_image(index, image_key) + except Exception as e: + self.log_fn(f'get_raw_image | Error: {e}') + ret = Image.new("RGB", (256, 256), (255, 255, 255)) + return ret + + @staticmethod + def random_crop_image(image, origin_size, target_size): + aspect_ratio = float(origin_size[0]) / float(origin_size[1]) + if origin_size[0] < origin_size[1]: + new_width = target_size[0] + new_height = int(new_width / aspect_ratio) + else: + new_height = target_size[1] + new_width = int(new_height * aspect_ratio) + + image = image.resize((new_width, new_height), Image.LANCZOS) + + if new_width > target_size[0]: + x_start = random.randint(0, new_width - target_size[0]) + y_start = 0 + else: + x_start = 0 + y_start = random.randint(0, new_height - target_size[1]) + image_crop = image.crop((x_start, y_start, x_start + target_size[0], y_start + target_size[1])) + crops_coords_top_left = (x_start, y_start) + return image_crop, crops_coords_top_left + + def get_style(self, index): + "Here we use a default learned embedder layer for future extension." + style = 0 + return style + + def get_image_with_hwxy(self, index, image_key="image"): + + image = self.get_raw_image(index, image_key=image_key) + origin_size = image.size + + if self.multireso: + target_size = self.index_manager.get_target_size(index) + image, crops_coords_top_left = self.index_manager.resize_and_crop( + image, target_size, resample=Image.LANCZOS, crop_type='random') + image_tensor = self.flip_norm(image) + else: + target_size = (self.resolution, self.resolution) + image_crop, crops_coords_top_left = self.random_crop_image(image, origin_size, target_size) + image_tensor = self.flip_norm(image_crop) + + if self.random_shrink_size_cond: + origin_size = (1024 if origin_size[0] < 1024 else origin_size[0], + 1024 if origin_size[1] < 1024 else origin_size[1]) + if self.merge_src_cond: + val = (origin_size[0] * origin_size[1]) ** 0.5 + origin_size = (val, val) + + image_meta_size = tuple(origin_size) + tuple(target_size) + tuple(crops_coords_top_left) + kwargs = { + 'image_meta_size': image_meta_size, + } + + style = self.get_style(index) + kwargs['style'] = style + + return image_tensor, kwargs + + def get_text_info_with_encoder(self, description): + pad_num = 0 + text_inputs = self.tokenizer( + description, + padding="max_length", + max_length=self.text_ctx_len, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids[0] + attention_mask = text_inputs.attention_mask[0].bool() + if pad_num > 0: + attention_mask[1:pad_num + 1] = False + return description, text_input_ids, attention_mask + + def fill_t5_token_mask(self, fill_tensor, fill_number, setting_length): + fill_length = setting_length - fill_tensor.shape[1] + if fill_length > 0: + fill_tensor = torch.cat((fill_tensor, fill_number * torch.ones(1, fill_length)), dim=1) + return fill_tensor + + def get_text_info_with_encoder_t5(self, description_t5): + text_tokens_and_mask = self.tokenizer_t5( + description_t5, + max_length=self.text_ctx_len_t5, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors='pt' + ) + text_input_ids_t5=self.fill_t5_token_mask(text_tokens_and_mask["input_ids"], fill_number=1, setting_length=self.text_ctx_len_t5).long() + attention_mask_t5=self.fill_t5_token_mask(text_tokens_and_mask["attention_mask"], fill_number=0, setting_length=self.text_ctx_len_t5).bool() + return description_t5, text_input_ids_t5, attention_mask_t5 + + def get_original_text(self, ind): + text = self.index_manager.get_attribute(ind, 'text_zh' if self.enable_CN else 'text_en') + text = str(text).strip() + return text + + def get_text(self, ind): + text = self.get_original_text(ind) + if text == '': + text = '随机生成一张图片' + return text + + def __getitem__(self, ind): + # Get text + if random.random() < self.uncond_p: + description = "" + else: + description = self.get_text(ind) + + # Get text for t5 + if random.random() < self.uncond_p_t5: + description_t5 = "" + else: + description_t5 = self.get_text(ind) + + original_pil_image, kwargs = self.get_image_with_hwxy(ind) + + # Use encoder to embed tokens online + text, text_embedding, text_embedding_mask = self.get_text_info_with_encoder(description) + + text_t5, text_embedding_t5, text_embedding_mask_t5 = self.get_text_info_with_encoder_t5(description_t5) + return ( + original_pil_image, + text_embedding.clone().detach(), + text_embedding_mask.clone().detach(), + text_embedding_t5.clone().detach(), + text_embedding_mask_t5.clone().detach(), + {k: torch.tensor(np.array(v)).clone().detach() for k, v in kwargs.items()}, + ) + + def __len__(self): + return len(self.index_manager) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/csv2arrow.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/csv2arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..b2e364c4af8d8a42f2b96ae957a78e33acd60bae --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/data_loader/csv2arrow.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +import datetime +import gc +import os +import time +from multiprocessing import Pool +import subprocess +import pandas as pd +import pyarrow as pa +from tqdm import tqdm +import hashlib +from PIL import Image +import sys + + +def parse_data(data): + try: + img_path = data[0] + + with open(img_path, "rb") as fp: + image = fp.read() + md5 = hashlib.md5(image).hexdigest() + + with Image.open(img_path) as f: + width, height = f.size + + return [data[1], md5, width, height, image] + + except Exception as e: + print(f'error: {e}') + return + +def make_arrow(csv_root, dataset_root, start_id=0, end_id=-1): + print(csv_root) + arrow_dir = dataset_root + print(arrow_dir) + + if not os.path.exists(arrow_dir): + os.makedirs(arrow_dir) + + data = pd.read_csv(csv_root) + data = data[["image_path", "text_zh"]] + columns_list = data.columns.tolist() + columns_list.append("image") + + if end_id < 0: + end_id = len(data) + print(f'start_id:{start_id} end_id:{end_id}') + data = data[start_id:end_id] + num_slice = 5000 + start_sub = int(start_id / num_slice) + sub_len = int(len(data) // num_slice) # if int(len(data) // num_slice) else 1 + subs = list(range(sub_len + 1)) + for sub in tqdm(subs): + arrow_path = os.path.join(arrow_dir, '{}.arrow'.format(str(sub + start_sub).zfill(5))) + if os.path.exists(arrow_path): + continue + print(f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} start {sub + start_sub}") + + sub_data = data[sub * num_slice: (sub + 1) * num_slice].values + + bs = pool.map(parse_data, sub_data) + bs = [b for b in bs if b] + print(f'length of this arrow:{len(bs)}') + + columns_list = ["text_zh", "md5", "width", "height", "image"] + dataframe = pd.DataFrame(bs, columns=columns_list) + table = pa.Table.from_pandas(dataframe) + + os.makedirs(dataset_root, exist_ok=True) + with pa.OSFile(arrow_path, "wb") as sink: + with pa.RecordBatchFileWriter(sink, table.schema) as writer: + writer.write_table(table) + del dataframe + del table + del bs + gc.collect() + + +if __name__ == '__main__': + + if len(sys.argv) != 3: + print("Usage: python hydit/data_loader/csv2arrow.py ${csv_root} ${output_arrow_data_path}") + sys.exit(1) + csv_root = sys.argv[1] + output_arrow_data_path = sys.argv[2] + pool = Pool(500) + make_arrow(csv_root, output_arrow_data_path) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a681113da5c6fc6a1cc216e2270171c4780898 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/__init__.py @@ -0,0 +1,49 @@ +from . import gaussian_diffusion as gd +from .respace import SpacedDiffusion, space_timesteps + + +def create_diffusion( + *, + steps=1000, + learn_sigma=True, + sigma_small=False, + noise_schedule="linear", + use_kl=False, + predict_type='epsilon', + rescale_timesteps=False, + rescale_learned_sigmas=False, + timestep_respacing="", + mse_loss_weight_type='constant', + beta_start=0.0001, + beta_end=0.02, + noise_offset=0.0, +): + betas = gd.get_named_beta_schedule(noise_schedule, steps, beta_start, beta_end) + if use_kl: + loss_type = gd.LossType.RESCALED_KL + elif rescale_learned_sigmas: + loss_type = gd.LossType.RESCALED_MSE + else: + loss_type = gd.LossType.MSE + if timestep_respacing is None or timestep_respacing == "": + timestep_respacing = [steps] + mean_type = gd.predict_type_dict[predict_type] + + return SpacedDiffusion( + use_timesteps=space_timesteps(steps, timestep_respacing), + betas=betas, + model_mean_type=mean_type, + model_var_type=( + ( + gd.ModelVarType.FIXED_LARGE + if not sigma_small + else gd.ModelVarType.FIXED_SMALL + ) + if not learn_sigma + else gd.ModelVarType.LEARNED_RANGE + ), + loss_type=loss_type, + rescale_timesteps=rescale_timesteps, + mse_loss_weight_type=mse_loss_weight_type, + noise_offset=noise_offset, + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/diffusion_utils.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/diffusion_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9c62d5b0f914eb8bef789d3612e3c48d68ec0cef --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/diffusion_utils.py @@ -0,0 +1,69 @@ +import torch as th +import numpy as np + + +def normal_kl(mean1, logvar1, mean2, logvar2): + """ + Compute the KL divergence between two gaussians. + Shapes are automatically broadcasted, so batches can be compared to + scalars, among other use cases. + """ + tensor = None + for obj in (mean1, logvar1, mean2, logvar2): + if isinstance(obj, th.Tensor): + tensor = obj + break + assert tensor is not None, "at least one argument must be a Tensor" + + # Force variances to be Tensors. Broadcasting helps convert scalars to + # Tensors, but it does not work for th.exp(). + logvar1, logvar2 = [ + x if isinstance(x, th.Tensor) else th.tensor(x).to(tensor) + for x in (logvar1, logvar2) + ] + + return 0.5 * ( + -1.0 + + logvar2 + - logvar1 + + th.exp(logvar1 - logvar2) + + ((mean1 - mean2) ** 2) * th.exp(-logvar2) + ) + + +def approx_standard_normal_cdf(x): + """ + A fast approximation of the cumulative distribution function of the + standard normal. + """ + return 0.5 * (1.0 + th.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * th.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x, *, means, log_scales): + """ + Compute the log-likelihood of a Gaussian distribution discretizing to a + given image. + :param x: the target images. It is assumed that this was uint8 values, + rescaled to the range [-1, 1]. + :param means: the Gaussian mean Tensor. + :param log_scales: the Gaussian log stddev Tensor. + :return: a tensor like x of log probabilities (in nats). + """ + assert x.shape == means.shape == log_scales.shape + centered_x = x - means + inv_stdv = th.exp(-log_scales) + plus_in = inv_stdv * (centered_x + 1.0 / 255.0) + cdf_plus = approx_standard_normal_cdf(plus_in) + min_in = inv_stdv * (centered_x - 1.0 / 255.0) + cdf_min = approx_standard_normal_cdf(min_in) + log_cdf_plus = th.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = th.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = th.where( + x < -0.999, + log_cdf_plus, + th.where(x > 0.999, log_one_minus_cdf_min, th.log(cdf_delta.clamp(min=1e-12))), + ) + assert log_probs.shape == x.shape + return log_probs + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/gaussian_diffusion.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/gaussian_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..23b0f44a3d4b9e565ca2ebdacf0000fe178d6a9f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/gaussian_diffusion.py @@ -0,0 +1,1369 @@ +import math + +import numpy as np +import torch as th +import enum + +from .diffusion_utils import discretized_gaussian_log_likelihood, normal_kl +from ..utils.tools import assert_shape + + +def mean_flat(tensor): + """ + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +class ModelMeanType(enum.Enum): + """ + Which type of output the model predicts. + """ + + PREVIOUS_X = enum.auto() # the model predicts x_{t-1} + START_X = enum.auto() # the model predicts x_0 + EPSILON = enum.auto() # the model predicts epsilon + VELOCITY = enum.auto() # the model predicts v + + +predict_type_dict = { + 'epsilon': ModelMeanType.EPSILON, + 'sample': ModelMeanType.START_X, + 'v_prediction': ModelMeanType.VELOCITY, +} + + +class ModelVarType(enum.Enum): + """ + What is used as the model's output variance. + The LEARNED_RANGE option has been added to allow the model to predict + values between FIXED_SMALL and FIXED_LARGE, making its job easier. + """ + + LEARNED = enum.auto() + FIXED_SMALL = enum.auto() + FIXED_LARGE = enum.auto() + LEARNED_RANGE = enum.auto() + + +class LossType(enum.Enum): + MSE = enum.auto() # use raw MSE loss (and KL when learning variances) + RESCALED_MSE = ( + enum.auto() + ) # use raw MSE loss (with RESCALED_KL when learning variances) + KL = enum.auto() # use the variational lower-bound + RESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB + + def is_vb(self): + return self == LossType.KL or self == LossType.RESCALED_KL + + +def _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, warmup_frac): + betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64) + warmup_time = int(num_diffusion_timesteps * warmup_frac) + betas[:warmup_time] = np.linspace(beta_start, beta_end, warmup_time, dtype=np.float64) + return betas + + +def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_timesteps): + """ + This is the deprecated API for creating beta schedules. + See get_named_beta_schedule() for the new library of schedules. + """ + if beta_schedule == "quad": + betas = ( + np.linspace( + beta_start ** 0.5, + beta_end ** 0.5, + num_diffusion_timesteps, + dtype=np.float64, + ) + ** 2 + ) + elif beta_schedule == "linear": + betas = np.linspace(beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64) + elif beta_schedule == "warmup10": + betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.1) + elif beta_schedule == "warmup50": + betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.5) + elif beta_schedule == "const": + betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64) + elif beta_schedule == "jsd": # 1/T, 1/(T-1), 1/(T-2), ..., 1 + betas = 1.0 / np.linspace( + num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64 + ) + else: + raise NotImplementedError(beta_schedule) + assert_shape(betas, (num_diffusion_timesteps,)) + return betas + + +def get_named_beta_schedule(schedule_name, num_diffusion_timesteps, beta_start=0.0001, beta_end=0.02): + """ + Get a pre-defined beta schedule for the given name. + The beta schedule library consists of beta schedules which remain similar + in the limit of num_diffusion_timesteps. + Beta schedules may be added, but should not be removed or changed once + they are committed to maintain backwards compatibility. + """ + if schedule_name == "linear": + # Linear schedule from Ho et al, extended to work for any number of + # diffusion steps. + scale = 1000 / num_diffusion_timesteps + return get_beta_schedule( + "linear", + beta_start=scale * beta_start, # DDPM + beta_end=scale * beta_end, # DDPM + num_diffusion_timesteps=num_diffusion_timesteps, # DDPM + ) + elif schedule_name == "scaled_linear": + return get_beta_schedule( + "quad", + beta_start=beta_start, # StableDiffusion, should be 0.00085 + beta_end=beta_end, # StableDiffusion, should be 0.012 + num_diffusion_timesteps=num_diffusion_timesteps, # StableDiffusion + ) + elif schedule_name == "squaredcos_cap_v2": + return betas_for_alpha_bar( + num_diffusion_timesteps, + lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2, + ) + else: + raise NotImplementedError(f"unknown beta schedule: {schedule_name}") + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + +class GaussianDiffusion: + """ + Utilities for training and sampling diffusion models. + + Ported directly from here, and then adapted over time to further experimentation. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py#L42 + + :param betas: a 1-D numpy array of betas for each diffusion timestep, + starting at T and going to 1. + :param model_mean_type: a ModelMeanType determining what the model outputs. + :param model_var_type: a ModelVarType determining how variance is output. + :param loss_type: a LossType determining the loss function to use. + :param rescale_timesteps: if True, pass floating point timesteps into the + model so that they are always scaled like in the + original paper (0 to 1000). + """ + + def __init__( + self, + *, + betas, + model_mean_type, + model_var_type, + loss_type, + rescale_timesteps=False, + mse_loss_weight_type='constant', + noise_offset=0.0, + ): + self.model_mean_type = model_mean_type + self.model_var_type = model_var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + self.mse_loss_weight_type = mse_loss_weight_type + self.noise_offset = noise_offset + + # Use float64 for accuracy. + betas = np.array(betas, dtype=np.float64) + alphas = 1.0 - betas + alphas_cumprod = np.cumprod(alphas, axis=0) + + self.betas = betas + assert len(betas.shape) == 1, "betas must be 1-D" + assert (betas > 0).all() and (betas <= 1).all() + + self.num_timesteps = int(betas.shape[0]) + + self.alphas_cumprod = alphas_cumprod + self.alphas_cumprod_prev = np.append(1.0, self.alphas_cumprod[:-1]) + self.alphas_cumprod_next = np.append(self.alphas_cumprod[1:], 0.0) + assert_shape(self.alphas_cumprod_prev, (self.num_timesteps,)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.sqrt_alphas_cumprod = np.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = np.sqrt(1.0 - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = np.log(1.0 - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod - 1) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + self.posterior_variance = ( + betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + ) + # log calculation clipped because the posterior variance is 0 at the + # beginning of the diffusion chain. + self.posterior_log_variance_clipped = np.log( + np.append(self.posterior_variance[1], self.posterior_variance[1:]) + ) if len(self.posterior_variance) > 1 else np.array([]) + + self.posterior_mean_coef1 = ( + betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + ) + self.posterior_mean_coef2 = ( + (1.0 - self.alphas_cumprod_prev) + * np.sqrt(alphas) + / (1.0 - self.alphas_cumprod) + ) + + self.sampler = { + "ddpm": self.p_sample_loop, + "ddim": self.ddim_sample_loop, + "plms": self.plms_sample_loop, + } + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + ) + variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape) + log_variance = _extract_into_tensor( + self.log_one_minus_alphas_cumprod, t, x_start.shape + ) + return mean, variance, log_variance + + def q_sample(self, x_start, t, noise=None): + """ + Diffuse the data for a given number of diffusion steps. + + In other words, sample from q(x_t | x_0). + + :param x_start: the initial data batch. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :param noise: if specified, the split-out normal noise. + :return: A noisy version of x_start. + """ + if noise is None: + noise = th.randn_like(x_start) + assert_shape(noise, x_start) + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) + * noise + ) + + def q_posterior_mean_variance(self, x_start, x_t, t): + """ + Compute the mean and variance of the diffusion posterior: + + q(x_{t-1} | x_t, x_0) + + """ + assert_shape(x_start, x_t) + posterior_mean = ( + _extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) * x_start + + _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t + ) + posterior_variance = _extract_into_tensor(self.posterior_variance, t, x_t.shape) + posterior_log_variance_clipped = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape + ) + assert_shape( + posterior_mean.shape[:1], + posterior_variance.shape[:1], + posterior_log_variance_clipped.shape[:1], + x_start.shape[:1], + ) + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def p_mean_variance( + self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None, + model_var_type=None, + ): + """ + Apply the model to get p(x_{t-1} | x_t), as well as a prediction of + the initial x, x_0. + + :param model: the model, which takes a signal and a batch of timesteps + as input. + :param x: the [N x C x ...] tensor at time t. + :param t: a 1-D Tensor of timesteps. + :param clip_denoised: if True, clip the denoised signal into [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. Applies before + clip_denoised. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :return: a dict with the following keys: + - 'mean': the model mean output. + - 'variance': the model variance output. + - 'log_variance': the log of 'variance'. + - 'pred_xstart': the prediction for x_0. + """ + if model_kwargs is None: + model_kwargs = {} + + if model_var_type is None: + model_var_type = self.model_var_type + + B, C = x.shape[:2] + assert_shape(t, (B,)) + out_dict = model(x, t, **model_kwargs) + model_output = out_dict['x'] + + if len(out_dict) > 1: + extra = {k: v for k, v in out_dict.items() if k != 'x'} + else: + extra = None + + # self.model_var_type corresponds to model output + if self.model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]: + assert_shape(model_output, (B, C * 2, *x.shape[2:])) + model_output, model_var_values = th.split(model_output, C, dim=1) + + # model_var_type corresponds to reverse diffusion process + if model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]: + if model_var_type == ModelVarType.LEARNED: + model_log_variance = model_var_values + model_variance = th.exp(model_log_variance) + else: + min_log = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x.shape + ) + max_log = _extract_into_tensor(np.log(self.betas), t, x.shape) + # The model_var_values is [-1, 1] for [min_var, max_var]. + frac = (model_var_values + 1) / 2 + model_log_variance = frac * max_log + (1 - frac) * min_log + model_variance = th.exp(model_log_variance) + else: + model_variance, model_log_variance = { + # for fixedlarge, we set the initial (log-)variance like so + # to get a better decoder log likelihood. + ModelVarType.FIXED_LARGE: ( + np.append(self.posterior_variance[1], self.betas[1:]), + np.log(np.append(self.posterior_variance[1], self.betas[1:])), + ), + ModelVarType.FIXED_SMALL: ( + self.posterior_variance, + self.posterior_log_variance_clipped, + ), + }[model_var_type] + model_variance = _extract_into_tensor(model_variance, t, x.shape) + model_log_variance = _extract_into_tensor(model_log_variance, t, x.shape) + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + if self.model_mean_type == ModelMeanType.PREVIOUS_X: + pred_xstart = process_xstart( + self._predict_xstart_from_xprev(x_t=x, t=t, xprev=model_output) + ) + model_mean = model_output + elif self.model_mean_type in [ + ModelMeanType.START_X, + ModelMeanType.EPSILON, + ModelMeanType.VELOCITY + ]: + if self.model_mean_type == ModelMeanType.START_X: + pred_xstart = process_xstart(model_output) + elif self.model_mean_type == ModelMeanType.EPSILON: + pred_xstart = process_xstart( + self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output) + ) + else: + pred_xstart = process_xstart( + self._predict_xstart_from_v(x_t=x, t=t, v=model_output) + ) + model_mean, _, _ = self.q_posterior_mean_variance( + x_start=pred_xstart, x_t=x, t=t + ) + else: + raise NotImplementedError(self.model_mean_type) + + assert_shape(model_mean, model_log_variance, pred_xstart, x) + return { + "mean": model_mean, + "variance": model_variance, + "log_variance": model_log_variance, + "pred_xstart": pred_xstart, + "extra": extra, + } + + def _predict_xstart_from_eps(self, x_t, t, eps): + assert_shape(x_t, eps) + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t + - _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * eps + ) + + def _predict_xstart_from_v(self, x_t, t, v): + assert_shape(x_t, v) + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_t.shape) * x_t + - _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_t.shape) * v + ) + + def _predict_xstart_from_xprev(self, x_t, t, xprev): + assert_shape(x_t, xprev) + return ( # (xprev - coef2*x_t) / coef1 + _extract_into_tensor(1.0 / self.posterior_mean_coef1, t, x_t.shape) * xprev + - _extract_into_tensor( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, x_t.shape + ) + * x_t + ) + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t + - pred_xstart + ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) + + def _velocity_from_xstart_and_noise(self, x_start, t, noise): + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * noise + - _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) + * x_start + ) + + def _vb_terms_bpd( + self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None + ): + """ + Get a term for the variational lower-bound. + + The resulting units are bits (rather than nats, as one might expect). + This allows for comparison to other papers. + + :return: a dict with the following keys: + - 'output': a shape [N] tensor of NLLs or KLs. + - 'pred_xstart': the x_0 predictions. + """ + true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance( + x_start=x_start, x_t=x_t, t=t + ) + out = self.p_mean_variance( + model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs + ) + kl = normal_kl( + true_mean, true_log_variance_clipped, out["mean"], out["log_variance"] + ) + kl = mean_flat(kl) / np.log(2.0) + + decoder_nll = -discretized_gaussian_log_likelihood( + x_start, means=out["mean"], log_scales=0.5 * out["log_variance"] + ) + assert_shape(decoder_nll, x_start) + decoder_nll = mean_flat(decoder_nll) / np.log(2.0) + + # At the first timestep return the decoder NLL, + # otherwise return KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t)) + output = th.where((t == 0), decoder_nll, kl) + return {"output": output, "pred_xstart": out["pred_xstart"], "extra": out["extra"]} + + def training_losses(self, model, x_start, model_kwargs=None, controlnet=None, noise=None): + """ + Compute training losses for a single timestep. + + :param model: the model to evaluate loss on. + :param x_start: the [N x C x ...] tensor of inputs. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param noise: if specified, the specific Gaussian noise to try to remove. + :return: a dict with the key "loss" containing a tensor of shape [N]. + Some mean or variance settings may also have other keys. + """ + if model_kwargs is None: + model_kwargs = {} + # Time steps + t = th.randint(0, self.num_timesteps, (x_start.shape[0],), device=x_start.device) + # Noise + if noise is None: + noise = th.randn_like(x_start) + if self.noise_offset > 0: + # Add channel wise noise offset + # https://www.crosslabs.org/blog/diffusion-with-offset-noise + noise = noise + self.noise_offset * th.randn(*x_start.shape[:2], 1, 1, device=x_start.device) + x_t = self.q_sample(x_start, t, noise=noise) + + terms = {} + + if self.mse_loss_weight_type == 'constant': + mse_loss_weight = th.ones_like(t) + elif self.mse_loss_weight_type.startswith("min_snr_"): + alpha = _extract_into_tensor(self.sqrt_alphas_cumprod, t, t.shape) + sigma = _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, t.shape) + snr = (alpha / sigma) ** 2 + + k = float(self.mse_loss_weight_type.split('min_snr_')[-1]) + # min{snr, k} + mse_loss_weight = th.stack([snr, k * th.ones_like(t)], dim=1).min(dim=1)[0] / snr + else: + raise ValueError(self.mse_loss_weight_type) + + if self.loss_type == LossType.KL or self.loss_type == LossType.RESCALED_KL: + out_dict = self._vb_terms_bpd( + model=model, + x_start=x_start, + x_t=x_t, + t=t, + clip_denoised=False, + model_kwargs=model_kwargs, + ) + terms["loss"] = out_dict["output"] + if self.loss_type == LossType.RESCALED_KL: + terms["loss"] *= self.num_timesteps + extra = out_dict["extra"] + elif self.loss_type == LossType.MSE or self.loss_type == LossType.RESCALED_MSE: + if controlnet != None: + controls = controlnet(x_t, t, **model_kwargs) + model_kwargs.pop('condition') + model_kwargs.update(controls) + out_dict = model(x_t, t, **model_kwargs) + model_output = out_dict['x'] + extra = {k: v for k, v in out_dict.items() if k != 'x'} + + if self.model_var_type in [ + ModelVarType.LEARNED, + ModelVarType.LEARNED_RANGE, + ]: + B, C = x_t.shape[:2] + assert_shape(model_output, (B, C * 2, *x_t.shape[2:])) + model_output, model_var_values = th.split(model_output, C, dim=1) + # Learn the variance using the variational bound, but don't let + # it affect our mean prediction. + frozen_out = th.cat([model_output.detach(), model_var_values], dim=1) + terms["vb"] = self._vb_terms_bpd( + model=lambda *args, r=frozen_out: dict(x=r), + x_start=x_start, + x_t=x_t, + t=t, + clip_denoised=False, + )["output"] + if self.loss_type == LossType.RESCALED_MSE: + # Divide by 1000 for equivalence with initial implementation. + # Without a factor of 1/1000, the VB term hurts the MSE term. + terms["vb"] *= self.num_timesteps / 1000.0 + + if self.model_mean_type == ModelMeanType.VELOCITY: + target = self._velocity_from_xstart_and_noise(x_start, t, noise) + else: + target = { + # ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance( + # x_start=x_start, x_t=x_t, t=t + # )[0], + ModelMeanType.START_X: x_start, + ModelMeanType.EPSILON: noise, + }[self.model_mean_type] + assert_shape(model_output, target, x_start) + raw_mse = mean_flat((target - model_output) ** 2).detach() + terms["mse"] = mse_loss_weight * mean_flat((target - model_output) ** 2) + if "vb" in terms: + terms["loss"] = terms["mse"] + terms["vb"] + terms["raw_loss"] = raw_mse + terms["vb"].detach() + else: + terms["loss"] = terms["mse"] + terms["raw_loss"] = raw_mse + else: + raise NotImplementedError(self.loss_type) + + terms.update(extra) + return terms + + def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, t, **model_kwargs) + new_mean = ( + p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float() + ) + return new_mean + + def condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute what the p_mean_variance output would have been, should the + model's score function be conditioned by cond_fn. + See condition_mean() for details on cond_fn. + Unlike condition_mean(), this instead uses the conditioning strategy + from Song et al (2020). + """ + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + + eps = self._predict_eps_from_xstart(x, t, p_mean_var["pred_xstart"]) + eps = eps - (1 - alpha_bar).sqrt() * cond_fn( + x, t, **model_kwargs + ) + + out = p_mean_var.copy() + out["pred_xstart"] = self._predict_xstart_from_eps(x, t, eps) + out["mean"], _, _ = self.q_posterior_mean_variance( + x_start=out["pred_xstart"], x_t=x, t=t + ) + return out + + def p_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + **kwargs, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + ) + noise = th.randn_like(x) + nonzero_mask = ( + (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out["mean"] = self.condition_mean( + cond_fn, out, x, t, model_kwargs=model_kwargs + ) + + sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noise + + return {"sample": sample, "pred_xstart": out["pred_xstart"]} + + def p_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + device=None, + progress=False, + progress_leave=True, + **kwargs, + ): + """ + Generate samples from the model. + + :param model: the model module. + :param shape: the shape of the samples, (N, C, H, W). + :param noise: if specified, the noise from the encoder to sample. + Should be of the same shape as `shape`. + :param clip_denoised: if True, clip x_start predictions to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :param device: if specified, the device to create the samples on. + If not specified, use a model parameter's device. + :param progress: if True, show a tqdm progress bar. + :return: a non-differentiable batch of samples. + """ + final = None + for sample in self.p_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + device=device, + progress=progress, + progress_leave=progress_leave, + **kwargs, + ): + final = sample + return final["sample"] + + def p_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + device=None, + progress=False, + progress_leave=True, + **kwargs, + ): + """ + Generate samples from the model and yield intermediate samples from + each timestep of diffusion. + + Arguments are the same as p_sample_loop(). + Returns a generator over dicts, where each dict is the return value of + p_sample(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.p_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + **kwargs, + ) + yield out + img = out["sample"] + + def ddim_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + ): + """ + Sample x_{t-1} from the model using DDIM. + + Same usage as p_sample(). + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score(cond_fn, out, x, t, model_kwargs=model_kwargs) + + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = self._predict_eps_from_xstart(x, t, out["pred_xstart"]) + + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, x.shape) + sigma = ( + eta + * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) + * th.sqrt(1 - alpha_bar / alpha_bar_prev) + ) + # Equation 12. + noise = th.randn_like(x) + mean_pred = ( + out["pred_xstart"] * th.sqrt(alpha_bar_prev) + + th.sqrt(1 - alpha_bar_prev - sigma ** 2) * eps + ) + nonzero_mask = ( + (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + sample = mean_pred + nonzero_mask * sigma * noise + return {"sample": sample, "pred_xstart": out["pred_xstart"]} + + def ddim_reverse_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + ): + """ + Sample x_{t+1} from the model using DDIM reverse ODE. + """ + assert eta == 0.0, "Reverse ODE only for deterministic path" + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score(cond_fn, out, x, t, model_kwargs=model_kwargs) + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x + - out["pred_xstart"] + ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x.shape) + alpha_bar_next = _extract_into_tensor(self.alphas_cumprod_next, t, x.shape) + + # Equation 12. reversed + mean_pred = ( + out["pred_xstart"] * th.sqrt(alpha_bar_next) + + th.sqrt(1 - alpha_bar_next) * eps + ) + + return {"sample": mean_pred, "pred_xstart": out["pred_xstart"]} + + def ddim_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + eta=0.0, + ): + """ + Generate samples from the model using DDIM. + + Same usage as p_sample_loop(). + """ + final = None + for sample in self.ddim_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + progress_leave=progress_leave, + eta=eta, + ): + final = sample + return final["sample"] + + def ddim_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + eta=0.0, + ): + """ + Use DDIM to sample from the model and yield intermediate samples from + each timestep of DDIM. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.ddim_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + eta=eta, + ) + yield out + img = out["sample"] + + def _prior_bpd(self, x_start): + """ + Get the prior KL term for the variational lower-bound, measured in + bits-per-dim. + + This term can't be optimized, as it only depends on the encoder. + + :param x_start: the [N x C x ...] tensor of inputs. + :return: a batch of [N] KL values (in bits), one per batch element. + """ + batch_size = x_start.shape[0] + t = th.tensor([self.num_timesteps - 1] * batch_size, device=x_start.device) + qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t) + kl_prior = normal_kl( + mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0 + ) + return mean_flat(kl_prior) / np.log(2.0) + + def calc_bpd_loop(self, model, x_start, clip_denoised=True, model_kwargs=None): + """ + Compute the entire variational lower-bound, measured in bits-per-dim, + as well as other related quantities. + + :param model: the model to evaluate loss on. + :param x_start: the [N x C x ...] tensor of inputs. + :param clip_denoised: if True, clip denoised samples. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + + :return: a dict containing the following keys: + - total_bpd: the total variational lower-bound, per batch element. + - prior_bpd: the prior term in the lower-bound. + - vb: an [N x T] tensor of terms in the lower-bound. + - xstart_mse: an [N x T] tensor of x_0 MSEs for each timestep. + - mse: an [N x T] tensor of epsilon MSEs for each timestep. + """ + device = x_start.device + batch_size = x_start.shape[0] + + vb = [] + xstart_mse = [] + mse = [] + for t in list(range(self.num_timesteps))[::-1]: + t_batch = th.tensor([t] * batch_size, device=device) + noise = th.randn_like(x_start) + x_t = self.q_sample(x_start=x_start, t=t_batch, noise=noise) + # Calculate VLB term at the current timestep + with th.no_grad(): + out = self._vb_terms_bpd( + model, + x_start=x_start, + x_t=x_t, + t=t_batch, + clip_denoised=clip_denoised, + model_kwargs=model_kwargs, + ) + vb.append(out["output"]) + xstart_mse.append(mean_flat((out["pred_xstart"] - x_start) ** 2)) + eps = self._predict_eps_from_xstart(x_t, t_batch, out["pred_xstart"]) + mse.append(mean_flat((eps - noise) ** 2)) + + vb = th.stack(vb, dim=1) + xstart_mse = th.stack(xstart_mse, dim=1) + mse = th.stack(mse, dim=1) + + prior_bpd = self._prior_bpd(x_start) + total_bpd = vb.sum(dim=1) + prior_bpd + return { + "total_bpd": total_bpd, + "prior_bpd": prior_bpd, + "vb": vb, + "xstart_mse": xstart_mse, + "mse": mse, + } + + def get_eps( + self, + model, + x, + t, + model_kwargs, + cond_fn=None, + ): + model_output = model(x, t, **model_kwargs)['x'] + if isinstance(model_output, tuple): + model_output, _ = model_output + eps = model_output[:, :4] + if cond_fn is not None: + alpha_bar = _extract_into_tensor_lerp(self.alphas_cumprod, t, x.shape) + eps = eps - th.sqrt(1 - alpha_bar) * cond_fn(x, t, **model_kwargs) + return eps + + def eps_to_pred_xstart( + self, + x, + eps, + t, + ): + alpha_bar = _extract_into_tensor_lerp(self.alphas_cumprod, t, x.shape) + return (x - eps * th.sqrt(1 - alpha_bar)) / th.sqrt(alpha_bar) + + def pndm_transfer( + self, + x, + eps, + t_1, + t_2, + ): + pred_xstart = self.eps_to_pred_xstart(x, eps, t_1) + alpha_bar_prev = _extract_into_tensor_lerp(self.alphas_cumprod, t_2, x.shape) + return pred_xstart * th.sqrt(alpha_bar_prev) + th.sqrt(1 - alpha_bar_prev) * eps + + def prk_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + ): + """ + Generate samples from the model using PRK. + + Same usage as p_sample_loop(). + """ + final = None + for sample in self.prk_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + ): + final = sample + return final["sample"] + + def prk_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + ): + """ + Use PRK to sample from the model and yield intermediate samples from + each timestep of PRK. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1][1:-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=False) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.prk_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + yield out + img = out["sample"] + + def prk_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model using fourth-order Pseudo Runge-Kutta + (https://openreview.net/forum?id=PlKWVd2yBkY). + + Same usage as p_sample(). + """ + if model_kwargs is None: + model_kwargs = {} + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + eps_1 = self.get_eps(model, x, t, model_kwargs, cond_fn) + x_1 = self.pndm_transfer(x, eps_1, t, t - 0.5) + eps_2 = self.get_eps(model, x_1, t - 0.5, model_kwargs, cond_fn) + x_2 = self.pndm_transfer(x, eps_2, t, t - 0.5) + eps_3 = self.get_eps(model, x_2, t - 0.5, model_kwargs, cond_fn) + x_3 = self.pndm_transfer(x, eps_3, t, t - 1) + eps_4 = self.get_eps(model, x_3, t - 1, model_kwargs, cond_fn) + eps_prime = (eps_1 + 2 * eps_2 + 2 * eps_3 + eps_4) / 6 + + sample = self.pndm_transfer(x, eps_prime, t, t - 1) + pred_xstart = self.eps_to_pred_xstart(x, eps_prime, t) + pred_xstart = process_xstart(pred_xstart) + return {"sample": sample, "pred_xstart": pred_xstart, "eps": eps_prime} + + def plms_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + ): + """ + Generate samples from the model using PLMS. + + Same usage as p_sample_loop(). + """ + assert self.model_mean_type == ModelMeanType.EPSILON, \ + 'plms_sample only support model_mean_type == ModelMeanType.EPSILON' + final = None + for sample in self.plms_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + progress_leave=progress_leave, + ): + final = sample + return final["sample"] + + def plms_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + ): + """ + Use PLMS to sample from the model and yield intermediate samples from + each timestep of PLMS. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1][1:-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + old_eps = [] + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + if len(old_eps) < 3: + out = self.prk_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + else: + out = self.plms_sample( + model, + img, + old_eps, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + old_eps.pop(0) + old_eps.append(out["eps"]) + yield out + img = out["sample"] + + def plms_sample( + self, + model, + x, + old_eps, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model using fourth-order Pseudo Linear Multistep + (https://openreview.net/forum?id=PlKWVd2yBkY). + """ + if model_kwargs is None: + model_kwargs = {} + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + eps = self.get_eps(model, x, t, model_kwargs, cond_fn) + eps_prime = (55 * eps - 59 * old_eps[-1] + 37 * old_eps[-2] - 9 * old_eps[-3]) / 24 + + sample = self.pndm_transfer(x, eps_prime, t, t - 1) + pred_xstart = self.eps_to_pred_xstart(x, eps, t) + pred_xstart = process_xstart(pred_xstart) + return {"sample": sample, "pred_xstart": pred_xstart, "eps": eps} + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + """ + Extract values from a 1-D numpy array for a batch of indices. + + :param arr: the 1-D numpy array. + :param timesteps: a tensor of indices into the array to extract. + :param broadcast_shape: a larger shape of K dimensions with the batch + dimension equal to the length of timesteps. + :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims. + """ + res = th.from_numpy(arr).to(device=timesteps.device)[timesteps].float() + while len(res.shape) < len(broadcast_shape): + res = res[..., None] + return res.expand(broadcast_shape) + + +def _extract_into_tensor_lerp(arr, timesteps, broadcast_shape): + """ + Extract values from arr with fractional time steps + """ + timesteps = timesteps.float() + frac = timesteps.frac() + while len(frac.shape) < len(broadcast_shape): + frac = frac[..., None] + res_1 = _extract_into_tensor(arr, timesteps.floor().long(), broadcast_shape) + res_2 = _extract_into_tensor(arr, timesteps.ceil().long(), broadcast_shape) + return th.lerp(res_1, res_2, frac) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..e765b7052328628b541ef65a0183f797d1233a57 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline.py @@ -0,0 +1,794 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import ( + deprecate, + logging, + replace_example_docstring, +) +from diffusers.utils.torch_utils import randn_tensor +from transformers import BertModel, BertTokenizer +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ..modules.models import HunYuanDiT + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]): + A `BertTokenizer` or `CLIPTokenizer` to tokenize text. + unet (Optional[`HunYuanDiT`, `UNet2DConditionModel`]): + A `HunYuanDiT` or `UNet2DConditionModel` to denoise the encoded image latents. + Notice: Here we still keep the word `unet` for compatibility with the previous version of the pipeline. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: Union[BertModel, CLIPTextModel], + tokenizer: Union[BertTokenizer, CLIPTokenizer], + unet: Union[HunYuanDiT, UNet2DConditionModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + progress_bar_config: Dict[str, Any] = None, + embedder_t5=None, + infer_mode='torch', + ): + super().__init__() + + # ======================================================== + self.embedder_t5 = embedder_t5 + self.infer_mode = infer_mode + + # ======================================================== + if progress_bar_config is None: + progress_bar_config = {} + if not hasattr(self, '_progress_bar_config'): + self._progress_bar_config = {} + self._progress_bar_config.update(progress_bar_config) + # ======================================================== + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + embedder=None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + embedder: + T5 embedder (including text encoder and tokenizer) + """ + if embedder is None: + text_encoder = self.text_encoder + tokenizer = self.tokenizer + max_length = self.tokenizer.model_max_length + else: + text_encoder = embedder.model + tokenizer = embedder.tokenizer + max_length = embedder.max_length + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode( + untruncated_ids[:, tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + else: + attention_mask = None + + if text_encoder is not None: + prompt_embeds_dtype = text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_attention_mask = uncond_input.attention_mask.to(device) + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + uncond_attention_mask = uncond_attention_mask.repeat(num_images_per_prompt, 1) + else: + uncond_attention_mask = None + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + height: int, + width: int, + prompt: Union[str, List[str]] = None, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_t5: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_t5: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + image_meta_size: Optional[torch.LongTensor] = None, + style: Optional[torch.LongTensor] = None, + progress: bool = True, + use_fp16: bool = False, + freqs_cis_img: Optional[tuple] = None, + learn_sigma: bool = True, + ): + r""" + The call function to the pipeline for generation. + + Args: + height (`int`): + The height in pixels of the generated image. + width (`int`): + The width in pixels of the generated image. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor, + pred_x0: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + prompt_embeds_t5, negative_prompt_embeds_t5, attention_mask_t5, uncond_attention_mask_t5 = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds_t5, + negative_prompt_embeds=negative_prompt_embeds_t5, + lora_scale=text_encoder_lora_scale, + embedder=self.embedder_t5, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([uncond_attention_mask, attention_mask]) + prompt_embeds_t5 = torch.cat([negative_prompt_embeds_t5, prompt_embeds_t5]) + attention_mask_t5 = torch.cat([uncond_attention_mask_t5, attention_mask_t5]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents(batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=latent_model_input.device) + + if use_fp16: + latent_model_input = latent_model_input.half() + t_expand = t_expand.half() + prompt_embeds = prompt_embeds.half() + ims = image_meta_size.half() if image_meta_size is not None else None + else: + ims = image_meta_size if image_meta_size is not None else None + + # predict the noise residual + if self.infer_mode in ["fa", "torch"]: + noise_pred = self.unet( + latent_model_input, + t_expand, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + ) + elif self.infer_mode == "trt": + noise_pred = self.unet( + x=latent_model_input.contiguous(), + t_emb=t_expand.contiguous(), + context=prompt_embeds.contiguous(), + image_meta_size=ims.contiguous(), + style=style.contiguous(), + freqs_cis_img0=freqs_cis_img[0].to(device).contiguous(), + freqs_cis_img1=freqs_cis_img[1].to(device).contiguous(), + text_embedding_mask=attention_mask.contiguous(), + encoder_hidden_states_t5=prompt_embeds_t5.contiguous(), + text_embedding_mask_t5=attention_mask_t5.contiguous(), + ) + else: + raise ValueError("Unknown infer_mode: {self.infer_mode}") + if learn_sigma: + noise_pred, _ = noise_pred.chunk(2, dim=1) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + results = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=True) + latents = results.prev_sample + pred_x0 = results.pred_original_sample if hasattr(results, 'pred_original_sample') else None + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents, pred_x0) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..7130b3d39a404c7fe6d09349ef6a11fe7c61d7d6 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/pipeline_controlnet.py @@ -0,0 +1,817 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import ( + deprecate, + logging, + replace_example_docstring, +) +from diffusers.utils.torch_utils import randn_tensor +from transformers import BertModel, BertTokenizer +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ..modules.models import HunYuanDiT + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionControlNetPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]): + A `BertTokenizer` or `CLIPTokenizer` to tokenize text. + unet (Optional[`HunYuanDiT`, `UNet2DConditionModel`]): + A `HunYuanDiT` or `UNet2DConditionModel` to denoise the encoded image latents. + Notice: Here we still keep the word `unet` for compatibility with the previous version of the pipeline. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: Union[BertModel, CLIPTextModel], + tokenizer: Union[BertTokenizer, CLIPTokenizer], + unet: Union[HunYuanDiT, UNet2DConditionModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + progress_bar_config: Dict[str, Any] = None, + embedder_t5=None, + infer_mode='torch', + controlnet=None, + ): + super().__init__() + + # ======================================================== + self.embedder_t5 = embedder_t5 + self.infer_mode = infer_mode + + # ======================================================== + if progress_bar_config is None: + progress_bar_config = {} + if not hasattr(self, '_progress_bar_config'): + self._progress_bar_config = {} + self._progress_bar_config.update(progress_bar_config) + # ======================================================== + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + controlnet=controlnet, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + embedder=None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + embedder: + T5 embedder (including text encoder and tokenizer) + """ + if embedder is None: + text_encoder = self.text_encoder + tokenizer = self.tokenizer + max_length = self.tokenizer.model_max_length + else: + text_encoder = embedder.model + tokenizer = embedder.tokenizer + max_length = embedder.max_length + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode( + untruncated_ids[:, tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + else: + attention_mask = None + + if text_encoder is not None: + prompt_embeds_dtype = text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_attention_mask = uncond_input.attention_mask.to(device) + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + uncond_attention_mask = uncond_attention_mask.repeat(num_images_per_prompt, 1) + else: + uncond_attention_mask = None + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + height: int, + width: int, + prompt: Union[str, List[str]] = None, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_t5: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_t5: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + image_meta_size: Optional[torch.LongTensor] = None, + style: Optional[torch.LongTensor] = None, + progress: bool = True, + use_fp16: bool = False, + freqs_cis_img: Optional[tuple] = None, + learn_sigma: bool = True, + image=None, + control_weight=1.0 + ): + r""" + The call function to the pipeline for generation. + + Args: + height (`int`): + The height in pixels of the generated image. + width (`int`): + The width in pixels of the generated image. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor, + pred_x0: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + prompt_embeds_t5, negative_prompt_embeds_t5, attention_mask_t5, uncond_attention_mask_t5 = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds_t5, + negative_prompt_embeds=negative_prompt_embeds_t5, + lora_scale=text_encoder_lora_scale, + embedder=self.embedder_t5, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([uncond_attention_mask, attention_mask]) + prompt_embeds_t5 = torch.cat([negative_prompt_embeds_t5, prompt_embeds_t5]) + attention_mask_t5 = torch.cat([uncond_attention_mask_t5, attention_mask_t5]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents(batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + condition = self.vae.encode(image.float()).latent_dist.sample(generator).mul_(self.vae.config.scaling_factor).half() + condition = torch.cat([condition] * 2) if do_classifier_free_guidance else condition + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=latent_model_input.device) + + if use_fp16: + latent_model_input = latent_model_input.half() + t_expand = t_expand.half() + prompt_embeds = prompt_embeds.half() + ims = image_meta_size.half() if image_meta_size is not None else None + else: + ims = image_meta_size if image_meta_size is not None else None + + # predict the noise residual + if self.infer_mode in ["fa", "torch"]: + controls = self.controlnet( + latent_model_input, + t_expand, + condition, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + ) + controls = [control * control_weight for control in controls] + noise_pred = self.unet( + latent_model_input, + t_expand, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + controls=controls, + ) + elif self.infer_mode == "trt": + noise_pred = self.unet( + x=latent_model_input.contiguous(), + t_emb=t_expand.contiguous(), + context=prompt_embeds.contiguous(), + image_meta_size=ims.contiguous(), + style=style.contiguous(), + freqs_cis_img0=freqs_cis_img[0].to(device).contiguous(), + freqs_cis_img1=freqs_cis_img[1].to(device).contiguous(), + text_embedding_mask=attention_mask.contiguous(), + encoder_hidden_states_t5=prompt_embeds_t5.contiguous(), + text_embedding_mask_t5=attention_mask_t5.contiguous(), + ) + else: + raise ValueError("Unknown infer_mode: {self.infer_mode}") + if learn_sigma: + noise_pred, _ = noise_pred.chunk(2, dim=1) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + results = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=True) + latents = results.prev_sample + pred_x0 = results.pred_original_sample if hasattr(results, 'pred_original_sample') else None + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents, pred_x0) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/respace.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/respace.py new file mode 100644 index 0000000000000000000000000000000000000000..1e2bf7c0a6bfd78a0295efba8e9f99ef95ca8471 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/diffusion/respace.py @@ -0,0 +1,144 @@ +import numpy as np +import torch as th + +from .gaussian_diffusion import GaussianDiffusion + + +def space_timesteps(num_timesteps, section_counts): + """ + Create a list of timesteps to use from an original diffusion process, + given the number of timesteps we want to take from equally-sized portions + of the original process. + For example, if there's 300 timesteps and the section counts are [10,15,20] + then the first 100 timesteps are strided to be 10 timesteps, the second 100 + are strided to be 15 timesteps, and the final 100 are strided to be 20. + If the stride is a string starting with "ddim", then the fixed striding + from the DDIM paper is used, and only one section is allowed. + :param num_timesteps: the number of diffusion steps in the original + process to divide up. + :param section_counts: either a list of numbers, or a string containing + comma-separated numbers, indicating the step count + per section. As a special case, use "ddimN" where N + is a number of steps to use the striding from the + DDIM paper. + :return: a set of diffusion steps from the original process to use. + """ + if isinstance(section_counts, str): + if section_counts.startswith("ddim"): + desired_count = int(section_counts[len("ddim") :]) + for i in range(1, num_timesteps): + if len(range(0, num_timesteps, i)) == desired_count: + return set(range(0, num_timesteps, i)) + raise ValueError( + f"cannot create exactly {num_timesteps} steps with an integer stride" + ) + section_counts = [int(x) for x in section_counts.split(",")] + size_per = num_timesteps // len(section_counts) + extra = num_timesteps % len(section_counts) + start_idx = 0 + all_steps = [] + for i, section_count in enumerate(section_counts): + size = size_per + (1 if i < extra else 0) + if size < section_count: + raise ValueError( + f"cannot divide section of {size} steps into {section_count}" + ) + if section_count <= 1: + frac_stride = 1 + else: + frac_stride = (size - 1) / (section_count - 1) + cur_idx = 0.0 + taken_steps = [] + for _ in range(section_count): + taken_steps.append(start_idx + round(cur_idx)) + cur_idx += frac_stride + all_steps += taken_steps + start_idx += size + return set(all_steps) + + +class SpacedDiffusion(GaussianDiffusion): + """ + Improved DDPM + + A diffusion process which can skip steps in a base diffusion process. + :param use_timesteps: a collection (sequence or set) of timesteps from the + original diffusion process to retain. + :param kwargs: the kwargs to create the base diffusion process. + """ + + def __init__(self, use_timesteps, **kwargs): + self.use_timesteps = set(use_timesteps) + self.timestep_map = [] + self.original_num_steps = len(kwargs["betas"]) + + base_diffusion = GaussianDiffusion(**kwargs) # pylint: disable=missing-kwoa + last_alpha_cumprod = 1.0 + new_betas = [] + for i, alpha_cumprod in enumerate(base_diffusion.alphas_cumprod): + if i in self.use_timesteps: + new_betas.append(1 - alpha_cumprod / last_alpha_cumprod) + last_alpha_cumprod = alpha_cumprod + self.timestep_map.append(i) + kwargs["betas"] = np.array(new_betas) + super().__init__(**kwargs) + + def p_mean_variance( + self, model, *args, **kwargs + ): # pylint: disable=signature-differs + return super().p_mean_variance(self._wrap_model(model), *args, **kwargs) + + def training_losses( + self, model, controlnet=None, *args, **kwargs + ): # pylint: disable=signature-differs + if controlnet != None: + return super().training_losses(self._wrap_model(model), controlnet=self._wrap_model(controlnet), *args, **kwargs) + else: + return super().training_losses(self._wrap_model(model), *args, **kwargs) + + def condition_mean(self, cond_fn, *args, **kwargs): + return super().condition_mean(self._wrap_model(cond_fn), *args, **kwargs) + + def condition_score(self, cond_fn, *args, **kwargs): + return super().condition_score(self._wrap_model(cond_fn), *args, **kwargs) + + def get_eps(self, model, *args, **kwargs): + return super().get_eps(self._wrap_model(model), *args, **kwargs) + + def _wrap_model(self, model): + if isinstance(model, _WrappedModel): + return model + return _WrappedModel( + model, self.timestep_map, self.original_num_steps + ) + + def _scale_timesteps(self, t): + # Scaling is done by the wrapped model. + return t + + +class _WrappedModel: + """ + Improved DDPM + + When using a subsequent timesteps (e.g., 250), we must wrap the model + for mapping the timesteps 1-250 with step 1 to 1-1000 with step 4 + """ + def __init__(self, model, timestep_map, original_num_steps): + self.model = model + self.timestep_map = timestep_map + # self.rescale_timesteps = rescale_timesteps + self.original_num_steps = original_num_steps + + def __call__(self, x, ts, **kwargs): + """ + Here we must make a interpolation because `ts` maybe a float (e.g., 4.5) + in the PLMS/PNDM sampler. + """ + ts = ts.float() + frac = ts.frac() + map_tensor = th.tensor(self.timestep_map, device=ts.device, dtype=ts.dtype) + new_ts_1 = map_tensor[ts.floor().long()] + new_ts_2 = map_tensor[ts.ceil().long()] + new_ts = th.lerp(new_ts_1, new_ts_2, frac) + return self.model(x, new_ts, **kwargs) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/ds_config.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/ds_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d108a6de496de852dd21ed755e04e7e7515a89ca --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/ds_config.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +import os + +def deepspeed_config_from_args(args, global_batch_size): + if args.use_zero_stage == 2: + deepspeed_config = { + "train_batch_size": global_batch_size, + "train_micro_batch_size_per_gpu": args.batch_size, + "gradient_accumulation_steps": args.grad_accu_steps, + "steps_per_print": args.log_every, + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": args.weight_decay + } + }, + + "zero_optimization": { + "stage": 2, + "reduce_scatter": False, + "reduce_bucket_size": 1e9, + }, + + "gradient_clipping": 1.0, + "prescale_gradients": True, + + "fp16": { + "enabled": args.use_fp16, + "loss_scale": 0, + "loss_scale_window": 500, + "hysteresis": 2, + "min_loss_scale": 1e-3, + "initial_scale_power": 15 + }, + + "bf16": { + "enabled": False + }, + + "wall_clock_breakdown": False + } + + elif args.use_zero_stage == 3: + deepspeed_config = { + "train_batch_size": args.global_batch_size, + # "train_micro_batch_size_per_gpu": args.batch_size, + "gradient_accumulation_steps": args.grad_accu_steps, + "steps_per_print": args.log_every, + + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": args.weight_decay + } + }, + + "zero_optimization": { + "stage": 3, + "allgather_partitions": True, + "overlap_comm": True, + "reduce_scatter": True, + "contiguous_gradients": True, + "stage3_prefetch_bucket_size": 5e8, + "stage3_max_live_parameters" : 6e8, + "reduce_bucket_size": 1.2e9, + "sub_group_size": 1e9, + "sub_group_buffer_num": 10, + "pipeline_optimizer": True, + "max_contigous_event_size": 0, + "cache_sub_group_rate": 0.0, + "prefetch_cache_sub_group_rate": 1.0, + "max_contigous_params_size": -1, + "max_param_reduce_events": 0, + "stage3_param_persistence_threshold": 9e9, + "is_communication_time_profiling": False, + "save_large_model_multi_slice": True, + "use_fused_op_with_grad_norm_overflow": False, + "offload_optimizer": { + "device": "cpu", + "pin_memory": True + }, + "offload_param": { + "device": "cpu", + "pin_memory": True + } + }, + + "gradient_clipping": 1.0, + "prescale_gradients": False, + + "fp16": { + "enabled": True, + "loss_scale": 0, + "loss_scale_window": 500, + "hysteresis": 2, + "min_loss_scale": 1, + "initial_scale_power": 15 + }, + + "bf16": { + "enabled": False + }, + + "wall_clock_breakdown": False, + "mem_chunk": { + "default_chunk_size": 536870911, + "use_fake_dist": False, + "client": { + "mem_tracer": { + "use_async_mem_monitor": True, + "warmup_gpu_chunk_mem_ratio": 0.8, + "overall_gpu_mem_ratio": 0.8, + "overall_cpu_mem_ratio": 1.0, + "margin_use_ratio": 0.8, + "use_fake_dist": False + }, + "opts": { + "with_mem_cache": True, + "with_async_move": True + } + } + } + } + else: + raise ValueError + return deepspeed_config + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..682f29d7faa2d17a567e9f4d623f6eb8e1361181 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference.py @@ -0,0 +1,413 @@ +import random +import time +from pathlib import Path + +import numpy as np +import torch + +# For reproducibility +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +from diffusers import schedulers +from diffusers.models import AutoencoderKL +from loguru import logger +from transformers import BertModel, BertTokenizer +from transformers.modeling_utils import logger as tf_logger + +from .constants import SAMPLER_FACTORY, NEGATIVE_PROMPT, TRT_MAX_WIDTH, TRT_MAX_HEIGHT, TRT_MAX_BATCH_SIZE +from .diffusion.pipeline import StableDiffusionPipeline +from .modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG +from .modules.posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from .modules.text_encoder import MT5Embedder +from .utils.tools import set_seeds +from peft import LoraConfig + + +class Resolution: + def __init__(self, width, height): + self.width = width + self.height = height + + def __str__(self): + return f'{self.height}x{self.width}' + + +class ResolutionGroup: + def __init__(self): + self.data = [ + Resolution(1024, 1024), # 1:1 + Resolution(1280, 1280), # 1:1 + Resolution(1024, 768), # 4:3 + Resolution(1152, 864), # 4:3 + Resolution(1280, 960), # 4:3 + Resolution(768, 1024), # 3:4 + Resolution(864, 1152), # 3:4 + Resolution(960, 1280), # 3:4 + Resolution(1280, 768), # 16:9 + Resolution(768, 1280), # 9:16 + ] + self.supported_sizes = set([(r.width, r.height) for r in self.data]) + + def is_valid(self, width, height): + return (width, height) in self.supported_sizes + + +STANDARD_RATIO = np.array([ + 1.0, # 1:1 + 4.0 / 3.0, # 4:3 + 3.0 / 4.0, # 3:4 + 16.0 / 9.0, # 16:9 + 9.0 / 16.0, # 9:16 +]) +STANDARD_SHAPE = [ + [(1024, 1024), (1280, 1280)], # 1:1 + [(1280, 960)], # 4:3 + [(960, 1280)], # 3:4 + [(1280, 768)], # 16:9 + [(768, 1280)], # 9:16 +] +STANDARD_AREA = [ + np.array([w * h for w, h in shapes]) + for shapes in STANDARD_SHAPE +] + + +def get_standard_shape(target_width, target_height): + """ + Map image size to standard size. + """ + target_ratio = target_width / target_height + closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio)) + closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height)) + width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx] + return width, height + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +def get_pipeline(args, vae, text_encoder, tokenizer, model, device, rank, + embedder_t5, infer_mode, sampler=None): + """ + Get scheduler and pipeline for sampling. The sampler and pipeline are both + based on diffusers and make some modifications. + + Returns + ------- + pipeline: StableDiffusionPipeline + sampler_name: str + """ + sampler = sampler or args.sampler + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args.noise_schedule + kwargs['beta_start'] = args.beta_start + kwargs['beta_end'] = args.beta_end + kwargs['prediction_type'] = args.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(args.infer_steps, device) + + # Only enable progress bar for rank 0 + progress_bar_config = {} if rank == 0 else {'disable': True} + + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=embedder_t5, + infer_mode=infer_mode, + ) + + pipeline = pipeline.to(device) + + return pipeline, sampler + + +class End2End(object): + def __init__(self, args, models_root_path): + self.args = args + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + # Disable BertModel logging checkpoint info + tf_logger.setLevel('ERROR') + + # ======================================================================== + logger.info(f"Loading CLIP Text Encoder...") + text_encoder_path = self.root / "clip_text_encoder" + self.clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + logger.info(f"Loading CLIP Text Encoder finished") + + # ======================================================================== + logger.info(f"Loading CLIP Tokenizer...") + tokenizer_path = self.root / "tokenizer" + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + logger.info(f"Loading CLIP Tokenizer finished") + + # ======================================================================== + logger.info(f"Loading T5 Text Encoder and T5 Tokenizer...") + t5_text_encoder_path = self.root / 'mt5' + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.embedder_t5 = embedder_t5 + logger.info(f"Loading t5_text_encoder and t5_tokenizer finished") + + # ======================================================================== + logger.info(f"Loading VAE...") + vae_path = self.root / "sdxl-vae-fp16-fix" + self.vae = AutoencoderKL.from_pretrained(str(vae_path)).to(self.device) + logger.info(f"Loading VAE finished") + + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + self.patch_size = model_config['patch_size'] + self.head_size = model_config['hidden_size'] // model_config['num_heads'] + self.resolutions, self.freqs_cis_img = self.standard_shapes() # Used for TensorRT models + self.image_size = _to_tuple(self.args.image_size) + latent_size = (self.image_size[0] // 8, self.image_size[1] // 8) + + self.infer_mode = self.args.infer_mode + if self.infer_mode in ['fa', 'torch']: + model_dir = self.root / "model" + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + # Build model structure + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + # Load model checkpoint + logger.info(f"Loading torch model {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + + lora_ckpt = args.lora_ckpt + if lora_ckpt is not None and lora_ckpt != "": + logger.info(f"Loading Lora checkpoint {lora_ckpt}...") + + self.model.load_adapter(lora_ckpt) + self.model.merge_and_unload() + + + self.model.eval() + logger.info(f"Loading torch model finished") + elif self.infer_mode == 'trt': + from .modules.trt.hcf_model import TRTModel + + trt_dir = self.root / "model_trt" + engine_dir = trt_dir / "engine" + plugin_path = trt_dir / "fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so" + model_name = "model_onnx" + + logger.info(f"Loading TensorRT model {engine_dir}/{model_name}...") + self.model = TRTModel(model_name=model_name, + engine_dir=str(engine_dir), + image_height=TRT_MAX_HEIGHT, + image_width=TRT_MAX_WIDTH, + text_maxlen=args.text_len, + embedding_dim=args.text_states_dim, + plugin_path=str(plugin_path), + max_batch_size=TRT_MAX_BATCH_SIZE, + ) + logger.info(f"Loading TensorRT model finished") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Build inference pipeline. We use a customized StableDiffusionPipeline. + logger.info(f"Loading inference pipeline...") + self.pipeline, self.sampler = self.load_sampler() + logger.info(f'Loading pipeline finished') + + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_sampler(self, sampler=None): + pipeline, sampler = get_pipeline(self.args, + self.vae, + self.clip_text_encoder, + self.tokenizer, + self.model, + device=self.device, + rank=0, + embedder_t5=self.embedder_t5, + infer_mode=self.infer_mode, + sampler=sampler, + ) + return pipeline, sampler + + def calc_rope(self, height, width): + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(self.head_size, *sub_args) + return rope + + def standard_shapes(self): + resolutions = ResolutionGroup() + freqs_cis_img = {} + for reso in resolutions.data: + freqs_cis_img[str(reso)] = self.calc_rope(reso.height, reso.width) + return resolutions, freqs_cis_img + + def predict(self, + user_prompt, + height=1024, + width=1024, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=100, + guidance_scale=6, + batch_size=1, + src_size_cond=(1024, 1024), + sampler=None, + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = set_seeds(seed, device=self.device) + # ======================================================================== + # Arguments: target_width, target_height + # ======================================================================== + if width <= 0 or height <= 0: + raise ValueError(f"`height` and `width` must be positive integers, got height={height}, width={width}") + logger.info(f"Input (height, width) = ({height}, {width})") + if self.infer_mode in ['fa', 'torch']: + # We must force height and width to align to 16 and to be an integer. + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + logger.info(f"Align to 16: (height, width) = ({target_height}, {target_width})") + elif self.infer_mode == 'trt': + target_width, target_height = get_standard_shape(width, height) + logger.info(f"Align to standard shape: (height, width) = ({target_height}, {target_width})") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is None or negative_prompt == '': + negative_prompt = self.default_negative_prompt + if not isinstance(negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + # ======================================================================== + # Arguments: style. (A fixed argument. Don't Change it.) + # ======================================================================== + style = torch.as_tensor([0, 0] * batch_size, device=self.device) + + # ======================================================================== + # Inner arguments: image_meta_size (Please refer to SDXL.) + # ======================================================================== + if isinstance(src_size_cond, int): + src_size_cond = [src_size_cond, src_size_cond] + if not isinstance(src_size_cond, (list, tuple)): + raise TypeError(f"`src_size_cond` must be a list or tuple, but got {type(src_size_cond)}") + if len(src_size_cond) != 2: + raise ValueError(f"`src_size_cond` must be a tuple of 2 integers, but got {len(src_size_cond)}") + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * 2 * batch_size, device=self.device) + + # ======================================================================== + start_time = time.time() + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + (height, width): {(target_height, target_width)} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + image_meta_size: {size_cond} + """) + reso = f'{target_height}x{target_width}' + if reso in self.freqs_cis_img: + freqs_cis_img = self.freqs_cis_img[reso] + else: + freqs_cis_img = self.calc_rope(target_height, target_width) + + if sampler is not None and sampler != self.sampler: + self.pipeline, self.sampler = self.load_sampler(sampler) + + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + )[0] + gen_time = time.time() - start_time + logger.debug(f"Success, time: {gen_time}") + + return { + 'images': samples, + 'seed': seed, + } diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_comfyui.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_comfyui.py new file mode 100644 index 0000000000000000000000000000000000000000..5696eff6429bdee8c0967b5d5cd8b52fa9de3af8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_comfyui.py @@ -0,0 +1,433 @@ +import random +import time +from pathlib import Path + +import numpy as np +import torch + +# For reproducibility +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +from diffusers import schedulers +from diffusers.models import AutoencoderKL +from loguru import logger +from transformers import BertModel, BertTokenizer +from transformers.modeling_utils import logger as tf_logger + +from .constants import SAMPLER_FACTORY, NEGATIVE_PROMPT +from .diffusion.pipeline import StableDiffusionPipeline +from .diffusion.pipeline_controlnet import StableDiffusionControlNetPipeline +from .modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG +from .modules.posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from .modules.text_encoder import MT5Embedder +from .utils.tools import set_seeds +from peft import LoraConfig + + +class Resolution: + def __init__(self, width, height): + self.width = width + self.height = height + + def __str__(self): + return f'{self.height}x{self.width}' + + +class ResolutionGroup: + def __init__(self): + self.data = [ + Resolution(768, 768), # 1:1 + Resolution(1024, 1024), # 1:1 + Resolution(1280, 1280), # 1:1 + Resolution(1024, 768), # 4:3 + Resolution(1152, 864), # 4:3 + Resolution(1280, 960), # 4:3 + Resolution(768, 1024), # 3:4 + Resolution(864, 1152), # 3:4 + Resolution(960, 1280), # 3:4 + Resolution(1280, 768), # 16:9 + Resolution(768, 1280), # 9:16 + ] + self.supported_sizes = set([(r.width, r.height) for r in self.data]) + + def is_valid(self, width, height): + return (width, height) in self.supported_sizes + + +STANDARD_RATIO = np.array([ + 1.0, # 1:1 + 4.0 / 3.0, # 4:3 + 3.0 / 4.0, # 3:4 + 16.0 / 9.0, # 16:9 + 9.0 / 16.0, # 9:16 +]) +STANDARD_SHAPE = [ + [(768, 768), (1024, 1024), (1280, 1280)], # 1:1 + [(1024, 768), (1152, 864), (1280, 960)], # 4:3 + [(768, 1024), (864, 1152), (960, 1280)], # 3:4 + [(1280, 768)], # 16:9 + [(768, 1280)], # 9:16 +] +STANDARD_AREA = [ + np.array([w * h for w, h in shapes]) + for shapes in STANDARD_SHAPE +] + + +def get_standard_shape(target_width, target_height): + """ + Map image size to standard size. + """ + target_ratio = target_width / target_height + closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio)) + closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height)) + width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx] + return width, height + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +def get_pipeline(args, vae, text_encoder, tokenizer, model, device, rank, + embedder_t5, infer_mode, sampler=None): + """ + Get scheduler and pipeline for sampling. The sampler and pipeline are both + based on diffusers and make some modifications. + + Returns + ------- + pipeline: StableDiffusionPipeline + sampler_name: str + """ + sampler = sampler or args.sampler + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args.noise_schedule + kwargs['beta_start'] = args.beta_start + kwargs['beta_end'] = args.beta_end + kwargs['prediction_type'] = args.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + #print(scheduler) + #assert(0) + + # Set timesteps for inference steps. + scheduler.set_timesteps(args.infer_steps, device) + + # Only enable progress bar for rank 0 + progress_bar_config = {} if rank == 0 else {'disable': True} + + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=embedder_t5, + infer_mode=infer_mode, + ) + + pipeline = pipeline.to(device) + + return pipeline, sampler + + +class End2End(object): + def __init__(self, args, models_root_path, MODEL_PATH = None, VAE_PATH = None, LOAR_PATH = None): + self.args = args + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + # Disable BertModel logging checkpoint info + tf_logger.setLevel('ERROR') + + # ======================================================================== + model_dir = self.root / "model" + + # ======================================================================== + logger.info(f"Loading CLIP Text Encoder...") + text_encoder_path = self.root / "clip_text_encoder" + self.clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + logger.info(f"Loading CLIP Text Encoder finished") + #print(self.clip_text_encoder) + + # ======================================================================== + logger.info(f"Loading CLIP Tokenizer...") + tokenizer_path = self.root / "tokenizer" + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + logger.info(f"Loading CLIP Tokenizer finished") + #print(self.tokenizer) + #assert(0) + + # ======================================================================== + logger.info(f"Loading T5 Text Encoder and T5 Tokenizer...") + t5_text_encoder_path = self.root / 'mt5' + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.embedder_t5 = embedder_t5 + logger.info(f"Loading t5_text_encoder and t5_tokenizer finished") + + # ======================================================================== + logger.info(f"Loading VAE...") + if VAE_PATH: + vae_path = VAE_PATH + self.vae = AutoencoderKL.from_single_file(str(vae_path)).to(self.device) + else: + vae_path = self.root / "sdxl-vae-fp16-fix" + self.vae = AutoencoderKL.from_pretrained(str(vae_path)).to(self.device) + + + logger.info(f"Loading VAE finished") + + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + #print(self.args.model) + #print(self.args) + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + self.patch_size = model_config['patch_size'] + self.head_size = model_config['hidden_size'] // model_config['num_heads'] + self.resolutions, self.freqs_cis_img = self.standard_shapes() # Used for TensorRT models + self.image_size = _to_tuple(self.args.image_size) + latent_size = (self.image_size[0] // 8, self.image_size[1] // 8) + + self.infer_mode = self.args.infer_mode + if self.infer_mode in ['fa', 'torch']: + if MODEL_PATH: + model_path = Path(MODEL_PATH) + else: + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + # Build model structure + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + # Load model checkpoint + logger.info(f"Loading model checkpoint {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + if LOAR_PATH: + self.model.load_adapter(LOAR_PATH) + self.model.merge_and_unload() + + self.model.eval() + elif self.infer_mode == 'trt': + raise NotImplementedError("TensorRT model is not supported yet.") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Build inference pipeline. We use a customized StableDiffusionPipeline. + logger.info(f"Loading inference pipeline...") + #self.pipeline, self.sampler = self.load_sampler() + logger.info(f'Loading pipeline finished') + + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_sampler(self, sampler=None): + pipeline, sampler = get_pipeline(self.args, + self.vae, + self.clip_text_encoder, + self.tokenizer, + self.model, + device=self.device, + rank=0, + embedder_t5=self.embedder_t5, + infer_mode=self.infer_mode, + sampler=sampler, + ) + return pipeline, sampler + + def calc_rope(self, height, width): + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(self.head_size, *sub_args) + return rope + + def standard_shapes(self): + resolutions = ResolutionGroup() + freqs_cis_img = {} + for reso in resolutions.data: + freqs_cis_img[str(reso)] = self.calc_rope(reso.height, reso.width) + return resolutions, freqs_cis_img + + def predict(self, + user_prompt, + image, + height=1024, + width=1024, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=100, + guidance_scale=6, + batch_size=1, + src_size_cond=(1024, 1024), + sampler=None, + control_weight = 1.0 + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = set_seeds(seed, device=self.device) + + # ======================================================================== + # Arguments: target_width, target_height + # ======================================================================== + if width <= 0 or height <= 0: + raise ValueError(f"`height` and `width` must be positive integers, got height={height}, width={width}") + logger.info(f"Input (height, width) = ({height}, {width})") + if self.infer_mode in ['fa', 'torch']: + # We must force height and width to align to 16 and to be an integer. + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + logger.info(f"Align to 16: (height, width) = ({target_height}, {target_width})") + elif self.infer_mode == 'trt': + target_width, target_height = get_standard_shape(width, height) + logger.info(f"Align to standard shape: (height, width) = ({target_height}, {target_width})") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is None or negative_prompt == '': + negative_prompt = self.default_negative_prompt + if not isinstance(negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + # ======================================================================== + # Arguments: style. (A fixed argument. Don't Change it.) + # ======================================================================== + style = torch.as_tensor([0, 0] * batch_size, device=self.device) + + # ======================================================================== + # Inner arguments: image_meta_size (Please refer to SDXL.) + # ======================================================================== + if isinstance(src_size_cond, int): + src_size_cond = [src_size_cond, src_size_cond] + if not isinstance(src_size_cond, (list, tuple)): + raise TypeError(f"`src_size_cond` must be a list or tuple, but got {type(src_size_cond)}") + if len(src_size_cond) != 2: + raise ValueError(f"`src_size_cond` must be a tuple of 2 integers, but got {len(src_size_cond)}") + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * 2 * batch_size, device=self.device) + + # ======================================================================== + start_time = time.time() + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + (height, width): {(target_height, target_width)} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + image_meta_size: {size_cond} + """) + reso = f'{target_height}x{target_width}' + if reso in self.freqs_cis_img: + freqs_cis_img = self.freqs_cis_img[reso] + else: + freqs_cis_img = self.calc_rope(target_height, target_width) + + #if sampler is not None and sampler != self.sampler: + # self.pipeline, self.sampler = self.load_sampler(sampler) + if isinstance(image, torch.Tensor): + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + image=image, + control_weight=control_weight, + )[0] + else: + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + )[0] + gen_time = time.time() - start_time + logger.debug(f"Success, time: {gen_time}") + + return { + 'images': samples, + 'seed': seed, + } diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..1c50ab2c627fb33be2568c01b69601101f8c292c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/inference_controlnet.py @@ -0,0 +1,432 @@ +import random +import time +from pathlib import Path + +import numpy as np +import torch + +# For reproducibility +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +from diffusers import schedulers +from diffusers.models import AutoencoderKL +from loguru import logger +from transformers import BertModel, BertTokenizer +from transformers.modeling_utils import logger as tf_logger + +from .constants import SAMPLER_FACTORY, NEGATIVE_PROMPT, TRT_MAX_WIDTH, TRT_MAX_HEIGHT, TRT_MAX_BATCH_SIZE +from .diffusion.pipeline_controlnet import StableDiffusionControlNetPipeline +from .modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG +from .modules.controlnet import HunYuanControlNet +from .modules.posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from .modules.text_encoder import MT5Embedder +from .utils.tools import set_seeds +from peft import LoraConfig + + +class Resolution: + def __init__(self, width, height): + self.width = width + self.height = height + + def __str__(self): + return f'{self.height}x{self.width}' + + +class ResolutionGroup: + def __init__(self): + self.data = [ + Resolution(1024, 1024), # 1:1 + Resolution(1280, 1280), # 1:1 + Resolution(1024, 768), # 4:3 + Resolution(1152, 864), # 4:3 + Resolution(1280, 960), # 4:3 + Resolution(768, 1024), # 3:4 + Resolution(864, 1152), # 3:4 + Resolution(960, 1280), # 3:4 + Resolution(1280, 768), # 16:9 + Resolution(768, 1280), # 9:16 + ] + self.supported_sizes = set([(r.width, r.height) for r in self.data]) + + def is_valid(self, width, height): + return (width, height) in self.supported_sizes + + +STANDARD_RATIO = np.array([ + 1.0, # 1:1 + 4.0 / 3.0, # 4:3 + 3.0 / 4.0, # 3:4 + 16.0 / 9.0, # 16:9 + 9.0 / 16.0, # 9:16 +]) +STANDARD_SHAPE = [ + [(1024, 1024), (1280, 1280)], # 1:1 + [(1280, 960)], # 4:3 + [(960, 1280)], # 3:4 + [(1280, 768)], # 16:9 + [(768, 1280)], # 9:16 +] +STANDARD_AREA = [ + np.array([w * h for w, h in shapes]) + for shapes in STANDARD_SHAPE +] + + +def get_standard_shape(target_width, target_height): + """ + Map image size to standard size. + """ + target_ratio = target_width / target_height + closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio)) + closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height)) + width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx] + return width, height + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +def get_pipeline(args, vae, text_encoder, tokenizer, model, device, rank, + embedder_t5, infer_mode, controlnet, sampler=None): + """ + Get scheduler and pipeline for sampling. The sampler and pipeline are both + based on diffusers and make some modifications. + + Returns + ------- + pipeline: StableDiffusionControlNetPipeline + sampler_name: str + """ + sampler = sampler or args.sampler + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args.noise_schedule + kwargs['beta_start'] = args.beta_start + kwargs['beta_end'] = args.beta_end + kwargs['prediction_type'] = args.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(args.infer_steps, device) + + # Only enable progress bar for rank 0 + progress_bar_config = {} if rank == 0 else {'disable': True} + + pipeline = StableDiffusionControlNetPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=embedder_t5, + infer_mode=infer_mode, + controlnet=controlnet + ) + + pipeline = pipeline.to(device) + + return pipeline, sampler + + +class End2End(object): + def __init__(self, args, models_root_path): + self.args = args + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + # Disable BertModel logging checkpoint info + tf_logger.setLevel('ERROR') + + # ======================================================================== + logger.info(f"Loading CLIP Text Encoder...") + text_encoder_path = self.root / "clip_text_encoder" + self.clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + logger.info(f"Loading CLIP Text Encoder finished") + + # ======================================================================== + logger.info(f"Loading CLIP Tokenizer...") + tokenizer_path = self.root / "tokenizer" + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + logger.info(f"Loading CLIP Tokenizer finished") + + # ======================================================================== + logger.info(f"Loading T5 Text Encoder and T5 Tokenizer...") + t5_text_encoder_path = self.root / 'mt5' + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.embedder_t5 = embedder_t5 + logger.info(f"Loading t5_text_encoder and t5_tokenizer finished") + + # ======================================================================== + logger.info(f"Loading VAE...") + vae_path = self.root / "sdxl-vae-fp16-fix" + self.vae = AutoencoderKL.from_pretrained(str(vae_path)).to(self.device) + logger.info(f"Loading VAE finished") + + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + self.patch_size = model_config['patch_size'] + self.head_size = model_config['hidden_size'] // model_config['num_heads'] + self.resolutions, self.freqs_cis_img = self.standard_shapes() # Used for TensorRT models + self.image_size = _to_tuple(self.args.image_size) + latent_size = (self.image_size[0] // 8, self.image_size[1] // 8) + + self.infer_mode = self.args.infer_mode + if self.infer_mode in ['fa', 'torch']: + model_dir = self.root / "model" + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + # Build model structure + controlnet_dir = self.root / "controlnet" + controlnet_path = controlnet_dir / f"pytorch_model_{self.args.control_type}_{self.args.load_key}.pt" + if not controlnet_path.exists(): + raise ValueError(f"controlnet_path not exists: {controlnet_path}") + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + self.controlnet = HunYuanControlNet(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) + controlnet_state_dict = torch.load(controlnet_path) + self.controlnet.load_state_dict(controlnet_state_dict) + logger.info(f"Loading controlnet finished") + # Load model checkpoint + logger.info(f"Loading torch model {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + + lora_ckpt = args.lora_ckpt + if lora_ckpt is not None and lora_ckpt != "": + logger.info(f"Loading Lora checkpoint {lora_ckpt}...") + + self.model.load_adapter(lora_ckpt) + self.model.merge_and_unload() + + + self.model.eval() + self.controlnet.eval() + logger.info(f"Loading torch model finished") + elif self.infer_mode == 'trt': + from .modules.trt.hcf_model import TRTModel + + trt_dir = self.root / "model_trt" + engine_dir = trt_dir / "engine" + plugin_path = trt_dir / "fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so" + model_name = "model_onnx" + + logger.info(f"Loading TensorRT model {engine_dir}/{model_name}...") + self.model = TRTModel(model_name=model_name, + engine_dir=str(engine_dir), + image_height=TRT_MAX_HEIGHT, + image_width=TRT_MAX_WIDTH, + text_maxlen=args.text_len, + embedding_dim=args.text_states_dim, + plugin_path=str(plugin_path), + max_batch_size=TRT_MAX_BATCH_SIZE, + ) + logger.info(f"Loading TensorRT model finished") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Build inference pipeline. We use a customized StableDiffusionControlNetPipeline. + logger.info(f"Loading inference pipeline...") + self.pipeline, self.sampler = self.load_sampler() + logger.info(f'Loading pipeline finished') + + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_sampler(self, sampler=None): + pipeline, sampler = get_pipeline(self.args, + self.vae, + self.clip_text_encoder, + self.tokenizer, + self.model, + device=self.device, + rank=0, + embedder_t5=self.embedder_t5, + infer_mode=self.infer_mode, + sampler=sampler, + controlnet=self.controlnet + ) + return pipeline, sampler + + def calc_rope(self, height, width): + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(self.head_size, *sub_args) + return rope + + def standard_shapes(self): + resolutions = ResolutionGroup() + freqs_cis_img = {} + for reso in resolutions.data: + freqs_cis_img[str(reso)] = self.calc_rope(reso.height, reso.width) + return resolutions, freqs_cis_img + + def predict(self, + user_prompt, + image, + height=1024, + width=1024, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=100, + guidance_scale=6, + batch_size=1, + src_size_cond=(1024, 1024), + sampler=None, + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = set_seeds(seed, device=self.device) + # ======================================================================== + # Arguments: target_width, target_height + # ======================================================================== + if width <= 0 or height <= 0: + raise ValueError(f"`height` and `width` must be positive integers, got height={height}, width={width}") + logger.info(f"Input (height, width) = ({height}, {width})") + if self.infer_mode in ['fa', 'torch']: + # We must force height and width to align to 16 and to be an integer. + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + logger.info(f"Align to 16: (height, width) = ({target_height}, {target_width})") + elif self.infer_mode == 'trt': + target_width, target_height = get_standard_shape(width, height) + logger.info(f"Align to standard shape: (height, width) = ({target_height}, {target_width})") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is None or negative_prompt == '': + negative_prompt = self.default_negative_prompt + if not isinstance(negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + # ======================================================================== + # Arguments: style. (A fixed argument. Don't Change it.) + # ======================================================================== + style = torch.as_tensor([0, 0] * batch_size, device=self.device) + + # ======================================================================== + # Inner arguments: image_meta_size (Please refer to SDXL.) + # ======================================================================== + if isinstance(src_size_cond, int): + src_size_cond = [src_size_cond, src_size_cond] + if not isinstance(src_size_cond, (list, tuple)): + raise TypeError(f"`src_size_cond` must be a list or tuple, but got {type(src_size_cond)}") + if len(src_size_cond) != 2: + raise ValueError(f"`src_size_cond` must be a tuple of 2 integers, but got {len(src_size_cond)}") + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * 2 * batch_size, device=self.device) + + # ======================================================================== + start_time = time.time() + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + (height, width): {(target_height, target_width)} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + image_meta_size: {size_cond} + """) + reso = f'{target_height}x{target_width}' + if reso in self.freqs_cis_img: + freqs_cis_img = self.freqs_cis_img[reso] + else: + freqs_cis_img = self.calc_rope(target_height, target_width) + + if sampler is not None and sampler != self.sampler: + self.pipeline, self.sampler = self.load_sampler(sampler) + + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + image=image, + control_weight=self.args.control_weight, + )[0] + gen_time = time.time() - start_time + logger.debug(f"Success, time: {gen_time}") + + return { + 'images': samples, + 'seed': seed, + } diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/lr_scheduler.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..027c7e6b42eaa8bfeac48a2405d479869bdda683 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/lr_scheduler.py @@ -0,0 +1,761 @@ +""" +Implementation of learning rate schedules. + +Taken and modified from PyTorch v1.0.1 source +https://github.com/pytorch/pytorch/blob/v1.1.0/torch/optim/lr_scheduler.py +""" + +import argparse +from torch.optim import Optimizer +import math + +LR_SCHEDULE = 'lr_schedule' +LR_RANGE_TEST = 'LRRangeTest' +ONE_CYCLE = 'OneCycle' +WARMUP_LR = 'WarmupLR' +WARMUP_DECAY_LR = 'WarmupDecayLR' +VALID_LR_SCHEDULES = [LR_RANGE_TEST, ONE_CYCLE, WARMUP_LR, WARMUP_DECAY_LR] + +LR_RANGE_TEST_MIN_LR = 'lr_range_test_min_lr' +LR_RANGE_TEST_STEP_RATE = 'lr_range_test_step_rate' +LR_RANGE_TEST_STEP_SIZE = 'lr_range_test_step_size' +LR_RANGE_TEST_STAIRCASE = 'lr_range_test_staircase' + +EDGE_VALUE = 'edge_value' +MID_VALUE = 'mid_value' + +CYCLE_FIRST_STEP_SIZE = 'cycle_first_step_size' +CYCLE_FIRST_STAIR_COUNT = 'cycle_first_stair_count' +CYCLE_SECOND_STEP_SIZE = 'cycle_second_step_size' +CYCLE_SECOND_STAIR_COUNT = 'cycle_second_stair_count' +DECAY_STEP_SIZE = 'decay_step_size' + +CYCLE_MIN_LR = 'cycle_min_lr' +CYCLE_MAX_LR = 'cycle_max_lr' +DECAY_LR_RATE = 'decay_lr_rate' + +CYCLE_MIN_MOM = 'cycle_min_mom' +CYCLE_MAX_MOM = 'cycle_max_mom' +DECAY_MOM_RATE = 'decay_mom_rate' + +WARMUP_MIN_LR = 'warmup_min_lr' +WARMUP_MAX_LR = 'warmup_max_lr' +WARMUP_NUM_STEPS = 'warmup_num_steps' +WARMUP_TYPE = 'warmup_type' +WARMUP_LOG_RATE = 'log' +WARMUP_LINEAR_RATE = 'linear' + +TOTAL_NUM_STEPS = 'total_num_steps' + + +def add_tuning_arguments(parser): + group = parser.add_argument_group('Convergence Tuning', 'Convergence tuning configurations') + + # LR scheduler + group.add_argument('--lr_schedule', type=str, default=None, help='LR schedule for training.') + + # Learning rate range test + group.add_argument("--lr_range_test_min_lr", type=float, default=0.001, help='Starting lr value.') + group.add_argument("--lr_range_test_step_rate", type=float, default=1.0, help='scaling rate for LR range test.') + group.add_argument("--lr_range_test_step_size", type=int, default=1000, help='training steps per LR change.') + group.add_argument("--lr_range_test_staircase", + type=bool, + default=False, + help='use staircase scaling for LR range test.') + + # OneCycle schedule + group.add_argument("--cycle_first_step_size", + type=int, + default=1000, + help='size of first step of 1Cycle schedule (training steps).') + group.add_argument("--cycle_first_stair_count", + type=int, + default=-1, + help='first stair count for 1Cycle schedule.') + group.add_argument("--cycle_second_step_size", + type=int, + default=-1, + help='size of second step of 1Cycle schedule (default first_step_size).') + group.add_argument("--cycle_second_stair_count", + type=int, + default=-1, + help='second stair count for 1Cycle schedule.') + group.add_argument("--decay_step_size", + type=int, + default=1000, + help='size of intervals for applying post cycle decay (training steps).') + + # 1Cycle LR + group.add_argument("--cycle_min_lr", type=float, default=0.01, help='1Cycle LR lower bound.') + group.add_argument("--cycle_max_lr", type=float, default=0.1, help='1Cycle LR upper bound.') + group.add_argument("--decay_lr_rate", type=float, default=0.0, help='post cycle LR decay rate.') + + # 1Cycle Momentum + group.add_argument('--cycle_momentum', default=False, action='store_true', help='Enable 1Cycle momentum schedule.') + group.add_argument("--cycle_min_mom", type=float, default=0.8, help='1Cycle momentum lower bound.') + group.add_argument("--cycle_max_mom", type=float, default=0.9, help='1Cycle momentum upper bound.') + group.add_argument("--decay_mom_rate", type=float, default=0.0, help='post cycle momentum decay rate.') + + # Warmup LR + group.add_argument('--warmup_min_lr', type=float, default=0, help='WarmupLR minimum/initial LR value') + group.add_argument('--warmup_max_lr', type=float, default=0.001, help='WarmupLR maximum LR value.') + group.add_argument('--warmup_num_steps', type=int, default=1000, help='WarmupLR step count for LR warmup.') + group.add_argument('--warmup_type', + type=str, + default=WARMUP_LOG_RATE, + help='WarmupLR increasing function during warmup') + return parser + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser = add_tuning_arguments(parser) + + lr_sched_args, unknown_args = parser.parse_known_args() + return lr_sched_args, unknown_args + + +def override_lr_range_test_params(args, params): + if hasattr(args, LR_RANGE_TEST_MIN_LR) and args.lr_range_test_min_lr is not None: + params[LR_RANGE_TEST_MIN_LR] = args.lr_range_test_min_lr + + if hasattr(args, LR_RANGE_TEST_STEP_RATE) and args.lr_range_test_step_rate is not None: + params[LR_RANGE_TEST_STEP_RATE] = args.lr_range_test_step_rate + + if hasattr(args, LR_RANGE_TEST_STEP_SIZE) and args.lr_range_test_step_size is not None: + params[LR_RANGE_TEST_STEP_SIZE] = args.lr_range_test_step_size + + if hasattr(args, LR_RANGE_TEST_STAIRCASE) and args.lr_range_test_staircase is not None: + params[LR_RANGE_TEST_STAIRCASE] = args.lr_range_test_staircase + + +def override_1cycle_params(args, params): + if hasattr(args, CYCLE_FIRST_STEP_SIZE) and args.cycle_first_step_size is not None: + params[CYCLE_FIRST_STEP_SIZE] = args.cycle_first_step_size + + if hasattr(args, CYCLE_FIRST_STAIR_COUNT) and args.cycle_first_stair_count is not None: + params[CYCLE_FIRST_STAIR_COUNT] = args.cycle_first_stair_count + + if hasattr(args, CYCLE_SECOND_STEP_SIZE) and args.cycle_second_step_size is not None: + params[CYCLE_SECOND_STEP_SIZE] = args.cycle_second_step_size + + if hasattr(args, CYCLE_SECOND_STAIR_COUNT) and args.cycle_second_stair_count is not None: + params[CYCLE_SECOND_STAIR_COUNT] = args.cycle_second_stair_count + + if hasattr(args, DECAY_STEP_SIZE) and args.decay_step_size is not None: + params[DECAY_STEP_SIZE] = args.decay_step_size + + # 1Cycle LR params + if hasattr(args, CYCLE_MIN_LR) and args.cycle_min_lr is not None: + params[CYCLE_MIN_LR] = args.cycle_min_lr + + if hasattr(args, CYCLE_MAX_LR) and args.cycle_max_lr is not None: + params[CYCLE_MAX_LR] = args.cycle_max_lr + + if hasattr(args, DECAY_LR_RATE) and args.decay_lr_rate is not None: + params[DECAY_LR_RATE] = args.decay_lr_rate + + # 1Cycle MOM params + if hasattr(args, CYCLE_MIN_MOM) and args.cycle_min_mom is not None: + params[CYCLE_MIN_MOM] = args.cycle_min_mom + + if hasattr(args, CYCLE_MAX_MOM) and args.cycle_max_mom is not None: + params[CYCLE_MAX_MOM] = args.cycle_max_mom + + if hasattr(args, DECAY_MOM_RATE) and args.decay_mom_rate is not None: + params[DECAY_MOM_RATE] = args.decay_mom_rate + + +def override_warmupLR_params(args, params): + if hasattr(args, WARMUP_MIN_LR) and args.warmup_min_lr is not None: + params[WARMUP_MIN_LR] = args.warmup_min_lr + + if hasattr(args, WARMUP_MAX_LR) and args.warmup_max_lr is not None: + params[WARMUP_MAX_LR] = args.warmup_max_lr + + if hasattr(args, WARMUP_NUM_STEPS) and args.warmup_num_steps is not None: + params[WARMUP_NUM_STEPS] = args.warmup_num_steps + + if hasattr(args, WARMUP_TYPE) and args.warmup_type is not None: + params[WARMUP_TYPE] = args.warmup_type + + +def override_params(args, params): + # LR range test params + override_lr_range_test_params(args, params) + + # 1Cycle params + override_1cycle_params(args, params) + + # WarmupLR params + override_warmupLR_params(args, params) + + +def get_config_from_args(args): + if not hasattr(args, LR_SCHEDULE) or args.lr_schedule is None: + return None, '--{} not specified on command line'.format(LR_SCHEDULE) + + if not args.lr_schedule in VALID_LR_SCHEDULES: + return None, '{} is not supported LR schedule'.format(args.lr_schedule) + + config = {} + config['type'] = args.lr_schedule + config['params'] = {} + + if args.lr_schedule == LR_RANGE_TEST: + override_lr_range_test_params(args, config['params']) + elif args.lr_schedule == ONE_CYCLE: + override_1cycle_params(args, config['params']) + else: + override_warmupLR_params(args, config['params']) + + return config, None + + +def get_lr_from_config(config): + if not 'type' in config: + return None, 'LR schedule type not defined in config' + + if not 'params' in config: + return None, 'LR schedule params not defined in config' + + lr_schedule = config['type'] + lr_params = config['params'] + + if not lr_schedule in VALID_LR_SCHEDULES: + return None, '{} is not a valid LR schedule'.format(lr_schedule) + + if lr_schedule == LR_RANGE_TEST: + return lr_params[LR_RANGE_TEST_MIN_LR], '' + if lr_schedule == ONE_CYCLE: + return lr_params[CYCLE_MAX_LR], '' + # Warmup LR + return lr_params[WARMUP_MAX_LR], '' + + +""" +Only optimizers that are subclass of torch.optim.Optimizer are supported. So check the passed optimizer and wrapped +optimizer to see if requirement is satisfied. +TODO: Looking under the hood to examine the wrapped optimizer is a hack that requires a better long-term fix. +""" + + +def get_torch_optimizer(optimizer): + if isinstance(optimizer, Optimizer): + return optimizer + + if hasattr(optimizer, 'optimizer') and isinstance(optimizer.optimizer, Optimizer): + return optimizer.optimizer + + raise TypeError('{} is not a subclass of torch.optim.Optimizer'.format(type(optimizer).__name__)) + + +class LRRangeTest(object): + """Sets the learning rate of each parameter group according to + learning rate range test (LRRT) policy. The policy increases learning + rate starting from a base value with a constant frequency, as detailed in + the paper `A disciplined approach to neural network hyper-parameters: Part1`_. + + LRRT policy is used for finding maximum LR that trains a model without divergence, and can be used to + configure the LR boundaries for Cyclic LR schedules. + + LRRT changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_range_test_min_lr (float or list): Initial learning rate which is the + lower boundary in the range test for each parameter group. + lr_range_test_step_size (int): Interval of training steps to increase learning rate. Default: 2000 + lr_range_test_step_rate (float): Scaling rate for range test. Default: 1.0 + lr_range_test_staircase (bool): Scale in staircase fashion, rather than continuous. Default: False. + last_batch_iteration (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_batch_iteration=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = LRRangeTest(optimizer) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + _A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay: + https://arxiv.org/abs/1803.09820 +""" + + def __init__(self, + optimizer: Optimizer, + lr_range_test_min_lr: float = 1e-3, + lr_range_test_step_size: int = 2000, + lr_range_test_step_rate: float = 1.0, + lr_range_test_staircase: bool = False, + last_batch_iteration: int = -1): + + self.optimizer = get_torch_optimizer(optimizer) + + if isinstance(lr_range_test_min_lr, list) or isinstance(lr_range_test_min_lr, tuple): + if len(lr_range_test_min_lr) != len(self.optimizer.param_groups): + raise ValueError("expected {} lr_range_test_min_lr, got {}".format(len(self.optimizer.param_groups), + len(lr_range_test_min_lr))) + self.min_lr = list(lr_range_test_min_lr) + else: + self.min_lr = [lr_range_test_min_lr] * len(self.optimizer.param_groups) + + self.step_size = lr_range_test_step_size + self.step_rate = lr_range_test_step_rate + self.last_batch_iteration = last_batch_iteration + self.staircase = lr_range_test_staircase + self.interval_fn = self._staircase_interval if lr_range_test_staircase else self._continuous_interval + + if last_batch_iteration == -1: + self._update_optimizer(self.min_lr) + + def _staircase_interval(self): + return math.floor(float(self.last_batch_iteration + 1) / self.step_size) + + def _continuous_interval(self): + return float(self.last_batch_iteration + 1) / self.step_size + + def _get_increase(self): + return (1 + self.step_rate * self.interval_fn()) + + def get_lr(self): + lr_increase = self._get_increase() + return [lr_range_test_min_lr * lr_increase for lr_range_test_min_lr in self.min_lr] + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def _update_optimizer(self, group_lrs): + for param_group, lr in zip(self.optimizer.param_groups, group_lrs): + param_group['lr'] = lr + + def step(self, batch_iteration=None): + if batch_iteration is None: + batch_iteration = self.last_batch_iteration + 1 + self.last_batch_iteration = batch_iteration + self._update_optimizer(self.get_lr()) + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + + +class OneCycle(object): + """Sets the learning rate of each parameter group according to + 1Cycle learning rate policy (1CLR). 1CLR is a variation of the + Cyclical Learning Rate (CLR) policy that involves one cycle followed by + decay. The policy simultaneously cycles the learning rate (and momentum) + between two boundaries with a constant frequency, as detailed in + the paper `A disciplined approach to neural network hyper-parameters`_. + + 1CLR policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This implementation was adapted from the github repo: `pytorch/pytorch`_ + + Args: + optimizer (Optimizer): Wrapped optimizer. + cycle_min_lr (float or list): Initial learning rate which is the + lower boundary in the cycle for each parameter group. + cycle_max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (cycle_max_lr - cycle_min_lr). + The lr at any cycle is the sum of cycle_min_lr + and some scaling of the amplitude; therefore + cycle_max_lr may not actually be reached depending on + scaling function. + decay_lr_rate(float): Decay rate for learning rate. Default: 0. + cycle_first_step_size (int): Number of training iterations in the + increasing half of a cycle. Default: 2000 + cycle_second_step_size (int): Number of training iterations in the + decreasing half of a cycle. If cycle_second_step_size is None, + it is set to cycle_first_step_size. Default: None + cycle_first_stair_count(int): Number of stairs in first half of cycle phase. This means + lr/mom are changed in staircase fashion. Default 0, means staircase disabled. + cycle_second_stair_count(int): Number of stairs in second half of cycle phase. This means + lr/mom are changed in staircase fashion. Default 0, means staircase disabled. + decay_step_size (int): Intervals for applying decay in decay phase. Default: 0, means no decay. + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'cycle_min_mom' and 'cycle_max_mom'. + Default: True + cycle_min_mom (float or list): Initial momentum which is the + lower boundary in the cycle for each parameter group. + Default: 0.8 + cycle_max_mom (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (cycle_max_mom - cycle_min_mom). + The momentum at any cycle is the difference of cycle_max_mom + and some scaling of the amplitude; therefore + cycle_min_mom may not actually be reached depending on + scaling function. Default: 0.9 + decay_mom_rate (float): Decay rate for momentum. Default: 0. + last_batch_iteration (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_batch_iteration=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = OneCycle(optimizer, 0.0001, 0.0010) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + + .. _A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay: https://arxiv.org/abs/1803.09820 + """ + + def __init__(self, + optimizer, + cycle_min_lr, + cycle_max_lr, + decay_lr_rate=0., + cycle_first_step_size=2000, + cycle_second_step_size=None, + cycle_first_stair_count=0, + cycle_second_stair_count=None, + decay_step_size=0, + cycle_momentum=True, + cycle_min_mom=0.8, + cycle_max_mom=0.9, + decay_mom_rate=0., + last_batch_iteration=-1): + + self.optimizer = get_torch_optimizer(optimizer) + + # Initialize cycle shape + self._initialize_cycle(cycle_first_step_size, cycle_second_step_size, cycle_first_stair_count, + cycle_second_stair_count, decay_step_size) + + # Initialize cycle lr + self._initialize_lr(self.optimizer, cycle_min_lr, cycle_max_lr, decay_lr_rate, last_batch_iteration) + + # Initialize cyclic momentum + self.cycle_momentum = cycle_momentum + if cycle_momentum: + self._initialize_momentum(self.optimizer, cycle_min_mom, cycle_max_mom, decay_mom_rate, + last_batch_iteration) + + # Initialize batch iteration tracker + self.last_batch_iteration = last_batch_iteration + + # Configure cycle shape + + def _initialize_cycle(self, cycle_first_step_size, cycle_second_step_size, cycle_first_stair_count, + cycle_second_stair_count, decay_step_size): + cycle_first_step_size = float(cycle_first_step_size) + cycle_second_step_size = float( + cycle_second_step_size) if cycle_second_step_size is not None else cycle_first_step_size + + self.total_size = cycle_first_step_size + cycle_second_step_size + self.step_ratio = cycle_first_step_size / self.total_size + self.first_stair_count = cycle_first_stair_count + self.second_stair_count = cycle_first_stair_count if cycle_second_stair_count is None else cycle_second_stair_count + self.decay_step_size = decay_step_size + + if math.isclose(self.decay_step_size, 0): + self.skip_lr_decay = True + self.skip_mom_decay = True + else: + self.skip_lr_decay = False + self.skip_mom_decay = False + + # Configure lr schedule + def _initialize_lr(self, optimizer, cycle_min_lr, cycle_max_lr, decay_lr_rate, last_batch_iteration): + self.min_lrs = [cycle_min_lr] * len(optimizer.param_groups) + if last_batch_iteration == -1: + for lr, group in zip(self.min_lrs, optimizer.param_groups): + group['lr'] = lr + + self.max_lrs = [cycle_max_lr] * len(optimizer.param_groups) + self.decay_lr_rate = decay_lr_rate + + if math.isclose(self.decay_lr_rate, 0): + self.skip_lr_decay = True + + # Configure momentum schedule + def _initialize_momentum(self, optimizer, cycle_min_mom, cycle_max_mom, decay_mom_rate, last_batch_iteration): + if 'betas' not in optimizer.defaults: + optimizer_name = type(optimizer).__name__ + print( + f"cycle_momentum is disabled because optimizer {optimizer_name} does not support momentum, no betas attribute in defaults" + ) + self.cycle_momentum = False + return + + self.decay_mom_rate = decay_mom_rate + self.min_moms = [(cycle_min_mom, 0.99)] * len(optimizer.param_groups) + self.max_moms = [(cycle_max_mom, 0.99)] * len(optimizer.param_groups) + + if last_batch_iteration == -1: + for momentum, group in zip(self.min_moms, optimizer.param_groups): + group['betas'] = momentum + + if math.isclose(self.decay_mom_rate, 0): + self.skip_mom_decay = True + + def _get_scale_factor(self): + batch_iteration = (self.last_batch_iteration + 1) + cycle = math.floor(1 + batch_iteration / self.total_size) + x = 1. + batch_iteration / self.total_size - cycle + if x <= self.step_ratio: + scale_factor = x / self.step_ratio + else: + scale_factor = (x - 1) / (self.step_ratio - 1) + + return scale_factor + + def _get_cycle_mom(self): + scale_factor = self._get_scale_factor() + momentums = [] + for base_betas, max_betas in zip(self.min_moms, self.max_moms): + cycle_min_mom = base_betas[0] + cycle_max_mom = max_betas[0] + base_height = (cycle_max_mom - cycle_min_mom) * scale_factor + momentum = cycle_max_mom - base_height + momentums.append((momentum, base_betas[1])) + return momentums + + def _get_cycle_lr(self): + scale_factor = self._get_scale_factor() + lrs = [] + for cycle_min_lr, cycle_max_lr in zip(self.min_lrs, self.max_lrs): + base_height = (cycle_max_lr - cycle_min_lr) * scale_factor + lr = cycle_min_lr + base_height + lrs.append(lr) + + return lrs + + def _get_decay_mom(self, decay_batch_iteration): + if self.skip_mom_decay: + return self.max_moms + + decay_interval = decay_batch_iteration / self.decay_step_size + mom_decay_factor = (1 + self.decay_mom_rate * decay_interval) + momentums = [(beta0 * mom_decay_factor, beta1) for beta0, beta1 in self.max_moms] + + return momentums + + def _get_decay_lr(self, decay_batch_iteration): + """Calculates the learning rate at batch index. This function is used + after the cycle completes and post cycle decaying of lr/mom is enabled. + This function treats `self.last_batch_iteration` as the last batch index. + """ + if self.skip_lr_decay: + return self.min_lrs + + decay_interval = decay_batch_iteration / self.decay_step_size + lr_decay_factor = (1 + self.decay_lr_rate * decay_interval) + lrs = [cycle_min_lr / lr_decay_factor for cycle_min_lr in self.min_lrs] + + return lrs + + def get_lr(self): + """Calculates the learning rate at batch index. This function treats + `self.last_batch_iteration` as the last batch index. + """ + if self.last_batch_iteration < self.total_size: + return self._get_cycle_lr() + return self._get_decay_lr(self.last_batch_iteration - self.total_size + 1) + + def get_mom(self): + """Calculates the momentum at batch index. This function treats + `self.last_batch_iteration` as the last batch index. + """ + if not self.cycle_momentum: + return None + + if self.last_batch_iteration < self.total_size: + return self._get_cycle_mom() + return self._get_decay_mom(self.last_batch_iteration - self.total_size + 1) + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def step(self, batch_iteration=None): + """ Updates the optimizer with the learning rate for the last batch index. + `self.last_batch_iteration` is treated as the last batch index. + + If self.cycle_momentum is true, also updates optimizer momentum. + """ + if batch_iteration is None: + batch_iteration = self.last_batch_iteration + 1 + + self.last_batch_iteration = batch_iteration + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + if self.cycle_momentum: + momentums = self.get_mom() + for param_group, momentum in zip(self.optimizer.param_groups, momentums): + param_group['betas'] = momentum + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + + +class WarmupLR(object): + """Increase the learning rate of each parameter group from min lr to max lr + over warmup_num_steps steps, and then fix at max lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + warmup_min_lr (float or list): minimum learning rate. Default: 0 + warmup_max_lr (float or list): maximum learning rate. Default: 0.001 + warmup_num_steps (int): number of steps to warm up from min_lr to max_lr. Default: 1000 + warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log + last_batch_iteration (int): The index of the last batch. Default: -1. + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = WarmupLR(optimizer) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + """ + + def __init__(self, + optimizer: Optimizer, + warmup_min_lr: float = 0.0, + warmup_max_lr: float = 0.001, + warmup_num_steps: int = 1000, + warmup_type: str = WARMUP_LOG_RATE, + last_batch_iteration: int = -1): + + self.optimizer = get_torch_optimizer(optimizer) + + self.min_lrs = self._format_param(self.optimizer, warmup_min_lr, "min_lr") + self.max_lrs = self._format_param(self.optimizer, warmup_max_lr, "max_lr") + self.delta_lrs = [big - small for big, small in zip(self.max_lrs, self.min_lrs)] + self.warmup_num_steps = max(2, warmup_num_steps) + # Currently only support linear and log function + if warmup_type not in {WARMUP_LOG_RATE, WARMUP_LINEAR_RATE}: + print(f"Using unknown warmup_type: {warmup_type}. The increasing function " + f"is set to default (log)") + warmup_type = WARMUP_LOG_RATE + self.warmup_type = warmup_type + self.inverse_log_warm_up = 1.0 / math.log(self.warmup_num_steps) + self.last_batch_iteration = last_batch_iteration + + def get_lr(self): + if self.last_batch_iteration < 0: + print("Attempting to get learning rate from scheduler before it has started") + return [0.0] + gamma = self._get_gamma() + return [min_lr + (delta_lr * gamma) for min_lr, delta_lr in zip(self.min_lrs, self.delta_lrs)] + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def step(self, last_batch_iteration=None): + if last_batch_iteration is None: + last_batch_iteration = self.last_batch_iteration + 1 + self.last_batch_iteration = last_batch_iteration + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + def _get_gamma(self): + if self.last_batch_iteration < self.warmup_num_steps: + if self.warmup_type == WARMUP_LOG_RATE: + return self.inverse_log_warm_up * math.log(self.last_batch_iteration + 1) + elif self.warmup_type == WARMUP_LINEAR_RATE: + return self.last_batch_iteration / self.warmup_num_steps + return 1.0 + + def _format_param(self, optimizer, param_value, param_name): + if isinstance(param_value, list) or isinstance(param_value, tuple): + if len(param_value) != len(optimizer.param_groups): + raise ValueError("expected {} value for {}, got {}".format(len(optimizer.param_groups), param_name, + FileNotFoundError(param_value))) + return list(param_value) + return [param_value] * len(optimizer.param_groups) + + + +class WarmupDecayLR(WarmupLR): + """Increase the learning rate of each parameter group from min lr to max lr + over warmup_num_steps steps, and then decay at linear rate over the remaining training steps. + + Args: + optimizer (Optimizer): Wrapped optimizer. + total_num_steps (int): total number of training steps + warmup_min_lr (float or list): minimum learning rate. Default: 0 + warmup_max_lr (float or list): maximum learning rate. Default: 0.001 + warmup_num_steps (int): number of steps to warm up from min_lr to max_lr. Default: 1000 + warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log + last_batch_iteration (int): The index of the last batch. Default: -1. + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = WarmupDecayLR(optimizer, 1000000) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + """ + + def __init__(self, + optimizer: Optimizer, + total_num_steps: int, + warmup_min_lr: float = 0.0, + warmup_max_lr: float = 0.001, + warmup_num_steps: int = 1000, + warmup_type: str = WARMUP_LOG_RATE, + last_batch_iteration: int = -1): + + self.total_num_steps = total_num_steps + super(WarmupDecayLR, self).__init__(optimizer, warmup_min_lr, warmup_max_lr, warmup_num_steps, warmup_type, + last_batch_iteration) + if self.total_num_steps < self.warmup_num_steps: + print('total_num_steps {} is less than warmup_num_steps {}'.format( + total_num_steps, warmup_num_steps)) + + def _get_gamma(self): + if self.last_batch_iteration < self.warmup_num_steps: + if self.warmup_type == WARMUP_LOG_RATE: + return self.inverse_log_warm_up * math.log(self.last_batch_iteration + 1) + elif self.warmup_type == WARMUP_LINEAR_RATE: + return self.last_batch_iteration / self.warmup_num_steps + return max( + 0.0, + float(self.total_num_steps - self.last_batch_iteration) / + float(max(1.0, self.total_num_steps - self.warmup_num_steps))) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/attn_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/attn_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..4308af93293c87d101519ef6b1e07663d91cd2ef --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/attn_layers.py @@ -0,0 +1,377 @@ +import torch +import torch.nn as nn +from typing import Tuple, Union, Optional + +try: + import flash_attn + if hasattr(flash_attn, '__version__') and int(flash_attn.__version__[0]) == 2: + from flash_attn.flash_attn_interface import flash_attn_kvpacked_func + from flash_attn.modules.mha import FlashSelfAttention, FlashCrossAttention + else: + from flash_attn.flash_attn_interface import flash_attn_unpadded_kvpacked_func + from flash_attn.modules.mha import FlashSelfAttention, FlashCrossAttention +except Exception as e: + print(f'flash_attn import failed: {e}') + + +def reshape_for_broadcast(freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], x: torch.Tensor, head_first=False): + """ + Reshape frequency tensor for broadcasting it with another tensor. + + This function reshapes the frequency tensor to have the same shape as the target tensor 'x' + for the purpose of broadcasting the frequency tensor during element-wise operations. + + Args: + freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Frequency tensor to be reshaped. + x (torch.Tensor): Target tensor for broadcasting compatibility. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + torch.Tensor: Reshaped frequency tensor. + + Raises: + AssertionError: If the frequency tensor doesn't match the expected shape. + AssertionError: If the target tensor 'x' doesn't have the expected number of dimensions. + """ + ndim = x.ndim + assert 0 <= 1 < ndim + + if isinstance(freqs_cis, tuple): + # freqs_cis: (cos, sin) in real space + if head_first: + assert freqs_cis[0].shape == (x.shape[-2], x.shape[-1]), f'freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}' + shape = [d if i == ndim - 2 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + else: + assert freqs_cis[0].shape == (x.shape[1], x.shape[-1]), f'freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}' + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis[0].view(*shape), freqs_cis[1].view(*shape) + else: + # freqs_cis: values in complex space + if head_first: + assert freqs_cis.shape == (x.shape[-2], x.shape[-1]), f'freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}' + shape = [d if i == ndim - 2 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + else: + assert freqs_cis.shape == (x.shape[1], x.shape[-1]), f'freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}' + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def rotate_half(x): + x_real, x_imag = x.float().reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + return torch.stack([-x_imag, x_real], dim=-1).flatten(3) + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: Optional[torch.Tensor], + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + head_first: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. + + This function applies rotary embeddings to the given query 'xq' and key 'xk' tensors using the provided + frequency tensor 'freqs_cis'. The input tensors are reshaped as complex numbers, and the frequency tensor + is reshaped for broadcasting compatibility. The resulting tensors contain rotary embeddings and are + returned as real tensors. + + Args: + xq (torch.Tensor): Query tensor to apply rotary embeddings. [B, S, H, D] + xk (torch.Tensor): Key tensor to apply rotary embeddings. [B, S, H, D] + freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Precomputed frequency tensor for complex exponentials. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + + """ + xk_out = None + if isinstance(freqs_cis, tuple): + cos, sin = reshape_for_broadcast(freqs_cis, xq, head_first) # [S, D] + cos, sin = cos.to(xq.device), sin.to(xq.device) + xq_out = (xq.float() * cos + rotate_half(xq.float()) * sin).type_as(xq) + if xk is not None: + xk_out = (xk.float() * cos + rotate_half(xk.float()) * sin).type_as(xk) + else: + xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) # [B, S, H, D//2] + freqs_cis = reshape_for_broadcast(freqs_cis, xq_, head_first).to(xq.device) # [S, D//2] --> [1, S, 1, D//2] + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3).type_as(xq) + if xk is not None: + xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) # [B, S, H, D//2] + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3).type_as(xk) + + return xq_out, xk_out + + +class FlashSelfMHAModified(nn.Module): + """ + Use QK Normalization. + """ + def __init__(self, + dim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.dim = dim + self.num_heads = num_heads + assert self.dim % num_heads == 0, "self.kdim must be divisible by num_heads" + self.head_dim = self.dim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + + self.Wqkv = nn.Linear(dim, 3 * dim, bias=qkv_bias, **factory_kwargs) + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.inner_attn = FlashSelfAttention(attention_dropout=attn_drop) + self.out_proj = nn.Linear(dim, dim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // 2), RoPE for image + """ + b, s, d = x.shape + + qkv = self.Wqkv(x) + qkv = qkv.view(b, s, 3, self.num_heads, self.head_dim) # [b, s, 3, h, d] + q, k, v = qkv.unbind(dim=2) # [b, s, h, d] + q = self.q_norm(q).half() # [b, s, h, d] + k = self.k_norm(k).half() + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, kk = apply_rotary_emb(q, k, freqs_cis_img) + assert qq.shape == q.shape and kk.shape == k.shape, f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}' + q, k = qq, kk + + qkv = torch.stack([q, k, v], dim=2) # [b, s, 3, h, d] + context = self.inner_attn(qkv) + out = self.out_proj(context.view(b, s, d)) + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class FlashCrossMHAModified(nn.Module): + """ + Use QK Normalization. + """ + def __init__(self, + qdim, + kdim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.qdim = qdim + self.kdim = kdim + self.num_heads = num_heads + assert self.qdim % num_heads == 0, "self.qdim must be divisible by num_heads" + self.head_dim = self.qdim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + + self.scale = self.head_dim ** -0.5 + + self.q_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.kv_proj = nn.Linear(kdim, 2 * qdim, bias=qkv_bias, **factory_kwargs) + + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + + self.inner_attn = FlashCrossAttention(attention_dropout=attn_drop) + self.out_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, y, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen1, hidden_dim) (where hidden_dim = num_heads * head_dim) + y: torch.Tensor + (batch, seqlen2, hidden_dim2) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // num_heads), RoPE for image + """ + b, s1, _ = x.shape # [b, s1, D] + _, s2, _ = y.shape # [b, s2, 1024] + + q = self.q_proj(x).view(b, s1, self.num_heads, self.head_dim) # [b, s1, h, d] + kv = self.kv_proj(y).view(b, s2, 2, self.num_heads, self.head_dim) # [b, s2, 2, h, d] + k, v = kv.unbind(dim=2) # [b, s2, h, d] + q = self.q_norm(q).half() # [b, s1, h, d] + k = self.k_norm(k).half() # [b, s2, h, d] + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, _ = apply_rotary_emb(q, None, freqs_cis_img) + assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}' + q = qq # [b, s1, h, d] + kv = torch.stack([k, v], dim=2) # [b, s1, 2, h, d] + context = self.inner_attn(q, kv) # [b, s1, h, d] + context = context.view(b, s1, -1) # [b, s1, D] + + out = self.out_proj(context) + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class CrossAttention(nn.Module): + """ + Use QK Normalization. + """ + def __init__(self, + qdim, + kdim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.qdim = qdim + self.kdim = kdim + self.num_heads = num_heads + assert self.qdim % num_heads == 0, "self.qdim must be divisible by num_heads" + self.head_dim = self.qdim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + self.scale = self.head_dim ** -0.5 + + self.q_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.kv_proj = nn.Linear(kdim, 2 * qdim, bias=qkv_bias, **factory_kwargs) + + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.out_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, y, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen1, hidden_dim) (where hidden_dim = num heads * head dim) + y: torch.Tensor + (batch, seqlen2, hidden_dim2) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // 2), RoPE for image + """ + b, s1, c = x.shape # [b, s1, D] + _, s2, c = y.shape # [b, s2, 1024] + + q = self.q_proj(x).view(b, s1, self.num_heads, self.head_dim) # [b, s1, h, d] + kv = self.kv_proj(y).view(b, s2, 2, self.num_heads, self.head_dim) # [b, s2, 2, h, d] + k, v = kv.unbind(dim=2) # [b, s, h, d] + q = self.q_norm(q) + k = self.k_norm(k) + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, _ = apply_rotary_emb(q, None, freqs_cis_img) + assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}' + q = qq + + q = q * self.scale + q = q.transpose(-2, -3).contiguous() # q -> B, L1, H, C - B, H, L1, C + k = k.permute(0, 2, 3, 1).contiguous() # k -> B, L2, H, C - B, H, C, L2 + attn = q @ k # attn -> B, H, L1, L2 + attn = attn.softmax(dim=-1) # attn -> B, H, L1, L2 + attn = self.attn_drop(attn) + x = attn @ v.transpose(-2, -3) # v -> B, L2, H, C - B, H, L2, C x-> B, H, L1, C + context = x.transpose(1, 2) # context -> B, H, L1, C - B, L1, H, C + + context = context.contiguous().view(b, s1, -1) + + out = self.out_proj(context) # context.reshape - B, L1, -1 + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class Attention(nn.Module): + """ + We rename some layer names to align with flash attention + """ + def __init__(self, dim, num_heads, qkv_bias=True, qk_norm=False, attn_drop=0., proj_drop=0., + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + assert self.dim % num_heads == 0, 'dim should be divisible by num_heads' + self.head_dim = self.dim // num_heads + # This assertion is aligned with flash attention + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + self.scale = self.head_dim ** -0.5 + + # qkv --> Wqkv + self.Wqkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.out_proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, freqs_cis_img=None): + B, N, C = x.shape + qkv = self.Wqkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) # [3, b, h, s, d] + q, k, v = qkv.unbind(0) # [b, h, s, d] + q = self.q_norm(q) # [b, h, s, d] + k = self.k_norm(k) # [b, h, s, d] + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, kk = apply_rotary_emb(q, k, freqs_cis_img, head_first=True) + assert qq.shape == q.shape and kk.shape == k.shape, \ + f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}' + q, k = qq, kk + + q = q * self.scale + attn = q @ k.transpose(-2, -1) # [b, h, s, d] @ [b, h, d, s] + attn = attn.softmax(dim=-1) # [b, h, s, s] + attn = self.attn_drop(attn) + x = attn @ v # [b, h, s, d] + + x = x.transpose(1, 2).reshape(B, N, C) # [b, s, h, d] + x = self.out_proj(x) + x = self.proj_drop(x) + + out_tuple = (x,) + + return out_tuple diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c051f2b66ca7df022e0be2c66552d8b02d3e521f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/controlnet.py @@ -0,0 +1,273 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from timm.models.vision_transformer import Mlp + +from .attn_layers import Attention, FlashCrossMHAModified, FlashSelfMHAModified, CrossAttention +from .embedders import TimestepEmbedder, PatchEmbed, timestep_embedding +from .norm_layers import RMSNorm +from .poolers import AttentionPool + +from .models import FP32_Layernorm, FP32_SiLU, HunYuanDiTBlock + +def zero_module(module): + for p in module.parameters(): + nn.init.zeros_(p) + return module + +class HunYuanControlNet(ModelMixin, ConfigMixin): + """ + HunYuanDiT: Diffusion model with a Transformer backbone. + + Inherit ModelMixin and ConfigMixin to be compatible with the sampler StableDiffusionPipeline of diffusers. + + Parameters + ---------- + args: argparse.Namespace + The arguments parsed by argparse. + input_size: tuple + The size of the input image. + patch_size: int + The size of the patch. + in_channels: int + The number of input channels. + hidden_size: int + The hidden size of the transformer backbone. + depth: int + The number of transformer blocks. + num_heads: int + The number of attention heads. + mlp_ratio: float + The ratio of the hidden size of the MLP in the transformer block. + log_fn: callable + The logging function. + """ + @register_to_config + def __init__( + self, args, + input_size=(32, 32), + patch_size=2, + in_channels=4, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + log_fn=print, + ): + super().__init__() + self.args = args + self.log_fn = log_fn + self.depth = depth + self.learn_sigma = args.learn_sigma + self.in_channels = in_channels + self.patch_size = patch_size + self.num_heads = num_heads + self.hidden_size = hidden_size + self.text_states_dim = args.text_states_dim + self.text_states_dim_t5 = args.text_states_dim_t5 + self.text_len = args.text_len + self.text_len_t5 = args.text_len_t5 + self.norm = args.norm + + use_flash_attn = args.infer_mode == 'fa' or args.use_flash_attn + if use_flash_attn: + log_fn(f" Enable Flash Attention.") + qk_norm = args.qk_norm # See http://arxiv.org/abs/2302.05442 for details. + + self.mlp_t5 = nn.Sequential( + nn.Linear(self.text_states_dim_t5, self.text_states_dim_t5 * 4, bias=True), + FP32_SiLU(), + nn.Linear(self.text_states_dim_t5 * 4, self.text_states_dim, bias=True), + ) + # learnable replace + self.text_embedding_padding = nn.Parameter( + torch.randn(self.text_len + self.text_len_t5, self.text_states_dim, dtype=torch.float32)) + + # Attention pooling + self.pooler = AttentionPool(self.text_len_t5, self.text_states_dim_t5, num_heads=8, output_dim=1024) + + # Here we use a default learned embedder layer for future extension. + self.style_embedder = nn.Embedding(1, hidden_size) + + # Image size and crop size conditions + self.extra_in_dim = 256 * 6 + hidden_size + + # Text embedding for `add` + self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size) + self.t_embedder = TimestepEmbedder(hidden_size) + self.extra_in_dim += 1024 + self.extra_embedder = nn.Sequential( + nn.Linear(self.extra_in_dim, hidden_size * 4), + FP32_SiLU(), + nn.Linear(hidden_size * 4, hidden_size, bias=True), + ) + + # Image embedding + num_patches = self.x_embedder.num_patches + log_fn(f" Number of tokens: {num_patches}") + + # HUnYuanDiT Blocks + self.blocks = nn.ModuleList([ + HunYuanDiTBlock(hidden_size=hidden_size, + c_emb_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + text_states_dim=self.text_states_dim, + use_flash_attn=use_flash_attn, + qk_norm=qk_norm, + norm_type=self.norm, + skip=False, + ) + for _ in range(19) + ]) + + # Input zero linear for the first block + self.before_proj = zero_module(nn.Linear(self.hidden_size, self.hidden_size)) + + # Output zero linear for the every block + self.after_proj_list = nn.ModuleList( + [zero_module(nn.Linear(self.hidden_size, self.hidden_size)) for _ in range(len(self.blocks))] + ) + + self.fix_weight_modules = ['mlp_t5', 'text_embedding_padding', 'pooler', 'style_embedder', 'x_embedder', 't_embedder', 'extra_embedder'] + + + def from_dit(self, dit): + """ + Load the parameters from a pre-trained HunYuanDiT model. + + Parameters + ---------- + dit: HunYuanDiT + The pre-trained HunYuanDiT model. + """ + + + self.mlp_t5.load_state_dict(dit.mlp_t5.state_dict()) + + self.text_embedding_padding.data = dit.text_embedding_padding.data + self.pooler.load_state_dict(dit.pooler.state_dict()) + self.style_embedder.load_state_dict(dit.style_embedder.state_dict()) + self.x_embedder.load_state_dict(dit.x_embedder.state_dict()) + self.t_embedder.load_state_dict(dit.t_embedder.state_dict()) + self.extra_embedder.load_state_dict(dit.extra_embedder.state_dict()) + + for i, block in enumerate(self.blocks): + block.load_state_dict(dit.blocks[i].state_dict()) + + def set_trainable(self): + + self.mlp_t5.requires_grad_(False) + self.text_embedding_padding.requires_grad_(False) + self.pooler.requires_grad_(False) + self.style_embedder.requires_grad_(False) + self.x_embedder.requires_grad_(False) + self.t_embedder.requires_grad_(False) + self.extra_embedder.requires_grad_(False) + + self.blocks.requires_grad_(True) + self.before_proj.requires_grad_(True) + self.after_proj_list.requires_grad_(True) + + self.blocks.train() + self.before_proj.train() + self.after_proj_list.train() + + + + def forward(self, + x, + t, + condition, + encoder_hidden_states=None, + text_embedding_mask=None, + encoder_hidden_states_t5=None, + text_embedding_mask_t5=None, + image_meta_size=None, + style=None, + cos_cis_img=None, + sin_cis_img=None, + return_dict=True, + ): + """ + Forward pass of the encoder. + + Parameters + ---------- + x: torch.Tensor + (B, D, H, W) + t: torch.Tensor + (B) + encoder_hidden_states: torch.Tensor + CLIP text embedding, (B, L_clip, D) + text_embedding_mask: torch.Tensor + CLIP text embedding mask, (B, L_clip) + encoder_hidden_states_t5: torch.Tensor + T5 text embedding, (B, L_t5, D) + text_embedding_mask_t5: torch.Tensor + T5 text embedding mask, (B, L_t5) + image_meta_size: torch.Tensor + (B, 6) + style: torch.Tensor + (B) + cos_cis_img: torch.Tensor + sin_cis_img: torch.Tensor + return_dict: bool + Whether to return a dictionary. + """ + text_states = encoder_hidden_states # 2,77,1024 + text_states_t5 = encoder_hidden_states_t5 # 2,256,2048 + text_states_mask = text_embedding_mask.bool() # 2,77 + text_states_t5_mask = text_embedding_mask_t5.bool() # 2,256 + b_t5, l_t5, c_t5 = text_states_t5.shape + text_states_t5 = self.mlp_t5(text_states_t5.view(-1, c_t5)) + text_states = torch.cat([text_states, text_states_t5.view(b_t5, l_t5, -1)], dim=1) # 2,205，1024 + clip_t5_mask = torch.cat([text_states_mask, text_states_t5_mask], dim=-1) + + clip_t5_mask = clip_t5_mask + text_states = torch.where(clip_t5_mask.unsqueeze(2), text_states, self.text_embedding_padding.to(text_states)) + + _, _, oh, ow = x.shape + th, tw = oh // self.patch_size, ow // self.patch_size + + # ========================= Build time and image embedding ========================= + t = self.t_embedder(t) + x = self.x_embedder(x) + + # Get image RoPE embedding according to `reso`lution. + freqs_cis_img = (cos_cis_img, sin_cis_img) + + # ========================= Concatenate all extra vectors ========================= + # Build text tokens with pooling + extra_vec = self.pooler(encoder_hidden_states_t5) + + # Build image meta size tokens + image_meta_size = timestep_embedding(image_meta_size.view(-1), 256) # [B * 6, 256] + if self.args.use_fp16: + image_meta_size = image_meta_size.half() + image_meta_size = image_meta_size.view(-1, 6 * 256) + extra_vec = torch.cat([extra_vec, image_meta_size], dim=1) # [B, D + 6 * 256] + + # Build style tokens + style_embedding = self.style_embedder(style) + extra_vec = torch.cat([extra_vec, style_embedding], dim=1) + + # Concatenate all extra vectors + c = t + self.extra_embedder(extra_vec) # [B, D] + + # ========================= Deal with Condition ========================= + condition = self.x_embedder(condition) + + # ========================= Forward pass through HunYuanDiT blocks ========================= + controls = [] + x = x + self.before_proj(condition) # add condition + for layer, block in enumerate(self.blocks): + x = block(x, c, text_states, freqs_cis_img) + controls.append(self.after_proj_list[layer](x)) # zero linear for output + + + if return_dict: + return {'controls': controls} + return controls \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/ema.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..acc31ee296f7f30bb37f101fdd911539df54e20a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/ema.py @@ -0,0 +1,127 @@ +from collections import OrderedDict +from copy import deepcopy + +import torch +from deepspeed.utils import instrument_w_nvtx +from pathlib import Path + +def requires_grad(model, flag=True): + """ + Set requires_grad flag for all parameters in a model. + """ + for p in model.parameters(): + p.requires_grad = flag + + +class EMA(object): + def __init__(self, args, model, device, logger): + if args.ema_dtype == 'fp32': + self.warmup = args.ema_warmup + self.update_after_step = 0 + self.max_value = args.ema_decay if args.ema_decay is not None else 0.9999 + self.inv_gamma = 1.0 + self.power = args.ema_warmup_power if args.ema_warmup_power is not None else 2 / 3 + self.min_value = 0.0 + else: + self.warmup = args.ema_warmup + self.update_after_step = 0 + self.max_value = args.ema_decay if args.ema_decay is not None else 0.992 + self.inv_gamma = 1.0 + self.power = args.ema_warmup_power if args.ema_warmup_power is not None else 0.446249 + # 0.446249 == math.log(1 - 0.992) / math.log(50000) + self.min_value = 0.0 + + self.ema_reset_decay = args.ema_reset_decay + self.decay_steps = 0 + + if args.ema_dtype == 'none': + ema_dtype = 'fp16' if args.use_fp16 else 'fp32' + else: + ema_dtype = args.ema_dtype + + # 由于module.half()和module.float()会发生inplace类型修改，因此需要先copy后修改类型 + self.ema_model = deepcopy(model) + if ema_dtype == 'fp16': + self.ema_model = self.ema_model.half().to(device) + elif ema_dtype == 'fp32': + self.ema_model = self.ema_model.float().to(device) + else: + raise ValueError(f"Unknown EMA dtype {ema_dtype}.") + + requires_grad(self.ema_model, False) + + logger.info(f" Using EMA with date type {args.ema_dtype} " + f"(decay={args.ema_decay}, warmup={args.ema_warmup}, warmup_power={args.ema_warmup_power}, " + f"reset_decay={args.ema_reset_decay}).") + + def get_decay(self): + """ + @crowsonkb's notes on EMA Warmup: + If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan + to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), + gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 + at 215.4k steps). + + @jarvizhang's notes on EMA max_value when enabling FP16: + If using FP16 for EMA, max_value=0.995 is better (Don't larger than 0.999, unless you know + what you are doing). This is because FP16 has less precision than FP32, so the EMA value can + be pushed out of the range of FP16. + + gamma=1, power=0.446249 are good values for models (reaches decay factor 0.99 at 30K steps, + 0.992 at 50K steps). + """ + if self.warmup: + step = max(0, self.decay_steps - self.update_after_step - 1) + value = 1 - (1 + step / self.inv_gamma) ** -self.power + + if step <= 0: + return 0.0 + + return max(self.min_value, min(value, self.max_value)) + else: + return self.max_value + + @torch.no_grad() + @instrument_w_nvtx + def update(self, model, step, decay=None): + """ + Step the EMA model towards the current model. + + Parameters + ---------- + model: nn.Module + The current model + step: int + The current training step. This is used to determine the decay factor. If you want to control + the decay, you can pass in a custom step instead. + For example, if you want to restart the EMA decay, you can pass in step=0 at start and increase + step by step. + decay: float + The decay factor. If None, will be determined by the current step. + """ + if decay is None: + if self.ema_reset_decay: + self.decay_steps += 1 + else: + self.decay_steps = step + decay = self.get_decay() + + ema_params = OrderedDict(self.ema_model.named_parameters()) + model_params = OrderedDict(model.named_parameters()) + for name, param in model_params.items(): + # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed + ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay) + + return None + + def state_dict(self, *args, **kwargs): + return self.ema_model.state_dict(*args, **kwargs) + + def load_state_dict(self, *args, **kwargs): + return self.ema_model.load_state_dict(*args, **kwargs) + + def train(self): + self.ema_model.train() + + def eval(self): + self.ema_model.eval() diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/embedders.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/embedders.py new file mode 100644 index 0000000000000000000000000000000000000000..9fe08cba22eef41ca9fd9f70fe6f062a4dd606c8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/embedders.py @@ -0,0 +1,111 @@ +import math +import torch +import torch.nn as nn +from einops import repeat + +from timm.models.layers import to_2tuple + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + + Image to Patch Embedding using Conv2d + + A convolution based approach to patchifying a 2D image w/ embedding projection. + + Based on the impl in https://github.com/google-research/vision_transformer + + Hacked together by / Copyright 2020 Ross Wightman + + Remove the _assert function in forward function to be compatible with multi-resolution images. + """ + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + bias=True, + ): + super().__init__() + if isinstance(img_size, int): + img_size = to_2tuple(img_size) + elif isinstance(img_size, (tuple, list)) and len(img_size) == 2: + img_size = tuple(img_size) + else: + raise ValueError(f"img_size must be int or tuple/list of length 2. Got {img_size}") + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def update_image_size(self, img_size): + self.img_size = img_size + self.grid_size = (img_size[0] // self.patch_size[0], img_size[1] // self.patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + + def forward(self, x): + # B, C, H, W = x.shape + # _assert(H == self.img_size[0], f"Input image height ({H}) doesn't match model ({self.img_size[0]}).") + # _assert(W == self.img_size[1], f"Input image width ({W}) doesn't match model ({self.img_size[1]}).") + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +def timestep_embedding(t, dim, max_period=10000, repeat_only=False): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + if not repeat_only: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half + ).to(device=t.device) # size: [dim/2], 一个指数衰减的曲线 + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1 + ) + else: + embedding = repeat(t, "b -> b d", d=dim) + return embedding + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + def __init__(self, hidden_size, frequency_embedding_size=256, out_size=None): + super().__init__() + if out_size is None: + out_size = hidden_size + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, out_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + def forward(self, t): + t_freq = timestep_embedding(t, self.frequency_embedding_size).type(self.mlp[0].weight.dtype) + t_emb = self.mlp(t_freq) + return t_emb diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/fp16_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/fp16_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..6812811045eccbacde3a60fc3d230cd3fb9da0ad --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/fp16_layers.py @@ -0,0 +1,86 @@ +import torch +from torch.autograd import Variable +from torch.nn.parameter import Parameter + + +_FLOAT_TYPES = (torch.FloatTensor, torch.cuda.FloatTensor) +_HALF_TYPES = (torch.HalfTensor, torch.cuda.HalfTensor) + + +def conversion_helper(val, conversion): + """Apply conversion to val. Recursively apply conversion if `val` + #is a nested tuple/list structure.""" + if isinstance(val, dict): + res_dict = {} + for k, v in val.items(): + if k!= 'cos_cis_img' and k != 'sin_cis_img': + res_dict[k] = conversion_helper(v, conversion) + else: + res_dict[k] = v + return res_dict + if not isinstance(val, (tuple, list)): + return conversion(val) + rtn = [conversion_helper(v, conversion) for v in val] + if isinstance(val, tuple): + rtn = tuple(rtn) + return rtn + + +def fp32_to_float16(val, float16_convertor): + """Convert fp32 `val` to fp16/bf16""" + def half_conversion(val): + val_typecheck = val + if isinstance(val_typecheck, (Parameter, Variable)): + val_typecheck = val.data + if isinstance(val_typecheck, _FLOAT_TYPES): + val = float16_convertor(val) + return val + return conversion_helper(val, half_conversion) + + +def float16_to_fp32(val): + """Convert fp16/bf16 `val` to fp32""" + def float_conversion(val): + val_typecheck = val + if isinstance(val_typecheck, (Parameter, Variable)): + val_typecheck = val.data + if isinstance(val_typecheck, (_HALF_TYPES,)): + val = val.float() + return val + return conversion_helper(val, float_conversion) + + +class Float16Module(torch.nn.Module): + + def __init__(self, module, args): + super(Float16Module, self).__init__() + + self.add_module('module', module.half()) + + def float16_convertor(val): + return val.half() + + self.float16_convertor = float16_convertor + + self.config = self.module.config + self.dtype = torch.float16 + + def forward(self, *inputs, **kwargs): + inputs = fp32_to_float16(inputs, self.float16_convertor) + kwargs = fp32_to_float16(kwargs, self.float16_convertor) + outputs = self.module(*inputs, **kwargs) + outputs = float16_to_fp32(outputs) + return outputs + + def state_dict(self, destination=None, prefix='', keep_vars=False): + return self.module.state_dict(destination, prefix, keep_vars) + + + def state_dict_for_save_checkpoint(self, destination=None, prefix='', + keep_vars=False): + return self.module.state_dict_for_save_checkpoint(destination, prefix, + keep_vars) + + def load_state_dict(self, state_dict, strict=True): + self.module.load_state_dict(state_dict, strict=strict) + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models.py new file mode 100644 index 0000000000000000000000000000000000000000..af991d347175dc745638340e6b717997917c6115 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models.py @@ -0,0 +1,510 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from timm.models.vision_transformer import Mlp + +from .attn_layers import Attention, FlashCrossMHAModified, FlashSelfMHAModified, CrossAttention +from .embedders import TimestepEmbedder, PatchEmbed, timestep_embedding +from .norm_layers import RMSNorm +from .poolers import AttentionPool + +from transformers.integrations import PeftAdapterMixin +from typing import Any, Optional, Union +from tqdm import tqdm +from peft.utils import ( + ModulesToSaveWrapper, + _get_submodules, +) + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class FP32_Layernorm(nn.LayerNorm): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + origin_dtype = inputs.dtype + return F.layer_norm(inputs.float(), self.normalized_shape, self.weight.float(), self.bias.float(), + self.eps).to(origin_dtype) + + +class FP32_SiLU(nn.SiLU): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.silu(inputs.float(), inplace=False).to(inputs.dtype) + + +class HunYuanDiTBlock(nn.Module): + """ + A HunYuanDiT block with `add` conditioning. + """ + def __init__(self, + hidden_size, + c_emb_size, + num_heads, + mlp_ratio=4.0, + text_states_dim=1024, + use_flash_attn=False, + qk_norm=False, + norm_type="layer", + skip=False, + ): + super().__init__() + self.use_flash_attn = use_flash_attn + use_ele_affine = True + + if norm_type == "layer": + norm_layer = FP32_Layernorm + elif norm_type == "rms": + norm_layer = RMSNorm + else: + raise ValueError(f"Unknown norm_type: {norm_type}") + + # ========================= Self-Attention ========================= + self.norm1 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + if use_flash_attn: + self.attn1 = FlashSelfMHAModified(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + else: + self.attn1 = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + + # ========================= FFN ========================= + self.norm2 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + approx_gelu = lambda: nn.GELU(approximate="tanh") + self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0) + + # ========================= Add ========================= + # Simply use add like SDXL. + self.default_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, hidden_size, bias=True) + ) + + # ========================= Cross-Attention ========================= + if use_flash_attn: + self.attn2 = FlashCrossMHAModified(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + else: + self.attn2 = CrossAttention(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + self.norm3 = norm_layer(hidden_size, elementwise_affine=True, eps=1e-6) + + # ========================= Skip Connection ========================= + if skip: + self.skip_norm = norm_layer(2 * hidden_size, elementwise_affine=True, eps=1e-6) + self.skip_linear = nn.Linear(2 * hidden_size, hidden_size) + else: + self.skip_linear = None + + def forward(self, x, c=None, text_states=None, freq_cis_img=None, skip=None): + # Long Skip Connection + if self.skip_linear is not None: + cat = torch.cat([x, skip], dim=-1) + cat = self.skip_norm(cat) + x = self.skip_linear(cat) + + # Self-Attention + shift_msa = self.default_modulation(c).unsqueeze(dim=1) + attn_inputs = ( + self.norm1(x) + shift_msa, freq_cis_img, + ) + x = x + self.attn1(*attn_inputs)[0] + + # Cross-Attention + cross_inputs = ( + self.norm3(x), text_states, freq_cis_img + ) + x = x + self.attn2(*cross_inputs)[0] + + # FFN Layer + mlp_inputs = self.norm2(x) + x = x + self.mlp(mlp_inputs) + + return x + + +class FinalLayer(nn.Module): + """ + The final layer of HunYuanDiT. + """ + def __init__(self, final_hidden_size, c_emb_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(final_hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(final_hidden_size, patch_size * patch_size * out_channels, bias=True) + self.adaLN_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, 2 * final_hidden_size, bias=True) + ) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class HunYuanDiT(ModelMixin, ConfigMixin, PeftAdapterMixin): + """ + HunYuanDiT: Diffusion model with a Transformer backbone. + + Inherit ModelMixin and ConfigMixin to be compatible with the sampler StableDiffusionPipeline of diffusers. + + Inherit PeftAdapterMixin to be compatible with the PEFT training pipeline. + + Parameters + ---------- + args: argparse.Namespace + The arguments parsed by argparse. + input_size: tuple + The size of the input image. + patch_size: int + The size of the patch. + in_channels: int + The number of input channels. + hidden_size: int + The hidden size of the transformer backbone. + depth: int + The number of transformer blocks. + num_heads: int + The number of attention heads. + mlp_ratio: float + The ratio of the hidden size of the MLP in the transformer block. + log_fn: callable + The logging function. + """ + @register_to_config + def __init__( + self, args, + input_size=(32, 32), + patch_size=2, + in_channels=4, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + log_fn=print, + ): + super().__init__() + self.args = args + self.log_fn = log_fn + self.depth = depth + self.learn_sigma = args.learn_sigma + self.in_channels = in_channels + self.out_channels = in_channels * 2 if args.learn_sigma else in_channels + self.patch_size = patch_size + self.num_heads = num_heads + self.hidden_size = hidden_size + self.text_states_dim = args.text_states_dim + self.text_states_dim_t5 = args.text_states_dim_t5 + self.text_len = args.text_len + self.text_len_t5 = args.text_len_t5 + self.norm = args.norm + + use_flash_attn = args.infer_mode == 'fa' or args.use_flash_attn + if use_flash_attn: + log_fn(f" Enable Flash Attention.") + qk_norm = args.qk_norm # See http://arxiv.org/abs/2302.05442 for details. + + self.mlp_t5 = nn.Sequential( + nn.Linear(self.text_states_dim_t5, self.text_states_dim_t5 * 4, bias=True), + FP32_SiLU(), + nn.Linear(self.text_states_dim_t5 * 4, self.text_states_dim, bias=True), + ) + # learnable replace + self.text_embedding_padding = nn.Parameter( + torch.randn(self.text_len + self.text_len_t5, self.text_states_dim, dtype=torch.float32)) + + # Attention pooling + self.pooler = AttentionPool(self.text_len_t5, self.text_states_dim_t5, num_heads=8, output_dim=1024) + + # Here we use a default learned embedder layer for future extension. + self.style_embedder = nn.Embedding(1, hidden_size) + + # Image size and crop size conditions + self.extra_in_dim = 256 * 6 + hidden_size + + # Text embedding for `add` + self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size) + self.t_embedder = TimestepEmbedder(hidden_size) + self.extra_in_dim += 1024 + self.extra_embedder = nn.Sequential( + nn.Linear(self.extra_in_dim, hidden_size * 4), + FP32_SiLU(), + nn.Linear(hidden_size * 4, hidden_size, bias=True), + ) + + # Image embedding + num_patches = self.x_embedder.num_patches + log_fn(f" Number of tokens: {num_patches}") + + # HUnYuanDiT Blocks + self.blocks = nn.ModuleList([ + HunYuanDiTBlock(hidden_size=hidden_size, + c_emb_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + text_states_dim=self.text_states_dim, + use_flash_attn=use_flash_attn, + qk_norm=qk_norm, + norm_type=self.norm, + skip=layer > depth // 2, + ) + for layer in range(depth) + ]) + + self.final_layer = FinalLayer(hidden_size, hidden_size, patch_size, self.out_channels) + self.unpatchify_channels = self.out_channels + + self.initialize_weights() + + def forward(self, + x, + t, + encoder_hidden_states=None, + text_embedding_mask=None, + encoder_hidden_states_t5=None, + text_embedding_mask_t5=None, + image_meta_size=None, + style=None, + cos_cis_img=None, + sin_cis_img=None, + return_dict=True, + controls=None, + ): + """ + Forward pass of the encoder. + + Parameters + ---------- + x: torch.Tensor + (B, D, H, W) + t: torch.Tensor + (B) + encoder_hidden_states: torch.Tensor + CLIP text embedding, (B, L_clip, D) + text_embedding_mask: torch.Tensor + CLIP text embedding mask, (B, L_clip) + encoder_hidden_states_t5: torch.Tensor + T5 text embedding, (B, L_t5, D) + text_embedding_mask_t5: torch.Tensor + T5 text embedding mask, (B, L_t5) + image_meta_size: torch.Tensor + (B, 6) + style: torch.Tensor + (B) + cos_cis_img: torch.Tensor + sin_cis_img: torch.Tensor + return_dict: bool + Whether to return a dictionary. + """ + #print(controls) + #assert(0) + text_states = encoder_hidden_states # 2,77,1024 + text_states_t5 = encoder_hidden_states_t5 # 2,256,2048 + text_states_mask = text_embedding_mask.bool() # 2,77 + text_states_t5_mask = text_embedding_mask_t5.bool() # 2,256 + b_t5, l_t5, c_t5 = text_states_t5.shape + text_states_t5 = self.mlp_t5(text_states_t5.view(-1, c_t5)) + text_states = torch.cat([text_states, text_states_t5.view(b_t5, l_t5, -1)], dim=1) # 2,205，1024 + clip_t5_mask = torch.cat([text_states_mask, text_states_t5_mask], dim=-1) + + clip_t5_mask = clip_t5_mask + text_states = torch.where(clip_t5_mask.unsqueeze(2), text_states, self.text_embedding_padding.to(text_states)) + + _, _, oh, ow = x.shape + th, tw = oh // self.patch_size, ow // self.patch_size + + # ========================= Build time and image embedding ========================= + t = self.t_embedder(t) + x = self.x_embedder(x) + + # Get image RoPE embedding according to `reso`lution. + freqs_cis_img = (cos_cis_img, sin_cis_img) + + # ========================= Concatenate all extra vectors ========================= + # Build text tokens with pooling + extra_vec = self.pooler(encoder_hidden_states_t5) + + # Build image meta size tokens + image_meta_size = timestep_embedding(image_meta_size.view(-1), 256) # [B * 6, 256] + if self.args.use_fp16: + image_meta_size = image_meta_size.half() + image_meta_size = image_meta_size.view(-1, 6 * 256) + extra_vec = torch.cat([extra_vec, image_meta_size], dim=1) # [B, D + 6 * 256] + + # Build style tokens + style_embedding = self.style_embedder(style) + extra_vec = torch.cat([extra_vec, style_embedding], dim=1) + + # Concatenate all extra vectors + c = t + self.extra_embedder(extra_vec) # [B, D] + + # ========================= Forward pass through HunYuanDiT blocks ========================= + skips = [] + for layer, block in enumerate(self.blocks): + if layer > self.depth // 2: + if controls is not None: + skip = skips.pop() + controls.pop() + else: + skip = skips.pop() + x = block(x, c, text_states, freqs_cis_img, skip) # (N, L, D) + else: + x = block(x, c, text_states, freqs_cis_img) # (N, L, D) + + if layer < (self.depth // 2 - 1): + skips.append(x) + if controls is not None and len(controls) != 0: + raise ValueError("The number of controls is not equal to the number of skip connections.") + + # ========================= Final layer ========================= + x = self.final_layer(x, c) # (N, L, patch_size ** 2 * out_channels) + x = self.unpatchify(x, th, tw) # (N, out_channels, H, W) + + if return_dict: + return {'x': x} + return x + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + self.apply(_basic_init) + + # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): + w = self.x_embedder.proj.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + nn.init.constant_(self.x_embedder.proj.bias, 0) + + # Initialize label embedding table: + nn.init.normal_(self.extra_embedder[0].weight, std=0.02) + nn.init.normal_(self.extra_embedder[2].weight, std=0.02) + + # Initialize timestep embedding MLP: + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + + # Zero-out adaLN modulation layers in HunYuanDiT blocks: + for block in self.blocks: + nn.init.constant_(block.default_modulation[-1].weight, 0) + nn.init.constant_(block.default_modulation[-1].bias, 0) + + # Zero-out output layers: + nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) + nn.init.constant_(self.final_layer.linear.weight, 0) + nn.init.constant_(self.final_layer.linear.bias, 0) + + def unpatchify(self, x, h, w): + """ + x: (N, T, patch_size**2 * C) + imgs: (N, H, W, C) + """ + c = self.unpatchify_channels + p = self.x_embedder.patch_size[0] + # h = w = int(x.shape[1] ** 0.5) + assert h * w == x.shape[1] + + x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p)) + return imgs + + def _replace_module(self, parent, child_name, new_module, child) -> None: + setattr(parent, child_name, new_module) + # It's not necessary to set requires_grad here, as that is handled by + # _mark_only_adapters_as_trainable + + # child layer wraps the original module, unpack it + if hasattr(child, "base_layer"): + child = child.get_base_layer() + elif hasattr(child, "quant_linear_module"): + # TODO maybe not necessary to have special treatment? + child = child.quant_linear_module + + if not hasattr(new_module, "base_layer"): + new_module.weight = child.weight + if hasattr(child, "bias"): + new_module.bias = child.bias + + if getattr(child, "state", None) is not None: + if hasattr(new_module, "base_layer"): + new_module.base_layer.state = child.state + else: + new_module.state = child.state + new_module.to(child.weight.device) + + # dispatch to correct device + for name, module in new_module.named_modules(): + # if any(prefix in name for prefix in PREFIXES): + # module.to(child.weight.device) + if "ranknum" in name: + module.to(child.weight.device) + + def merge_and_unload(self, + merge=True, + progressbar: bool = False, + safe_merge: bool = False, + adapter_names = None,): + if merge: + if getattr(self, "quantization_method", None) == "gptq": + raise ValueError("Cannot merge layers when the model is gptq quantized") + + def merge_recursively(module): + # helper function to recursively merge the base_layer of the target + path = [] + layer = module + while hasattr(layer, "base_layer"): + path.append(layer) + layer = layer.base_layer + for layer_before, layer_after in zip(path[:-1], path[1:]): + layer_after.merge(safe_merge=safe_merge, adapter_names=adapter_names) + layer_before.base_layer = layer_after.base_layer + module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + + key_list = [key for key, _ in self.named_modules()] + desc = "Unloading " + ("and merging " if merge else "") + "model" + + for key in tqdm(key_list, disable=not progressbar, desc=desc): + try: + parent, target, target_name = _get_submodules(self, key) + except AttributeError: + continue + + if hasattr(target, "base_layer"): + if merge: + merge_recursively(target) + self._replace_module(parent, target_name, target.get_base_layer(), target) + elif isinstance(target, ModulesToSaveWrapper): + # save any additional trainable modules part of `modules_to_save` + new_module = target.modules_to_save[target.active_adapter] + if hasattr(new_module, "base_layer"): + # check if the module is itself a tuner layer + if merge: + new_module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + new_module = new_module.get_base_layer() + setattr(parent, target_name, new_module) + + + +################################################################################# +# HunYuanDiT Configs # +################################################################################# + +HUNYUAN_DIT_CONFIG = { + 'DiT-g/2': {'depth': 40, 'hidden_size': 1408, 'patch_size': 2, 'num_heads': 16, 'mlp_ratio': 4.3637}, + 'DiT-XL/2': {'depth': 28, 'hidden_size': 1152, 'patch_size': 2, 'num_heads': 16}, +} + +def DiT_g_2(args, **kwargs): + return HunYuanDiT(args, depth=40, hidden_size=1408, patch_size=2, num_heads=16, mlp_ratio=4.3637, **kwargs) +def DiT_XL_2(args, **kwargs): + return HunYuanDiT(args, depth=28, hidden_size=1152, patch_size=2, num_heads=16, **kwargs) + +HUNYUAN_DIT_MODELS = { + 'DiT-g/2': DiT_g_2, + 'DiT-XL/2': DiT_XL_2, +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models_comfyui.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models_comfyui.py new file mode 100644 index 0000000000000000000000000000000000000000..e96ebd007e5020102b88beb061accc6ff1d25bb6 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/models_comfyui.py @@ -0,0 +1,506 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from timm.models.vision_transformer import Mlp + +from .attn_layers import Attention, FlashCrossMHAModified, FlashSelfMHAModified, CrossAttention +from .embedders import TimestepEmbedder, PatchEmbed, timestep_embedding +from .norm_layers import RMSNorm +from .poolers import AttentionPool +from .posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from transformers.integrations import PeftAdapterMixin +from peft.utils import ( + ModulesToSaveWrapper, + _get_submodules, +) +import tqdm + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class FP32_Layernorm(nn.LayerNorm): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + origin_dtype = inputs.dtype + return F.layer_norm(inputs.float(), self.normalized_shape, self.weight.float(), self.bias.float(), + self.eps).to(origin_dtype) + + +class FP32_SiLU(nn.SiLU): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.silu(inputs.float(), inplace=False).to(inputs.dtype) + + +class HunYuanDiTBlock(nn.Module): + """ + A HunYuanDiT block with `add` conditioning. + """ + def __init__(self, + hidden_size, + c_emb_size, + num_heads, + mlp_ratio=4.0, + text_states_dim=1024, + use_flash_attn=False, + qk_norm=False, + norm_type="layer", + skip=False, + ): + super().__init__() + self.use_flash_attn = use_flash_attn + use_ele_affine = True + + if norm_type == "layer": + norm_layer = FP32_Layernorm + elif norm_type == "rms": + norm_layer = RMSNorm + else: + raise ValueError(f"Unknown norm_type: {norm_type}") + + # ========================= Self-Attention ========================= + self.norm1 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + if use_flash_attn: + self.attn1 = FlashSelfMHAModified(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + else: + self.attn1 = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + + # ========================= FFN ========================= + self.norm2 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + approx_gelu = lambda: nn.GELU(approximate="tanh") + self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0) + + # ========================= Add ========================= + # Simply use add like SDXL. + self.default_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, hidden_size, bias=True) + ) + + # ========================= Cross-Attention ========================= + if use_flash_attn: + self.attn2 = FlashCrossMHAModified(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + else: + self.attn2 = CrossAttention(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + self.norm3 = norm_layer(hidden_size, elementwise_affine=True, eps=1e-6) + + # ========================= Skip Connection ========================= + if skip: + self.skip_norm = norm_layer(2 * hidden_size, elementwise_affine=True, eps=1e-6) + self.skip_linear = nn.Linear(2 * hidden_size, hidden_size) + else: + self.skip_linear = None + + def forward(self, x, c=None, text_states=None, freq_cis_img=None, skip=None): + # Long Skip Connection + if self.skip_linear is not None: + cat = torch.cat([x, skip], dim=-1) + cat = self.skip_norm(cat) + x = self.skip_linear(cat) + + # Self-Attention + shift_msa = self.default_modulation(c).unsqueeze(dim=1) + attn_inputs = ( + self.norm1(x) + shift_msa, freq_cis_img, + ) + x = x + self.attn1(*attn_inputs)[0] + + # Cross-Attention + cross_inputs = ( + self.norm3(x), text_states, freq_cis_img + ) + x = x + self.attn2(*cross_inputs)[0] + + # FFN Layer + mlp_inputs = self.norm2(x) + x = x + self.mlp(mlp_inputs) + + return x + + +class FinalLayer(nn.Module): + """ + The final layer of HunYuanDiT. + """ + def __init__(self, final_hidden_size, c_emb_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(final_hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(final_hidden_size, patch_size * patch_size * out_channels, bias=True) + self.adaLN_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, 2 * final_hidden_size, bias=True) + ) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class HunYuanDiT(ModelMixin, ConfigMixin,PeftAdapterMixin): + """ + HunYuanDiT: Diffusion model with a Transformer backbone. + + Inherit ModelMixin and ConfigMixin to be compatible with the sampler StableDiffusionPipeline of diffusers. + + Parameters + ---------- + args: argparse.Namespace + The arguments parsed by argparse. + input_size: tuple + The size of the input image. + patch_size: int + The size of the patch. + in_channels: int + The number of input channels. + hidden_size: int + The hidden size of the transformer backbone. + depth: int + The number of transformer blocks. + num_heads: int + The number of attention heads. + mlp_ratio: float + The ratio of the hidden size of the MLP in the transformer block. + log_fn: callable + The logging function. + """ + @register_to_config + def __init__( + self, args, + input_size=(32, 32), + patch_size=2, + in_channels=4, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + log_fn=print, + **kwargs, + ): + super().__init__() + self.args = args + self.log_fn = log_fn + self.depth = depth + self.learn_sigma = args.learn_sigma + self.in_channels = in_channels + self.out_channels = in_channels * 2 if args.learn_sigma else in_channels + self.patch_size = patch_size + self.num_heads = num_heads + self.hidden_size = hidden_size + self.text_states_dim = args.text_states_dim + self.text_states_dim_t5 = args.text_states_dim_t5 + self.text_len = args.text_len + self.text_len_t5 = args.text_len_t5 + self.norm = args.norm + + use_flash_attn = args.infer_mode == 'fa' + if use_flash_attn: + log_fn(f" Enable Flash Attention.") + qk_norm = True # See http://arxiv.org/abs/2302.05442 for details. + + self.mlp_t5 = nn.Sequential( + nn.Linear(self.text_states_dim_t5, self.text_states_dim_t5 * 4, bias=True), + FP32_SiLU(), + nn.Linear(self.text_states_dim_t5 * 4, self.text_states_dim, bias=True), + ) + # learnable replace + self.text_embedding_padding = nn.Parameter( + torch.randn(self.text_len + self.text_len_t5, self.text_states_dim, dtype=torch.float32)) + + # Attention pooling + self.pooler = AttentionPool(self.text_len_t5, self.text_states_dim_t5, num_heads=8, output_dim=1024) + + # Here we use a default learned embedder layer for future extension. + self.style_embedder = nn.Embedding(1, hidden_size) + + # Image size and crop size conditions + self.extra_in_dim = 256 * 6 + hidden_size + + # Text embedding for `add` + self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size) + self.t_embedder = TimestepEmbedder(hidden_size) + self.extra_in_dim += 1024 + self.extra_embedder = nn.Sequential( + nn.Linear(self.extra_in_dim, hidden_size * 4), + FP32_SiLU(), + nn.Linear(hidden_size * 4, hidden_size, bias=True), + ) + + # Image embedding + num_patches = self.x_embedder.num_patches + log_fn(f" Number of tokens: {num_patches}") + + # HUnYuanDiT Blocks + self.blocks = nn.ModuleList([ + HunYuanDiTBlock(hidden_size=hidden_size, + c_emb_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + text_states_dim=self.text_states_dim, + use_flash_attn=use_flash_attn, + qk_norm=qk_norm, + norm_type=self.norm, + skip=layer > depth // 2, + ) + for layer in range(depth) + ]) + + self.final_layer = FinalLayer(hidden_size, hidden_size, patch_size, self.out_channels) + self.unpatchify_channels = self.out_channels + + self.initialize_weights() + + def forward(self, + x, + t, + encoder_hidden_states=None, + text_embedding_mask=None, + encoder_hidden_states_t5=None, + text_embedding_mask_t5=None, + image_meta_size=None, + style=None, + cos_cis_img=None, + sin_cis_img=None, + return_dict=True, + ): + """ + Forward pass of the encoder. + + Parameters + ---------- + x: torch.Tensor + (B, D, H, W) + t: torch.Tensor + (B) + encoder_hidden_states: torch.Tensor + CLIP text embedding, (B, L_clip, D) + text_embedding_mask: torch.Tensor + CLIP text embedding mask, (B, L_clip) + encoder_hidden_states_t5: torch.Tensor + T5 text embedding, (B, L_t5, D) + text_embedding_mask_t5: torch.Tensor + T5 text embedding mask, (B, L_t5) + image_meta_size: torch.Tensor + (B, 6) + style: torch.Tensor + (B) + cos_cis_img: torch.Tensor + sin_cis_img: torch.Tensor + return_dict: bool + Whether to return a dictionary. + """ + + text_states = encoder_hidden_states # 2,77,1024 + text_states_t5 = encoder_hidden_states_t5 # 2,256,2048 + text_states_mask = text_embedding_mask.bool() # 2,77 + text_states_t5_mask = text_embedding_mask_t5.bool() # 2,256 + b_t5, l_t5, c_t5 = text_states_t5.shape + text_states_t5 = self.mlp_t5(text_states_t5.view(-1, c_t5)) + text_states = torch.cat([text_states, text_states_t5.view(b_t5, l_t5, -1)], dim=1) # 2,205，1024 + clip_t5_mask = torch.cat([text_states_mask, text_states_t5_mask], dim=-1) + + clip_t5_mask = clip_t5_mask + text_states = torch.where(clip_t5_mask.unsqueeze(2), text_states, self.text_embedding_padding.to(text_states)) + + _, _, oh, ow = x.shape + th, tw = oh // self.patch_size, ow // self.patch_size + + # ========================= Build time and image embedding ========================= + t = self.t_embedder(t) + x = self.x_embedder(x) + + # Get image RoPE embedding according to `reso`lution. + freqs_cis_img = (cos_cis_img, sin_cis_img) + + # ========================= Concatenate all extra vectors ========================= + # Build text tokens with pooling + extra_vec = self.pooler(encoder_hidden_states_t5) + + # Build image meta size tokens + image_meta_size = timestep_embedding(image_meta_size.view(-1), 256) # [B * 6, 256] + if self.args.use_fp16: + image_meta_size = image_meta_size.half() + image_meta_size = image_meta_size.view(-1, 6 * 256) + extra_vec = torch.cat([extra_vec, image_meta_size], dim=1) # [B, D + 6 * 256] + + # Build style tokens + style_embedding = self.style_embedder(style) + extra_vec = torch.cat([extra_vec, style_embedding], dim=1) + + # Concatenate all extra vectors + c = t + self.extra_embedder(extra_vec) # [B, D] + + # ========================= Forward pass through HunYuanDiT blocks ========================= + skips = [] + for layer, block in enumerate(self.blocks): + if layer > self.depth // 2: + skip = skips.pop() + x = block(x, c, text_states, freqs_cis_img, skip) # (N, L, D) + else: + x = block(x, c, text_states, freqs_cis_img) # (N, L, D) + + if layer < (self.depth // 2 - 1): + skips.append(x) + + # ========================= Final layer ========================= + x = self.final_layer(x, c) # (N, L, patch_size ** 2 * out_channels) + x = self.unpatchify(x, th, tw) # (N, out_channels, H, W) + + if return_dict: + return {'x': x} + return x + + def calc_rope(self, height, width): + """ + Probably not the best in terms of perf to have this here + """ + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + head_size = self.hidden_size // self.num_heads + rope = get_2d_rotary_pos_embed(head_size, *sub_args) + return rope + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + self.apply(_basic_init) + + # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): + w = self.x_embedder.proj.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + nn.init.constant_(self.x_embedder.proj.bias, 0) + + # Initialize label embedding table: + nn.init.normal_(self.extra_embedder[0].weight, std=0.02) + nn.init.normal_(self.extra_embedder[2].weight, std=0.02) + + # Initialize timestep embedding MLP: + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + + # Zero-out adaLN modulation layers in HunYuanDiT blocks: + for block in self.blocks: + nn.init.constant_(block.default_modulation[-1].weight, 0) + nn.init.constant_(block.default_modulation[-1].bias, 0) + + # Zero-out output layers: + nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) + nn.init.constant_(self.final_layer.linear.weight, 0) + nn.init.constant_(self.final_layer.linear.bias, 0) + + def unpatchify(self, x, h, w): + """ + x: (N, T, patch_size**2 * C) + imgs: (N, H, W, C) + """ + c = self.unpatchify_channels + p = self.x_embedder.patch_size[0] + # h = w = int(x.shape[1] ** 0.5) + assert h * w == x.shape[1] + + x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p)) + return imgs + + def merge_and_unload(self, + merge=True, + progressbar: bool = False, + safe_merge: bool = False, + adapter_names = None,): + if merge: + if getattr(self, "quantization_method", None) == "gptq": + raise ValueError("Cannot merge layers when the model is gptq quantized") + + def merge_recursively(module): + # helper function to recursively merge the base_layer of the target + path = [] + layer = module + while hasattr(layer, "base_layer"): + path.append(layer) + layer = layer.base_layer + for layer_before, layer_after in zip(path[:-1], path[1:]): + layer_after.merge(safe_merge=safe_merge, adapter_names=adapter_names) + layer_before.base_layer = layer_after.base_layer + module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + + key_list = [key for key, _ in self.named_modules()] + desc = "Unloading " + ("and merging " if merge else "") + "model" + #print(key_list) + + #for key in tqdm(key_list, disable=not progressbar, desc=desc): + for key in key_list: + try: + parent, target, target_name = _get_submodules(self, key) + except AttributeError: + continue + + if hasattr(target, "base_layer"): + if merge: + merge_recursively(target) + self._replace_module(parent, target_name, target.get_base_layer(), target) + elif isinstance(target, ModulesToSaveWrapper): + # save any additional trainable modules part of `modules_to_save` + new_module = target.modules_to_save[target.active_adapter] + if hasattr(new_module, "base_layer"): + # check if the module is itself a tuner layer + if merge: + new_module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + new_module = new_module.get_base_layer() + setattr(parent, target_name, new_module) + + def _replace_module(self, parent, child_name, new_module, child) -> None: + setattr(parent, child_name, new_module) + # It's not necessary to set requires_grad here, as that is handled by + # _mark_only_adapters_as_trainable + + # child layer wraps the original module, unpack it + if hasattr(child, "base_layer"): + child = child.get_base_layer() + elif hasattr(child, "quant_linear_module"): + # TODO maybe not necessary to have special treatment? + child = child.quant_linear_module + + if not hasattr(new_module, "base_layer"): + new_module.weight = child.weight + if hasattr(child, "bias"): + new_module.bias = child.bias + + if getattr(child, "state", None) is not None: + if hasattr(new_module, "base_layer"): + new_module.base_layer.state = child.state + else: + new_module.state = child.state + new_module.to(child.weight.device) + + # dispatch to correct device + for name, module in new_module.named_modules(): + # if any(prefix in name for prefix in PREFIXES): + # module.to(child.weight.device) + if "ranknum" in name: + module.to(child.weight.device) + + +################################################################################# +# HunYuanDiT Configs # +################################################################################# + +HUNYUAN_DIT_CONFIG = { + 'DiT-g/2': {'depth': 40, 'hidden_size': 1408, 'patch_size': 2, 'num_heads': 16, 'mlp_ratio': 4.3637}, + 'DiT-XL/2': {'depth': 28, 'hidden_size': 1152, 'patch_size': 2, 'num_heads': 16}, + 'DiT-L/2': {'depth': 24, 'hidden_size': 1024, 'patch_size': 2, 'num_heads': 16}, + 'DiT-B/2': {'depth': 12, 'hidden_size': 768, 'patch_size': 2, 'num_heads': 12}, +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/norm_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/norm_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..5204ad9e5aabd5dad05a0f84408ff85f96cfa8db --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/norm_layers.py @@ -0,0 +1,68 @@ +import torch +import torch.nn as nn + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, elementwise_affine=True, eps: float = 1e-6): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + super().__init__() + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim)) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +class GroupNorm32(nn.GroupNorm): + def __init__(self, num_groups, num_channels, eps=1e-5, dtype=None): + super().__init__(num_groups=num_groups, num_channels=num_channels, eps=eps, dtype=dtype) + + def forward(self, x): + y = super().forward(x).to(x.dtype) + return y + +def normalization(channels, dtype=None): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: an nn.Module for normalization. + """ + return GroupNorm32(num_channels=channels, num_groups=32, dtype=dtype) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/poolers.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/poolers.py new file mode 100644 index 0000000000000000000000000000000000000000..a4adcaca51fded2268a644ca4c70d5b33dfcd3b0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/poolers.py @@ -0,0 +1,39 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class AttentionPool(nn.Module): + def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter(torch.randn(spacial_dim + 1, embed_dim) / embed_dim ** 0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.permute(1, 0, 2) # NLC -> LNC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (L+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (L+1)NC + x, _ = F.multi_head_attention_forward( + query=x[:1], key=x, value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False + ) + return x.squeeze(0) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/posemb_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/posemb_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..dcb41a713cd94ea8472ff26e8865066887b1e486 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/posemb_layers.py @@ -0,0 +1,224 @@ +import torch +import numpy as np +from typing import Union + + +def _to_tuple(x): + if isinstance(x, int): + return x, x + else: + return x + + +def get_fill_resize_and_crop(src, tgt): + th, tw = _to_tuple(tgt) + h, w = _to_tuple(src) + + tr = th / tw # base resolution + r = h / w # target resolution + + # resize + if r > tr: + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) # resize the target resolution down based on the base resolution + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +def get_meshgrid(start, *args): + if len(args) == 0: + # start is grid_size + num = _to_tuple(start) + start = (0, 0) + stop = num + elif len(args) == 1: + # start is start, args[0] is stop, step is 1 + start = _to_tuple(start) + stop = _to_tuple(args[0]) + num = (stop[0] - start[0], stop[1] - start[1]) + elif len(args) == 2: + # start is start, args[0] is stop, args[1] is num + start = _to_tuple(start) + stop = _to_tuple(args[0]) + num = _to_tuple(args[1]) + else: + raise ValueError(f"len(args) should be 0, 1 or 2, but got {len(args)}") + + grid_h = np.linspace(start[0], stop[0], num[0], endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], num[1], endpoint=False, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) # [2, W, H] + return grid + +################################################################################# +# Sine/Cosine Positional Embedding Functions # +################################################################################# +# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + +def get_2d_sincos_pos_embed(embed_dim, start, *args, cls_token=False, extra_tokens=0): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid = get_meshgrid(start, *args) # [2, H, w] + # grid_h = np.arange(grid_size, dtype=np.float32) + # grid_w = np.arange(grid_size, dtype=np.float32) + # grid = np.meshgrid(grid_w, grid_h) # here w goes first + # grid = np.stack(grid, axis=0) # [2, W, H] + + grid = grid.reshape([2, 1, *grid.shape[1:]]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (W,H) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2. + omega = 1. / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +################################################################################# +# Rotary Positional Embedding Functions # +################################################################################# +# https://github.com/facebookresearch/llama/blob/main/llama/model.py#L443 + +def get_2d_rotary_pos_embed(embed_dim, start, *args, use_real=True): + """ + This is a 2d version of precompute_freqs_cis, which is a RoPE for image tokens with 2d structure. + + Parameters + ---------- + embed_dim: int + embedding dimension size + start: int or tuple of int + If len(args) == 0, start is num; If len(args) == 1, start is start, args[0] is stop, step is 1; + If len(args) == 2, start is start, args[0] is stop, args[1] is num. + use_real: bool + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + + Returns + ------- + pos_embed: torch.Tensor + [HW, D/2] + """ + grid = get_meshgrid(start, *args) # [2, H, w] + grid = grid.reshape([2, 1, *grid.shape[1:]]) # Returns a sampling matrix with the same resolution as the target resolution + pos_embed = get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=use_real) + return pos_embed + + +def get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=False): + assert embed_dim % 4 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_rotary_pos_embed(embed_dim // 2, grid[0].reshape(-1), use_real=use_real) # (H*W, D/4) + emb_w = get_1d_rotary_pos_embed(embed_dim // 2, grid[1].reshape(-1), use_real=use_real) # (H*W, D/4) + + if use_real: + cos = torch.cat([emb_h[0], emb_w[0]], dim=1) # (H*W, D/2) + sin = torch.cat([emb_h[1], emb_w[1]], dim=1) # (H*W, D/2) + return cos, sin + else: + emb = torch.cat([emb_h, emb_w], dim=1) # (H*W, D/2) + return emb + + +def get_1d_rotary_pos_embed(dim: int, pos: Union[np.ndarray, int], theta: float = 10000.0, use_real=False): + """ + Precompute the frequency tensor for complex exponentials (cis) with given dimensions. + + This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' + and the end index 'end'. The 'theta' parameter scales the frequencies. + The returned tensor contains complex values in complex64 data type. + + Args: + dim (int): Dimension of the frequency tensor. + pos (np.ndarray, int): Position indices for the frequency tensor. [S] or scalar + theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. + use_real (bool, optional): If True, return real part and imaginary part separately. + Otherwise, return complex numbers. + + Returns: + torch.Tensor: Precomputed frequency tensor with complex exponentials. [S, D/2] + + """ + if isinstance(pos, int): + pos = np.arange(pos) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) # [D/2] + t = torch.from_numpy(pos).to(freqs.device) # type: ignore # [S] + freqs = torch.outer(t, freqs).float() # type: ignore # [S, D/2] + if use_real: + freqs_cos = freqs.cos().repeat_interleave(2, dim=1) # [S, D] + freqs_sin = freqs.sin().repeat_interleave(2, dim=1) # [S, D] + return freqs_cos, freqs_sin + else: + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 # [S, D/2] + return freqs_cis + + + +def calc_sizes(rope_img, patch_size, th, tw): + if rope_img == 'extend': + # Expansion mode + sub_args = [(th, tw)] + elif rope_img.startswith('base'): + # Based on the specified dimensions, other dimensions are obtained through interpolation. + base_size = int(rope_img[4:]) // 8 // patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + else: + raise ValueError(f"Unknown rope_img: {rope_img}") + return sub_args + + +def init_image_posemb(rope_img, + resolutions, + patch_size, + hidden_size, + num_heads, + log_fn, + rope_real=True, + ): + freqs_cis_img = {} + for reso in resolutions: + th, tw = reso.height // 8 // patch_size, reso.width // 8 // patch_size + sub_args = calc_sizes(rope_img, patch_size, th, tw) + freqs_cis_img[str(reso)] = get_2d_rotary_pos_embed(hidden_size // num_heads, *sub_args, use_real=rope_real) + log_fn(f" Using image RoPE ({rope_img}) ({'real' if rope_real else 'complex'}): {sub_args} | ({reso}) " + f"{freqs_cis_img[str(reso)][0].shape if rope_real else freqs_cis_img[str(reso)].shape}") + return freqs_cis_img diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..eec49b7a71223b6120f25356f0a407cd1ad7c2d5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder.py @@ -0,0 +1,129 @@ +import torch +import torch.nn as nn +from transformers import AutoTokenizer, T5EncoderModel, T5ForConditionalGeneration,T5Config,modeling_utils +import os +import comfy.utils + +class MT5Embedder(nn.Module): + available_models = ["t5-v1_1-xxl"] + + def __init__( + self, + model_dir="t5-v1_1-xxl", + model_kwargs=None, + torch_dtype=None, + use_tokenizer_only=False, + conditional_generation=False, + max_length=128, + ksampler = False + ): + super().__init__() + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.torch_dtype = torch_dtype or torch.bfloat16 + self.max_length = max_length + if model_kwargs is None: + model_kwargs = { + # "low_cpu_mem_usage": True, + "torch_dtype": self.torch_dtype, + } + model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device} + if not ksampler: + self.tokenizer = AutoTokenizer.from_pretrained(model_dir) + if use_tokenizer_only: + return + """ + if conditional_generation: + self.model = None + self.generation_model = T5ForConditionalGeneration.from_pretrained( + model_dir + ) + return + """ + self.model = T5EncoderModel.from_pretrained(model_dir, **model_kwargs).eval().to(self.torch_dtype) + else: + tokenizer_path = os.path.join( # local + os.path.dirname(os.path.realpath(__file__)), "..", "..", + "tokenizer_mt5", + ) + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + #self.tokenizer.eval().to(self.device) + if use_tokenizer_only: + return + """ + if conditional_generation: + self.model = None + self.generation_model = T5ForConditionalGeneration.from_pretrained( + model_dir + ) + return + """ + textmodel_json_config = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "..", "..", + f"config_mt5.json" + ) + config = T5Config.from_json_file(textmodel_json_config) + with modeling_utils.no_init_weights(): + self.model = T5EncoderModel(config) + sd = comfy.utils.load_torch_file(model_dir) + self.model.load_state_dict(sd, strict=False) + self.model.eval().to(self.device) + + def get_tokens_and_mask(self, texts): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + tokens = text_tokens_and_mask["input_ids"][0] + mask = text_tokens_and_mask["attention_mask"][0] + # tokens = torch.tensor(tokens).clone().detach() + # mask = torch.tensor(mask, dtype=torch.bool).clone().detach() + return tokens, mask + + def get_text_embeddings(self, texts, attention_mask=True, layer_index=-1): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + with torch.no_grad(): + outputs = self.model( + input_ids=text_tokens_and_mask["input_ids"].to(self.device), + attention_mask=text_tokens_and_mask["attention_mask"].to(self.device) + if attention_mask + else None, + output_hidden_states=True, + ) + text_encoder_embs = outputs["hidden_states"][layer_index].detach() + + return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device) + + @torch.no_grad() + def __call__(self, tokens, attention_mask, layer_index=-1): + with torch.cuda.amp.autocast(): + outputs = self.model( + input_ids=tokens, + attention_mask=attention_mask, + output_hidden_states=True, + ) + + z = outputs.hidden_states[layer_index].detach() + return z + + def general(self, text: str): + # input_ids = input_ids = torch.tensor([list(text.encode("utf-8"))]) + num_special_tokens + input_ids = self.tokenizer(text, max_length=128).input_ids + print(input_ids) + outputs = self.generation_model(input_ids) + return outputs + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder_comfyui.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder_comfyui.py new file mode 100644 index 0000000000000000000000000000000000000000..7a16b21f4ffbd9acc896c93e2833caf5aecfa0a2 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/text_encoder_comfyui.py @@ -0,0 +1,95 @@ +import torch +import torch.nn as nn +from transformers import AutoTokenizer, T5EncoderModel, T5ForConditionalGeneration + + +class MT5Embedder(nn.Module): + available_models = ["t5-v1_1-xxl"] + + def __init__( + self, + model_dir="t5-v1_1-xxl", + model_kwargs=None, + torch_dtype=None, + use_tokenizer_only=False, + conditional_generation=False, + max_length=128, + ): + super().__init__() + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.torch_dtype = torch_dtype or torch.bfloat16 + self.max_length = max_length + if model_kwargs is None: + model_kwargs = { + # "low_cpu_mem_usage": True, + "torch_dtype": self.torch_dtype, + } + model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device} + self.tokenizer = AutoTokenizer.from_pretrained(model_dir) + if use_tokenizer_only: + return + if conditional_generation: + self.model = None + self.generation_model = T5ForConditionalGeneration.from_pretrained( + model_dir + ) + return + self.model = T5EncoderModel.from_pretrained(model_dir, **model_kwargs).eval().to(self.torch_dtype) + + def get_tokens_and_mask(self, texts): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + tokens = text_tokens_and_mask["input_ids"][0] + mask = text_tokens_and_mask["attention_mask"][0] + # tokens = torch.tensor(tokens).clone().detach() + # mask = torch.tensor(mask, dtype=torch.bool).clone().detach() + return tokens, mask + + def get_text_embeddings(self, texts, attention_mask=True, layer_index=-1): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + with torch.no_grad(): + outputs = self.model( + input_ids=text_tokens_and_mask["input_ids"].to(self.device), + attention_mask=text_tokens_and_mask["attention_mask"].to(self.device) + if attention_mask + else None, + output_hidden_states=True, + ) + text_encoder_embs = outputs["hidden_states"][layer_index].detach() + + return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device) + + @torch.no_grad() + def __call__(self, tokens, attention_mask, layer_index=-1): + with torch.cuda.amp.autocast(): + outputs = self.model( + input_ids=tokens, + attention_mask=attention_mask, + output_hidden_states=True, + ) + + z = outputs.hidden_states[layer_index].detach() + return z + + def general(self, text: str): + # input_ids = input_ids = torch.tensor([list(text.encode("utf-8"))]) + num_special_tokens + input_ids = self.tokenizer(text, max_length=128).input_ids + print(input_ids) + outputs = self.generation_model(input_ids) + return outputs \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/engine.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..bba110edf70dd4fd2c6df218a2d3aa1f915d6617 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/engine.py @@ -0,0 +1,152 @@ +# +# Copyright 2022 The HuggingFace Inc. team. +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from collections import OrderedDict +from copy import copy + +import numpy as np +import tensorrt as trt +import torch +from polygraphy import cuda +from polygraphy.backend.common import bytes_from_path +from polygraphy.backend.trt import CreateConfig, Profile +from polygraphy.backend.trt import engine_from_bytes, engine_from_network, network_from_onnx_path, save_engine +from polygraphy.backend.trt import util as trt_util +import ctypes +from glob import glob +from cuda import cudart + +TRT_LOGGER = trt.Logger(trt.Logger.INFO) +trt_util.TRT_LOGGER = TRT_LOGGER + + +class Engine(): + def __init__( + self, + model_name, + engine_dir, + onnx_file=None, + ): + self.engine_path = os.path.join(engine_dir, model_name + '.plan') + self.engine = None + self.context = None + self.buffers = OrderedDict() + self.tensors = OrderedDict() + + self.weightNameList = None + self.refitter = None + self.onnx_initializers = None + self.onnx_file = onnx_file + self.trt_lora_weight = None + self.trt_lora_weight_mem = None + self.torch_weight = None + + def __del__(self): + del self.engine + del self.context + del self.buffers + del self.tensors + + def build(self, onnx_path, fp16, input_profile=None, enable_preview=False, sparse_weights=False): + print(f"Building TensorRT engine for {onnx_path}: {self.engine_path}") + p = Profile() + if input_profile: + for name, dims in input_profile.items(): + assert len(dims) == 3 + p.add(name, min=dims[0], opt=dims[1], max=dims[2]) + + preview_features = [] + if enable_preview: + trt_version = [int(i) for i in trt.__version__.split(".")] + # FASTER_DYNAMIC_SHAPES_0805 should only be used for TRT 8.5.1 or above. + if trt_version[0] > 8 or \ + (trt_version[0] == 8 and (trt_version[1] > 5 or (trt_version[1] == 5 and trt_version[2] >= 1))): + preview_features = [trt.PreviewFeature.FASTER_DYNAMIC_SHAPES_0805] + + engine = engine_from_network(network_from_onnx_path(onnx_path), config=CreateConfig(fp16=fp16, profiles=[p], + preview_features=preview_features, + sparse_weights=sparse_weights)) + save_engine(engine, path=self.engine_path) + + def activate(self, plugin_path=""): + ctypes.cdll.LoadLibrary(plugin_path) + self.engine = engine_from_bytes(bytes_from_path(self.engine_path)) + self.context = self.engine.create_execution_context() + + def get_shared_memory(self): + _, device_memory = cudart.cudaMalloc(self.engine.device_memory_size) + self.device_memory = device_memory + return self.device_memory + + def set_shared_memory(self, device_memory_size): + self.context.device_memory = device_memory_size + + def binding_input(self, name, shape): + idx = self.engine.get_binding_index(name) + result = self.context.set_binding_shape(idx, shape) + return result + + def allocate_buffers(self, shape_dict=None, device='cuda'): + print("Allocate buffers and bindings inputs:") + for idx in range(trt_util.get_bindings_per_profile(self.engine)): + binding = self.engine[idx] + print("binding: ", binding) + if shape_dict and binding in shape_dict: + shape = shape_dict[binding] + else: + shape = self.engine.get_binding_shape(binding) + nv_dtype = self.engine.get_binding_dtype(binding) + dtype_map = {trt.DataType.FLOAT: np.float32, + trt.DataType.HALF: np.float16, + trt.DataType.INT8: np.int8, + trt.DataType.INT64: np.int64, + trt.DataType.BOOL: bool} + if hasattr(trt.DataType, 'INT32'): + dtype_map[trt.DataType.INT32] = np.int32 + dtype = dtype_map[nv_dtype] + if self.engine.binding_is_input(binding): + self.context.set_binding_shape(idx, shape) + # Workaround to convert np dtype to torch + np_type_tensor = np.empty(shape=[], dtype=dtype) + torch_type_tensor = torch.from_numpy(np_type_tensor) + tensor = torch.empty(tuple(shape), dtype=torch_type_tensor.dtype).to(device=device) + + print(f" binding={binding}, shape={shape}, dtype={tensor.dtype}") + self.tensors[binding] = tensor + self.buffers[binding] = cuda.DeviceView(ptr=tensor.data_ptr(), shape=shape, dtype=dtype) + + def infer(self, feed_dict, stream): + start_binding, end_binding = trt_util.get_active_profile_bindings(self.context) + # shallow copy of ordered dict + device_buffers = copy(self.buffers) + for name, buf in feed_dict.items(): + assert isinstance(buf, cuda.DeviceView) + device_buffers[name] = buf + self.binding_input(name, buf.shape) + bindings = [0] * start_binding + [buf.ptr for buf in device_buffers.values()] + noerror = self.context.execute_async_v2(bindings=bindings, stream_handle=stream.ptr) + if not noerror: + raise ValueError(f"ERROR: inference failed.") + + for idx in range(trt_util.get_bindings_per_profile(self.engine)): + binding = self.engine[idx] + if not self.engine.binding_is_input(binding): + shape = self.context.get_binding_shape(idx) + self.tensors[binding].resize_(tuple(shape)) + return self.tensors diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/hcf_model.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/hcf_model.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ded21c8192af6c03568f81eecf078eb56226a3 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/modules/trt/hcf_model.py @@ -0,0 +1,123 @@ +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from polygraphy import cuda + +from .engine import Engine + + +class TRTModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + in_channels=4, + model_name="unet-dyn", + engine_dir="./unet", + device_id=0, + fp16=True, + image_width=1024, + image_height=1024, + text_maxlen=77, + embedding_dim=768, + max_batch_size=1, + plugin_path="./ckpts/trt_model/fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so", + ): + super().__init__() + # create engine + self.in_channels = in_channels # For pipeline compatibility + self.fp16 = fp16 + self.max_batch_size = max_batch_size + self.model_name = model_name + self.engine_dir = engine_dir + self.engine = Engine(self.model_name, self.engine_dir) + self.engine.activate(plugin_path) + # create cuda stream + self.stream = cuda.Stream() + # create inputs buffer + self.latent_width = image_width // 8 + self.latent_height = image_height // 8 + self.text_maxlen = text_maxlen + self.embedding_dim = embedding_dim + shape_dict = { + 'x': (2 * self.max_batch_size, 4, self.latent_height, self.latent_width), + 't': (2 * self.max_batch_size,), + 'encoder_hidden_states': (2 * self.max_batch_size, self.text_maxlen, self.embedding_dim), + 'text_embedding_mask': (2 * self.max_batch_size, self.text_maxlen), + 'encoder_hidden_states_t5': (2 * self.max_batch_size, 256, 2048), + 'text_embedding_mask_t5': (2 * self.max_batch_size, 256), + 'image_meta_size': (2 * self.max_batch_size, 6), + 'style': (2 * self.max_batch_size,), + 'cos_cis_img': (6400, 88), + 'sin_cis_img': (6400, 88), + 'output': (2 * self.max_batch_size, 8, self.latent_height, self.latent_width), + } + device = "cuda:{}".format(device_id) + self.engine_device = torch.device(device) + self.engine.allocate_buffers(shape_dict=shape_dict, device=device) + + print("[INFO] Create hcf nv controlled unet success") + + @property + def device(self): + return self.engine_device + + def __call__(self, x, t_emb, context, image_meta_size, style, freqs_cis_img0, + freqs_cis_img1, text_embedding_mask, encoder_hidden_states_t5, text_embedding_mask_t5): + return self.forward(x=x, t_emb=t_emb, context=context, image_meta_size=image_meta_size, style=style, + freqs_cis_img0=freqs_cis_img0, freqs_cis_img1=freqs_cis_img1, + text_embedding_mask=text_embedding_mask, encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5) + + def get_shared_memory(self): + return self.engine.get_shared_memory() + + def set_shared_memory(self, shared_memory): + self.engine.set_shared_memory(shared_memory) + + def forward(self, x, t_emb, context, image_meta_size, style, freqs_cis_img0, + freqs_cis_img1, text_embedding_mask, encoder_hidden_states_t5, text_embedding_mask_t5): + x_c = x.half() + t_emb_c = t_emb.half() + context_c = context.half() + image_meta_size_c = image_meta_size.half() + style_c = style.long() + freqs_cis_img0_c = freqs_cis_img0.float() + freqs_cis_img1_c = freqs_cis_img1.float() + text_embedding_mask_c = text_embedding_mask.long() + encoder_hidden_states_t5_c = encoder_hidden_states_t5.half() + text_embedding_mask_t5_c = text_embedding_mask_t5.long() + dtype = np.float16 + batch_size = x.shape[0] // 2 + if batch_size <= self.max_batch_size: + sample_inp = cuda.DeviceView(ptr=x_c.reshape(-1).data_ptr(), shape=x_c.shape, dtype=np.float16) + t_emb_inp = cuda.DeviceView(ptr=t_emb_c.reshape(-1).data_ptr(), shape=t_emb_c.shape, dtype=np.float16) + embeddings_inp = cuda.DeviceView(ptr=context_c.reshape(-1).data_ptr(), shape=context_c.shape, + dtype=np.float16) + image_meta_size_inp = cuda.DeviceView(ptr=image_meta_size_c.reshape(-1).data_ptr(), + shape=image_meta_size_c.shape, dtype=np.float16) + style_inp = cuda.DeviceView(ptr=style_c.reshape(-1).data_ptr(), shape=style_c.shape, dtype=np.int64) + freqs_cis_img0_inp = cuda.DeviceView(ptr=freqs_cis_img0_c.reshape(-1).data_ptr(), + shape=freqs_cis_img0_c.shape, dtype=np.float32) + freqs_cis_img1_inp = cuda.DeviceView(ptr=freqs_cis_img1_c.reshape(-1).data_ptr(), + shape=freqs_cis_img1_c.shape, dtype=np.float32) + text_embedding_mask_inp = cuda.DeviceView(ptr=text_embedding_mask_c.reshape(-1).data_ptr(), + shape=text_embedding_mask_c.shape, dtype=np.int64) + encoder_hidden_states_t5_inp = cuda.DeviceView(ptr=encoder_hidden_states_t5_c.reshape(-1).data_ptr(), + shape=encoder_hidden_states_t5_c.shape, dtype=np.float16) + text_embedding_mask_t5_inp = cuda.DeviceView(ptr=text_embedding_mask_t5_c.reshape(-1).data_ptr(), + shape=text_embedding_mask_t5_c.shape, dtype=np.int64) + feed_dict = {"x": sample_inp, + "t": t_emb_inp, + "encoder_hidden_states": embeddings_inp, + "image_meta_size": image_meta_size_inp, + "text_embedding_mask": text_embedding_mask_inp, + "encoder_hidden_states_t5": encoder_hidden_states_t5_inp, + "text_embedding_mask_t5": text_embedding_mask_t5_inp, + "style": style_inp, "cos_cis_img": freqs_cis_img0_inp, + "sin_cis_img": freqs_cis_img1_inp} + latent = self.engine.infer(feed_dict, self.stream) + return latent['output'] + else: + raise ValueError( + "[ERROR] Input batch_size={} execeed max_batch_size={}".format(batch_size, self.max_batch_size)) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g.sh b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g.sh new file mode 100644 index 0000000000000000000000000000000000000000..50b043c0af14e1b54400dafa47292aa16f9f55d2 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g.sh @@ -0,0 +1,8 @@ +model='DiT-g/2' +params=" \ + --qk-norm \ + --model ${model} \ + --rope-img base512 \ + --rope-real \ + " +deepspeed hydit/train_deepspeed.py ${params} "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g_controlnet.sh b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g_controlnet.sh new file mode 100644 index 0000000000000000000000000000000000000000..ab4d854d6d65b6a430c69ecb87054a72878662ec --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/run_g_controlnet.sh @@ -0,0 +1,8 @@ +model='DiT-g/2' +params=" \ + --qk-norm \ + --model ${model} \ + --rope-img base512 \ + --rope-real \ + " +deepspeed hydit/train_deepspeed_controlnet.py ${params} "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train.sh b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..009d4bd90385744fb76ef8281c239d183e294104 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train.sh @@ -0,0 +1,41 @@ +task_flag="dit_g2_full_1024p" # the task flag is used to identify folders. +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=dataset/porcelain/jsons/porcelain.json # index file for dataloader +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=1 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + + +sh $(dirname "$0")/run_g.sh \ + --task-flag ${task_flag} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_controlnet.sh b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_controlnet.sh new file mode 100644 index 0000000000000000000000000000000000000000..09c21603e5f31283d5a154b849a6c149d9256e2b --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_controlnet.sh @@ -0,0 +1,46 @@ +task_flag="canny_controlnet" # the task flag is used to identify folders. +control_type=canny +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=/path/to/your/indexfile # index file for dataloader +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=2 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + + +sh $(dirname "$0")/run_g_controlnet.sh \ + --task-flag ${task_flag} \ + --control-type ${control_type} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --multireso \ + --reso-step 64 \ + --ema-to-module \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed.py new file mode 100644 index 0000000000000000000000000000000000000000..f82f6a3fd6eebd384c4d6258bda7dfc8c754452d --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed.py @@ -0,0 +1,517 @@ +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import deepspeed +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.distributed.optim import ZeroRedundancyOptimizer +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import create_logger, set_seeds, create_exp_folder, model_resume, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model + + +def deepspeed_initialize(args, logger, model, opt, deepspeed_config): + logger.info(f"Initialize deepspeed...") + logger.info(f" Using deepspeed optimizer") + + def get_learning_rate_scheduler(warmup_min_lr, lr, warmup_num_steps, opt): + return WarmupLR(opt, warmup_min_lr, lr, warmup_num_steps) + + logger.info(f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + model, opt, _, scheduler = deepspeed.initialize( + model=model, + model_parameters=get_trainable_params(model), + config_params=deepspeed_config, + args=args, + lr_scheduler=partial(get_learning_rate_scheduler, args.warmup_min_lr, args.lr, args.warmup_num_steps) if args.warmup_num_steps > 0 else None, + ) + return model, opt, scheduler + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + model.module.module.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{save_name}") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + dist.barrier() + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/zero_dp_rank_*_tp_rank_00_pp_rank_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach() + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_(vae_scaling_factor) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + ) + + return latents, model_kwargs + +def main(args): + if args.training_parts == "lora": + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + dist.init_process_group("nccl") + world_size = dist.get_world_size() + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = world_size * batch_size * grad_accu_steps + + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.global_seed * world_size + rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={world_size}.") + deepspeed_config = deepspeed_config_from_args(args, global_batch_size) + + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + args_dict['world_size'] = world_size + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + assert args.deepspeed, f"Must enable deepspeed in this script: train_deepspeed.py" + with deepspeed.zero.Init(data_parallel_group=torch.distributed.group.WORLD, + remote_device=None if args.remote_device == 'none' else args.remote_device, + config_dict_or_path=deepspeed_config, + mpu=None, + enabled=args.zero_stage == 3): + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + logger.info(f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained(TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder(mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info(f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + logger.info(" Using deepspeed optimizer") + opt = None + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + if args.resume is not None or len(args.resume) > 0: + model, ema, start_epoch, start_epoch_step, train_steps = model_resume(args, model, ema, logger, len(loader)) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + model, opt, scheduler = deepspeed_initialize(args, logger, model, opt, deepspeed_config) + + # =========================================================================== + # Training + # =========================================================================== + + model.train() + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + dist.barrier() + + iters_per_epoch = len(loader) + logger.info(" ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info(f" Number parameters: {sum(p.numel() for p in model.parameters()):,}") + logger.info(f" Number trainable params: {sum(p.numel() for p in get_trainable_params(model)):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info(f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info(f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info(f" Training epochs: {start_epoch}/{args.epochs}") + logger.info(f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info(f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info(f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info(f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info(f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info(" *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info(f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + for batch in loader: + step += 1 + + latents, model_kwargs = prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) + + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses(model=model, x_start=latents, model_kwargs=model_kwargs) + loss = loss_dict["loss"].mean() + model.backward(loss) + last_batch_iteration = (train_steps + 1) // (global_batch_size // (batch_size * world_size)) + model.step(lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader)-1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + # =========================================================================== + # Log loss values: + # =========================================================================== + running_loss += loss.item() + log_steps += 1 + train_steps += 1 + if train_steps % args.log_every == 0: + # Measure training speed: + torch.cuda.synchronize() + end_time = time.time() + steps_per_sec = log_steps / (end_time - start_time) + # Reduce loss history over all processes: + avg_loss = torch.tensor(running_loss / log_steps, device=device) + dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + avg_loss = avg_loss.item() / world_size + # get lr from deepspeed fused optimizer + logger.info(f"(step={train_steps:07d}) " + + (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + f"Train Loss: {avg_loss:.4f}, " + f"Lr: {opt.param_groups[0]['lr']:.6g}, " + f"Steps/Sec: {steps_per_sec:.2f}, " + f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # Reset monitoring variables: + running_loss = 0 + log_steps = 0 + start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps + ) and train_steps > 0: + save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + dist.destroy_process_group() + + +if __name__ == "__main__": + # Start + main(get_args()) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..4df1b58acd793b21100ee98fc8557ef983b6b51c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/train_deepspeed_controlnet.py @@ -0,0 +1,584 @@ +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import deepspeed +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.distributed.optim import ZeroRedundancyOptimizer +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.controlnet import HunYuanControlNet +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import create_logger, set_seeds, create_exp_folder, model_resume, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model + +from hydit.annotator.dwpose import DWposeDetector +torch.optim.lr_scheduler.LRScheduler = torch.optim.lr_scheduler._LRScheduler +from transformers import pipeline +import cv2 +from PIL import Image + +depth_estimator = pipeline('depth-estimation', device='cuda:{}'.format(int(os.getenv('LOCAL_RANK', '0')))) +pose_detector = DWposeDetector() + +def deepspeed_initialize(args, logger, model, opt, deepspeed_config): + logger.info(f"Initialize deepspeed...") + logger.info(f" Using deepspeed optimizer") + + def get_learning_rate_scheduler(warmup_min_lr, lr, warmup_num_steps, opt): + return WarmupLR(opt, warmup_min_lr, lr, warmup_num_steps) + + logger.info(f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + model, opt, _, scheduler = deepspeed.initialize( + model=model, + model_parameters=get_trainable_params(model), + config_params=deepspeed_config, + args=args, + lr_scheduler=partial(get_learning_rate_scheduler, args.warmup_min_lr, args.lr, args.warmup_num_steps) if args.warmup_num_steps > 0 else None, + ) + return model, opt, scheduler + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + model.module.module.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{save_name}") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + dist.barrier() + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/*_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + +def get_canny(np_img, low_threshold = 100, high_threshold = 200): + # tensor = deNormalize() + # image = tensor_to_img(tensor) + image = cv2.Canny(np_img, low_threshold,high_threshold) + image = image[:,:,None] + image = np.concatenate([image,image,image], axis=2) + ## 输出边缘图像 + ## 归一化为张量 + # canny_tensor = img_to_norm_tensor(canny_img) + return image + +def get_depth(np_img): + pil_img = Image.fromarray(np_img) + depth = depth_estimator(pil_img)['depth'] + depth = np.array(depth) + depth = depth[:, :, None] + depth = np.concatenate([depth, depth, depth], axis=2) + return depth + +def get_pose(np_img): + return pose_detector(np_img)[0] + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach() + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + np_img = image.squeeze(0).add(1).mul(255 / 2).permute(1, 2, 0).cpu().numpy().astype('uint8') + if args.control_type == 'canny': + condition = get_canny(np_img) + elif args.control_type == 'depth': + condition = get_depth(np_img) + elif args.control_type == 'pose': + condition = get_pose(np_img) + else: + raise NotImplementedError + condtion = Image.fromarray(condition) + condition = TF.to_tensor(condition) + condition = TF.normalize(condition, [0.5], [0.5]) + condition = condition.unsqueeze(0).to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_(vae_scaling_factor) + condition = vae.encode(condition).latent_dist.sample().mul_(vae_scaling_factor) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + condition=condition, + ) + + return latents, model_kwargs + +def main(args): + if args.training_parts == "lora": + args.use_ema = False + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + dist.init_process_group("nccl") + world_size = dist.get_world_size() + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = world_size * batch_size * grad_accu_steps + + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.global_seed * world_size + rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={world_size}.") + deepspeed_config = deepspeed_config_from_args(args, global_batch_size) + + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + args_dict['world_size'] = world_size + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + assert args.deepspeed, f"Must enable deepspeed in this script: train_deepspeed.py" + with deepspeed.zero.Init(data_parallel_group=torch.distributed.group.WORLD, + remote_device=None if args.remote_device == 'none' else args.remote_device, + config_dict_or_path=deepspeed_config, + mpu=None, + enabled=args.zero_stage == 3): + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + controlnet = HunYuanControlNet(args, + input_size=latent_size, + depth=40, hidden_size=1408, patch_size=2, num_heads=16, mlp_ratio=4.3637, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + controlnet = Float16Module(controlnet, args) + logger.info(f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained(TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder(mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info(f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + logger.info(" Using deepspeed optimizer") + opt = None + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + if args.resume is not None or len(args.resume) > 0: + model, ema, start_epoch, start_epoch_step, train_steps = model_resume(args, model, ema, logger, len(loader)) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + if args.use_fp16: + controlnet.module.from_dit(model.module) + controlnet.module.set_trainable() + else: + controlnet.from_dit(model) + controlnet.set_trainable() + logger.info(f" ControlNet loaded from DIT") + + + + controlnet, opt, scheduler = deepspeed_initialize(args, logger, controlnet, opt, deepspeed_config) + + # =========================================================================== + # Training + # =========================================================================== + + model.eval() + model.requires_grad_(False) + model = model.to(device) + + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + dist.barrier() + + iters_per_epoch = len(loader) + logger.info(" ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info(f" Number parameters: {sum(p.numel() for p in controlnet.parameters()):,}") + logger.info(f" Number trainable params: {sum(p.numel() for p in get_trainable_params(controlnet)):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info(f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info(f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info(f" Training epochs: {start_epoch}/{args.epochs}") + logger.info(f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info(f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info(f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info(f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info(f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info(" *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info(f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + for batch in loader: + step += 1 + + latents, model_kwargs = prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) + + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses(model=model, x_start=latents, model_kwargs=model_kwargs, controlnet=controlnet) + loss = loss_dict["loss"].mean() + controlnet.backward(loss) + last_batch_iteration = (train_steps + 1) // (global_batch_size // (batch_size * world_size)) + controlnet.step(lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader)-1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + # =========================================================================== + # Log loss values: + # =========================================================================== + running_loss += loss.item() + log_steps += 1 + train_steps += 1 + if train_steps % args.log_every == 0: + # Measure training speed: + torch.cuda.synchronize() + end_time = time.time() + steps_per_sec = log_steps / (end_time - start_time) + # Reduce loss history over all processes: + avg_loss = torch.tensor(running_loss / log_steps, device=device) + dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + avg_loss = avg_loss.item() / world_size + # get lr from deepspeed fused optimizer + logger.info(f"(step={train_steps:07d}) " + + (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + f"Train Loss: {avg_loss:.4f}, " + f"Lr: {opt.param_groups[0]['lr']:.6g}, " + f"Steps/Sec: {steps_per_sec:.2f}, " + f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # Reset monitoring variables: + running_loss = 0 + log_steps = 0 + start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps + ) and train_steps > 0: + save_checkpoint(args, rank, logger, controlnet, ema, epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + dist.destroy_process_group() + + +if __name__ == "__main__": + # Start + main(get_args()) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/utils/tools.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/utils/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..71d829227fbd6c9ee3d0f11ea873a5ac7abd70e1 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/hydit/utils/tools.py @@ -0,0 +1,190 @@ +import random +import logging +from pathlib import Path +import shutil + +import numpy as np +from PIL import Image +import torch +import torch.distributed as dist +from tqdm.auto import tqdm +import math +import torch.nn.functional as F +import os + +def get_trainable_params(model): + params = model.parameters() + params = [p for p in params if p.requires_grad] + return params + + +def set_seeds(seed_list, device=None): + if isinstance(seed_list, (tuple, list)): + seed = sum(seed_list) + else: + seed = seed_list + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + return torch.Generator(device).manual_seed(seed) + +def get_start_epoch(resume_path, ckpt, steps_per_epoch): + if 'epoch' in ckpt: + start_epoch = ckpt['epoch'] + else: + start_epoch = 0 + if 'steps' in ckpt: + train_steps = ckpt['steps'] + else: + try: + train_steps = int(Path(resume_path).stem) + except: + train_steps = start_epoch * steps_per_epoch + + start_epoch_step = train_steps % steps_per_epoch + 1 + return start_epoch, start_epoch_step, train_steps + +def assert_shape(*args): + if len(args) < 2: + return + cond = True + fail_str = f"{args[0] if isinstance(args[0], (list, tuple)) else args[0].shape}" + for i in range(1, len(args)): + shape1 = args[i] if isinstance(args[i], (list, tuple)) else args[i].shape + shape2 = args[i - 1] if isinstance(args[i - 1], (list, tuple)) else args[i - 1].shape + cond = cond and (shape1 == shape2) + fail_str += f" vs {args[i] if isinstance(args[i], (list, tuple)) else args[i].shape}" + assert cond, fail_str + + +def create_logger(logging_dir=None, logging_file=None, ddp=True): + """ + Create a logger that writes to a log file and stdout. + """ + if not ddp or (ddp and dist.get_rank() == 0): # real logger + if logging_file is not None: + file_handler = [logging.FileHandler(logging_file)] + elif logging_dir is not None: + file_handler = [logging.FileHandler(f"{logging_dir}/log.txt")] + else: + file_handler = [] + logging.basicConfig( + level=logging.INFO, + format='[\033[34m%(asctime)s\033[0m] %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler()] + file_handler + ) + logger = logging.getLogger(__name__) + else: + logger = logging.getLogger(__name__) + logger.addHandler(logging.NullHandler()) + return logger + +def create_exp_folder(args, rank): + if rank == 0: + os.makedirs(args.results_dir, exist_ok=True) + existed_experiments = list(Path(args.results_dir).glob("*dit*")) + if len(existed_experiments) == 0: + experiment_index = 1 + else: + existed_experiments.sort() + print('existed_experiments', existed_experiments) + experiment_index = max([int(x.stem.split('-')[0]) for x in existed_experiments]) + 1 + dist.barrier() + model_string_name = args.task_flag if args.task_flag else args.model.replace("/", "-") + experiment_dir = f"{args.results_dir}/{experiment_index:03d}-{model_string_name}" # Create an experiment folder + checkpoint_dir = f"{experiment_dir}/checkpoints" # Stores saved model checkpoints + if rank == 0: + os.makedirs(checkpoint_dir, exist_ok=True) + logger = create_logger(experiment_dir) + logger.info(f"Experiment directory created at {experiment_dir}") + else: + logger = create_logger() + experiment_dir = "" + + return experiment_dir, checkpoint_dir, logger + + +def model_resume(args, model, ema, logger, len_loader): + """ + Load pretrained weights. + """ + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + resume_path = args.resume + if not Path(resume_path).exists(): + raise FileNotFoundError(f" Cannot find checkpoint from {resume_path}") + + logger.info(f" Resume deepspeed={args.resume_deepspeed}, " + f"Resume split={args.resume_split}, " + f"Resume from checkpoint {resume_path}") + # Resume model and ema states (not include optimizer states) from a checkpoint saved by Deepspeed version of DIT. + if args.resume_deepspeed: + assert 'mp_rank_00_model_states.pt' in os.listdir(resume_path), f' Cannot find dp chkpt from {resume_path}' + resume_ckpt = torch.load(os.path.join(resume_path, 'mp_rank_00_model_states.pt'), + map_location=lambda storage, loc: storage) + # Resume main model + if args.ema_to_module: + logger.info(" Resume main model from the ema states.") + model.load_state_dict(resume_ckpt['ema'], strict=args.strict) + else: + logger.info(" Resume main model from the main states.") + model.load_state_dict(resume_ckpt['module'], strict=args.strict) + # Resume EMA model + if args.use_ema: + if args.module_to_ema: + logger.info(" Resume EMA model from the main states.") + ema.load_state_dict(resume_ckpt['module'], strict=args.strict) + else: + logger.info(" Resume EMA model from the EMA states.") + ema.load_state_dict(resume_ckpt['ema'], strict=args.strict) + if not args.reset_loader: + start_epoch, start_epoch_step, train_steps = get_start_epoch(args.resume, resume_ckpt, len_loader) + # Resume model and ema states (not include optimizer states) from two checkpoints separated from DeepSpeed ckpt. + elif args.resume_split: + # Resume main model + if args.ema_to_module: + assert 'pytorch_model_ema.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_ema.pt from {resume_path}' + logger.info(f" Resume main model from ema states.") + resume_ckpt_ema = torch.load(os.path.join(resume_path, 'pytorch_model_ema.pt'), + map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + assert 'pytorch_model_module.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_module.pt from {resume_path}' + logger.info(f" Resume main model from main states.") + resume_ckpt_module = torch.load(os.path.join(resume_path, 'pytorch_model_module.pt'), + map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_module, strict=args.strict) + # Resume ema model + if args.use_ema: + if args.module_to_ema: + if "resume_ckpt_module" in locals(): + logger.info(f" Resume ema model from main states.") + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + assert 'pytorch_model_module.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_module.pt from {resume_path}' + logger.info(f" Resume ema model from module states.") + resume_ckpt_module = torch.load(os.path.join(resume_path, 'pytorch_model_module.pt'), + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + if "resume_ckpt_ema" in locals(): + logger.info(f" Resume ema model from EMA states.") + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + assert 'pytorch_model_ema.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_ema.pt from {resume_path}' + logger.info(f" Resume ema model from EMA states.") + resume_ckpt_ema = torch.load(os.path.join(resume_path, 'pytorch_model_ema.pt'), + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + raise ValueError(" “If `resume` is True, then either `resume_split` or `resume_deepspeed` must be true.”") + + return model, ema, start_epoch, start_epoch_step, train_steps \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/nodes.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..18518a84c58ad877bb25f96e7a75219df8ceb44a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/nodes.py @@ -0,0 +1,443 @@ +import copy +import os +import torch +from .utils import convert_images_to_tensors +from comfy.model_management import get_torch_device +import folder_paths +from .hydit.diffusion.pipeline import StableDiffusionPipeline +from .hydit.diffusion.pipeline_controlnet import StableDiffusionControlNetPipeline +from .hydit.config_comfyui import get_args +from .hydit.inference_comfyui import End2End +from pathlib import Path +from .hydit.constants import SAMPLER_FACTORY +from diffusers import schedulers +from .constant import HUNYUAN_PATH, SCHEDULERS_hunyuan, T5_PATH, LORA_PATH +from .dit import load_dit, load_checkpoint, load_vae +from .clip import CLIP +from .hydit.modules.controlnet import HunYuanControlNet +from .hydit.modules.models import HUNYUAN_DIT_CONFIG +from loguru import logger +import numpy as np +from torchvision import transforms as T + +norm_transform = T.Compose( + [ + T.ToTensor(), + T.Normalize([0.5], [0.5]), + ] +) + +from PIL import Image + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +class DiffusersPipelineLoader_old: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": {"pipeline_folder_name": (os.listdir(HUNYUAN_PATH),), + "model_name": (["disable"] + folder_paths.get_filename_list("checkpoints"),), + "vae_name": (["disable"] + folder_paths.get_filename_list("vae"),), + "backend": (["ksampler", "diffusers"],), }} + + RETURN_TYPES = ("PIPELINE", "MODEL", "CLIP", "VAE") + + FUNCTION = "create_pipeline" + + CATEGORY = "Diffusers" + + def create_pipeline(self, pipeline_folder_name, model_name, vae_name, backend): + if model_name != "disable": + MODEL_PATH = folder_paths.get_full_path("checkpoints", model_name) + else: + MODEL_PATH = None + if vae_name != "disable": + VAE_PATH = folder_paths.get_full_path("vae", vae_name) + else: + VAE_PATH = None + + if backend == "diffusers": + args_hunyuan = get_args() + gen = End2End(args_hunyuan[0], Path(os.path.join(HUNYUAN_PATH, pipeline_folder_name)), MODEL_PATH, VAE_PATH) + return (gen, None, None, None) + elif backend == "ksampler": + out = load_dit(model_path=os.path.join(HUNYUAN_PATH, pipeline_folder_name), MODEL_PATH=MODEL_PATH, + VAE_PATH=VAE_PATH) + return (None,) + out[:3] + + +class DiffusersPipelineLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": {"pipeline_folder_name": (os.listdir(HUNYUAN_PATH),), }, + "optional": {"lora": ("lora_path",), }} + + RETURN_TYPES = ("PIPELINE",) + + FUNCTION = "create_pipeline" + + CATEGORY = "Diffusers" + + def create_pipeline(self, pipeline_folder_name, lora=None): + if lora != None: + LORA_PATH = lora + else: + LORA_PATH = None + + args_hunyuan = get_args() + gen = End2End(args_hunyuan[0], Path(os.path.join(HUNYUAN_PATH, pipeline_folder_name)), LOAR_PATH=LORA_PATH) + return (gen,) + + +class DiffusersCheckpointLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "model_name": (folder_paths.get_filename_list("checkpoints"),), }} + + RETURN_TYPES = ("MODEL",) + + FUNCTION = "load_checkpoint" + + CATEGORY = "Diffusers" + + def load_checkpoint(self, model_name): + MODEL_PATH = folder_paths.get_full_path("checkpoints", model_name) + out = load_checkpoint(MODEL_PATH) + return out + + +class DiffusersVAELoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "model_name": (folder_paths.get_filename_list("vae"),), }} + + RETURN_TYPES = ("VAE",) + + FUNCTION = "load_vae" + + CATEGORY = "Diffusers" + + def load_vae(self, model_name): + MODEL_PATH = folder_paths.get_full_path("vae", model_name) + out = load_vae(MODEL_PATH) + return out + + +class DiffusersLoraLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": {"lora_name": (os.listdir(LORA_PATH),), }} + + RETURN_TYPES = ("lora_path",) + + FUNCTION = "load_lora" + + CATEGORY = "Diffusers" + + def load_lora(self, lora_name): + MODEL_PATH = os.path.join(LORA_PATH, lora_name) + return (MODEL_PATH,) + + +class DiffusersCLIPLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "text_encoder_path": (folder_paths.get_filename_list("clip"),), + "t5_text_encoder_path": (os.listdir(T5_PATH),), }} + + RETURN_TYPES = ("CLIP",) + + FUNCTION = "load_clip" + + CATEGORY = "Diffusers" + + def load_clip(self, text_encoder_path, t5_text_encoder_path): + CLIP_PATH = folder_paths.get_full_path("clip", text_encoder_path) + t5_file = os.path.join(T5_PATH, t5_text_encoder_path) + root = None + out = CLIP(root, CLIP_PATH, t5_file) + return (out,) + + +class DiffusersControlNetLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + self.torch_device = get_torch_device() + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "controlnet_path": (folder_paths.get_filename_list("controlnet"),), }} + + RETURN_TYPES = ("CTRL",) + + FUNCTION = "load_controlnet" + + CATEGORY = "Diffusers" + + def load_controlnet(self, controlnet_path): + DiffusersControlNetLoader_PATH = folder_paths.get_full_path("controlnet", controlnet_path) + args_hunyuan = get_args() + args = args_hunyuan[0] + image_size = _to_tuple(args.image_size) + latent_size = (image_size[0] // 8, image_size[1] // 8) + model_config = HUNYUAN_DIT_CONFIG[args.model] + controlnet = HunYuanControlNet(args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.torch_device) + controlnet_state_dict = torch.load(DiffusersControlNetLoader_PATH) + controlnet.load_state_dict(controlnet_state_dict) + controlnet.eval() + return (controlnet,) + + +class DiffusersSchedulerLoader: + def __init__(self): + self.tmp_dir = folder_paths.get_temp_directory() + self.dtype = torch.float32 + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "scheduler_name": (list(SCHEDULERS_hunyuan),), + } + } + + RETURN_TYPES = ("SCHEDULER",) + + FUNCTION = "load_scheduler" + + CATEGORY = "Diffusers" + + def load_scheduler(self, scheduler_name): + # Load sampler from factory + kwargs = SAMPLER_FACTORY[scheduler_name]['kwargs'] + scheduler = SAMPLER_FACTORY[scheduler_name]['scheduler'] + args_hunyuan = get_args() + args_hunyuan = args_hunyuan[0] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args_hunyuan.noise_schedule + kwargs['beta_start'] = args_hunyuan.beta_start + kwargs['beta_end'] = args_hunyuan.beta_end + kwargs['prediction_type'] = args_hunyuan.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + return (scheduler,) + + +class DiffusersModelMakeup: + def __init__(self): + self.torch_device = get_torch_device() + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "pipeline": ("PIPELINE",), + "scheduler": ("SCHEDULER",), + }, + "optional": + {"controlnet": ("CTRL",), } + } + + RETURN_TYPES = ("MAKED_PIPELINE",) + + FUNCTION = "makeup_pipeline" + + CATEGORY = "Diffusers" + + def makeup_pipeline(self, pipeline, scheduler, controlnet=None): + progress_bar_config = {} + if not controlnet: + pipe = StableDiffusionPipeline(vae=pipeline.vae, + text_encoder=pipeline.clip_text_encoder, + tokenizer=pipeline.tokenizer, + unet=pipeline.model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=pipeline.embedder_t5, + infer_mode=pipeline.infer_mode, + ) + else: + pipe = StableDiffusionControlNetPipeline(vae=pipeline.vae, + text_encoder=pipeline.clip_text_encoder, + tokenizer=pipeline.tokenizer, + unet=pipeline.model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=pipeline.embedder_t5, + infer_mode=pipeline.infer_mode, + controlnet=controlnet + ) + pipe = pipe.to(pipeline.device) + pipeline.pipeline = pipe + return (pipeline,) + + +class DiffusersClipTextEncode: + @classmethod + def INPUT_TYPES(s): + return {"required": { + "positive": ("STRING", {"multiline": True}), + "negative": ("STRING", {"multiline": True}), + }} + + RETURN_TYPES = ("STRINGC", "STRINGC",) + RETURN_NAMES = ("positive", "negative",) + + FUNCTION = "concat_embeds" + + CATEGORY = "Diffusers" + + def concat_embeds(self, positive, negative): + return (positive, negative,) + + +class DiffusersSampler: + def __init__(self): + self.torch_device = get_torch_device() + + @classmethod + def INPUT_TYPES(s): + return {"required": { + "maked_pipeline": ("MAKED_PIPELINE",), + "positive": ("STRINGC",), + "negative": ("STRINGC",), + "batch_size": ("INT", {"default": 1, "min": 1, "max": 128, "step": 1}), + "width": ("INT", {"default": 1024, "min": 1, "max": 8192, "step": 1}), + "height": ("INT", {"default": 1024, "min": 1, "max": 8192, "step": 1}), + "steps": ("INT", {"default": 20, "min": 1, "max": 10000}), + "control_weight": ("FLOAT", {"default": 0.7, "min": 0, "max": 1.0}), + "cfg": ("FLOAT", {"default": 8.0, "min": 0.0, "max": 100.0, "step": 0.1, "round": 0.01}), + "seed": ("INT", {"default": 0, "min": 0, "max": 2 ** 32 - 1}), + }, + "optional": {"image": ("IMAGE",), }} + + RETURN_TYPES = ("IMAGE",) + + FUNCTION = "sample" + + CATEGORY = "Diffusers" + + def sample(self, maked_pipeline, positive, negative, batch_size, width, height, steps, cfg, seed, control_weight, + image=None): + if not isinstance(image, torch.Tensor): + results = maked_pipeline.predict(positive, + height=height, + width=width, + seed=int(seed), + enhanced_prompt=None, + negative_prompt=negative, + infer_steps=steps, + guidance_scale=cfg, + batch_size=batch_size, + src_size_cond=[height, width], + image=image + ) + else: + # import pdb + # pdb.set_trace() + img = 255. * image.cpu().numpy() + # print(i.size()) + img = Image.fromarray(np.clip(img[0], 0, 255).astype(np.uint8)) + img = img.convert('RGB').resize((height, width)) + image = norm_transform(img) + image = image.unsqueeze(0).cuda() + # print(image.size) + results = maked_pipeline.predict(positive, + height=height, + width=width, + seed=int(seed), + enhanced_prompt=None, + negative_prompt=negative, + infer_steps=steps, + guidance_scale=cfg, + batch_size=batch_size, + src_size_cond=[height, width], + image=image, + control_weight=control_weight, + ) + + images = results['images'] + return (convert_images_to_tensors(images),) + + +NODE_CLASS_MAPPINGS = { + "DiffusersPipelineLoader": DiffusersPipelineLoader, + "DiffusersSchedulerLoader": DiffusersSchedulerLoader, + "DiffusersModelMakeup": DiffusersModelMakeup, + "DiffusersClipTextEncode": DiffusersClipTextEncode, + "DiffusersSampler": DiffusersSampler, + "DiffusersCheckpointLoader": DiffusersCheckpointLoader, + "DiffusersVAELoader": DiffusersVAELoader, + "DiffusersCLIPLoader": DiffusersCLIPLoader, + "DiffusersControlNetLoader": DiffusersControlNetLoader, + "DiffusersLoraLoader": DiffusersLoraLoader +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "DiffusersPipelineLoader": "HunYuan Pipeline Loader", + "DiffusersSchedulerLoader": "HunYuan Scheduler Loader", + "DiffusersModelMakeup": "HunYuan Model Makeup", + "DiffusersClipTextEncode": "HunYuan Clip Text Encode", + "DiffusersSampler": "HunYuan Sampler", + "DiffusersCheckpointLoader": "HunYuan Checkpoint Loader", + "DiffusersVAELoader": "HunYuan VAE Loader", + "DiffusersCLIPLoader": "HunYuan CLIP Loader", + "DiffusersControlNetLoader": "HunYuan ControlNet Loader", + "DiffusersLoraLoader": "HunYuan Lora Loader", +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/requirements.txt b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7deb29fb02ee81b382eb2f47e40036c8e7903bec --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/requirements.txt @@ -0,0 +1,15 @@ +--extra-index-url https://pypi.ngc.nvidia.com +--extra-index-url https://download.pytorch.org/whl/cu117 +timm==1.0.3 +diffusers==0.21.2 +peft==0.10.0 +protobuf==5.27.0 +accelerate==0.30.1 +loguru==0.7.2 +sentencepiece==0.2.0 +cuda-python==12.5.0 +polygraphy==0.49.9 +pandas==2.2.2 +omegaconf==2.3.0 +xformers==0.0.20 +pytorch-lightning==2.2.5 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/supported_dit_models.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/supported_dit_models.py new file mode 100644 index 0000000000000000000000000000000000000000..1d3e77e322b602c4119c870d55503a6d2a476658 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/supported_dit_models.py @@ -0,0 +1,95 @@ +import comfy.supported_models_base +import comfy.latent_formats +import comfy.model_patcher +import comfy.model_base +import comfy.utils +import torch +from collections import namedtuple +from .hydit.modules.models_comfyui import HunYuanDiT as HYDiT +import pdb + +def batch_embeddings(embeds, batch_size): + bs_embed, seq_len, _ = embeds.shape + embeds = embeds.repeat(1, batch_size, 1) + embeds = embeds.view(bs_embed * batch_size, seq_len, -1) + return embeds + +class HunYuan_DiT(comfy.supported_models_base.BASE): + Conf = namedtuple('DiT', ['learn_sigma', 'text_states_dim', 'text_states_dim_t5', 'text_len', 'text_len_t5', 'norm', 'infer_mode', 'use_fp16']) + conf = { + 'learn_sigma': True, + 'text_states_dim': 1024, + 'text_states_dim_t5': 2048, + 'text_len': 77, + 'text_len_t5': 256, + 'norm': 'layer', + 'infer_mode': 'torch', + 'use_fp16': True + } + + unet_config = {} + unet_extra_config = { + "num_heads": 16 + } + latent_format = comfy.latent_formats.SDXL + + dit_conf = Conf(**conf) + + def __init__(self, model_conf): + self.unet_config = model_conf.get("unet_config", {}) + #print(model_conf) + print(self.unet_config) + self.sampling_settings = model_conf.get("sampling_settings", {}) + self.latent_format = self.latent_format() + self.unet_config["disable_unet_model_creation"] = True + #self.unet_config["disable_unet_model_creation"] = self.unet_config.get("disable_unet_model_creation", True) + + def model_type(self, state_dict, prefix=""): + return comfy.model_base.ModelType.V_PREDICTION + +class HYDiT_Model(comfy.model_base.BaseModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def extra_conds(self, **kwargs): + out = super().extra_conds(**kwargs) + addit_embeds = kwargs['cross_attn'].addit_embeds + for name in addit_embeds: + out[name] = comfy.conds.CONDRegular(addit_embeds[name]) + + return out + +class ModifiedHunYuanDiT(HYDiT): + def forward_core(self, *args, **kwargs): + return super().forward(*args, **kwargs) + + def forward(self, x, timesteps, context, t5_embeds=None, attention_mask=None, t5_attention_mask=None, image_meta_size=None, **kwargs): + batch_size, _, height, width = x.shape + #assert(0) + #pdb.set_trace() + + + style = torch.as_tensor([0, 0] * (batch_size//2), device=x.device) + src_size_cond = (height//2*16, width//2*16) + size_cond = list(src_size_cond) + [width*8, height*8, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * batch_size, device=x.device) + rope = self.calc_rope(*src_size_cond) + + + noise_pred = self.forward_core( + x = x.to(self.dtype), + t = timesteps.to(self.dtype), + encoder_hidden_states = context.to(self.dtype), + text_embedding_mask = attention_mask.to(self.dtype), + encoder_hidden_states_t5 = t5_embeds.to(self.dtype), + text_embedding_mask_t5 = t5_attention_mask.to(self.dtype), + image_meta_size = image_meta_size.to(self.dtype), + style = style, + cos_cis_img = rope[0], + sin_cis_img = rope[1], + return_dict=False + ) + noise_pred = noise_pred.to(torch.float) + eps, _ = noise_pred[:, :self.in_channels], noise_pred[:, self.in_channels:] + return eps + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tenc.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tenc.py new file mode 100644 index 0000000000000000000000000000000000000000..a3af09845e4c0de5da07208ba9b3c4c7891d1996 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tenc.py @@ -0,0 +1,181 @@ +# This is for loading the CLIP (bert?) + mT5 encoder for HunYuanDiT +import os +import torch +from transformers import AutoTokenizer, modeling_utils +from transformers import T5Config, T5EncoderModel, BertConfig, BertModel + +from comfy import model_management +import comfy.model_patcher +import comfy.utils + +class mT5Model(torch.nn.Module): + def __init__(self, textmodel_json_config=None, device="cpu", max_length=256, freeze=True, dtype=None): + super().__init__() + self.device = device + self.dtype = dtype + self.max_length = max_length + if textmodel_json_config is None: + textmodel_json_config = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + f"config_mt5.json" + ) + config = T5Config.from_json_file(textmodel_json_config) + with modeling_utils.no_init_weights(): + self.transformer = T5EncoderModel(config) + self.to(dtype) + if freeze: + self.freeze() + + def freeze(self): + self.transformer = self.transformer.eval() + for param in self.parameters(): + param.requires_grad = False + + def load_sd(self, sd): + return self.transformer.load_state_dict(sd, strict=False) + + def to(self, *args, **kwargs): + return self.transformer.to(*args, **kwargs) + +class hyCLIPModel(torch.nn.Module): + def __init__(self, textmodel_json_config=None, device="cpu", max_length=77, freeze=True, dtype=None): + super().__init__() + self.device = device + self.dtype = dtype + self.max_length = max_length + if textmodel_json_config is None: + textmodel_json_config = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + f"config_clip.json" + ) + config = BertConfig.from_json_file(textmodel_json_config) + with modeling_utils.no_init_weights(): + self.transformer = BertModel(config) + self.to(dtype) + if freeze: + self.freeze() + + def freeze(self): + self.transformer = self.transformer.eval() + for param in self.parameters(): + param.requires_grad = False + + def load_sd(self, sd): + return self.transformer.load_state_dict(sd, strict=False) + + def to(self, *args, **kwargs): + return self.transformer.to(*args, **kwargs) + +class EXM_HyDiT_Tenc_Temp: + def __init__(self, no_init=False, device="cpu", dtype=None, model_class="mT5", *kwargs): + if no_init: + return + + size = 8 if model_class == "mT5" else 2 + if dtype == torch.float32: + size *= 2 + size *= (1024**3) + + if device == "auto": + self.load_device = model_management.text_encoder_device() + self.offload_device = model_management.text_encoder_offload_device() + self.init_device = "cpu" + elif device == "cpu": + size = 0 # doesn't matter + self.load_device = "cpu" + self.offload_device = "cpu" + self.init_device="cpu" + elif device.startswith("cuda"): + print("Direct CUDA device override!\nVRAM will not be freed by default.") + size = 0 # not used + self.load_device = device + self.offload_device = device + self.init_device = device + else: + self.load_device = model_management.get_torch_device() + self.offload_device = "cpu" + self.init_device="cpu" + + self.dtype = dtype + self.device = self.load_device + if model_class == "mT5": + self.cond_stage_model = mT5Model( + device = self.load_device, + dtype = self.dtype, + ) + #tokenizer_args = {"subfolder": "t2i/mt5"} # web + tokenizer_path = os.path.join( # local + os.path.dirname(os.path.realpath(__file__)), + "tokenizer_mt5", + ) + else: + self.cond_stage_model = hyCLIPModel( + device = self.load_device, + dtype = self.dtype, + ) + #tokenizer_args = {"subfolder": "t2i/tokenizer",} # web + tokenizer_path = os.path.join( # local + os.path.dirname(os.path.realpath(__file__)), + "tokenizer_clip", + ) + # self.tokenizer = AutoTokenizer.from_pretrained( + # "Tencent-Hunyuan/HunyuanDiT", + # **tokenizer_args + # ) + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + self.patcher = comfy.model_patcher.ModelPatcher( + self.cond_stage_model, + load_device = self.load_device, + offload_device = self.offload_device, + current_device = self.load_device, + size = size, + ) + + def clone(self): + n = EXM_HyDiT_Tenc_Temp(no_init=True) + n.patcher = self.patcher.clone() + n.cond_stage_model = self.cond_stage_model + n.tokenizer = self.tokenizer + return n + + def load_sd(self, sd): + return self.cond_stage_model.load_sd(sd) + + def get_sd(self): + return self.cond_stage_model.state_dict() + + def load_model(self): + if self.load_device != "cpu": + model_management.load_model_gpu(self.patcher) + return self.patcher + + def add_patches(self, patches, strength_patch=1.0, strength_model=1.0): + return self.patcher.add_patches(patches, strength_patch, strength_model) + + def get_key_patches(self): + return self.patcher.get_key_patches() + +def load_clip(model_path, **kwargs): + model = EXM_HyDiT_Tenc_Temp(model_class="clip", **kwargs) + sd = comfy.utils.load_torch_file(model_path) + + prefix = "bert." + state_dict = {} + for key in sd: + nkey = key + if key.startswith(prefix): + nkey = key[len(prefix):] + state_dict[nkey] = sd[key] + + m, e = model.load_sd(state_dict) + if len(m) > 0 or len(e) > 0: + print(f"HYDiT: clip missing {len(m)} keys ({len(e)} extra)") + return model + +def load_t5(model_path, **kwargs): + model = EXM_HyDiT_Tenc_Temp(model_class="mT5", **kwargs) + sd = comfy.utils.load_torch_file(model_path) + m, e = model.load_sd(sd) + if len(m) > 0 or len(e) > 0: + print(f"HYDiT: mT5 missing {len(m)} keys ({len(e)} extra)") + return model diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/special_tokens_map.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a8b3208c2884c4efb86e49300fdd3dc877220cdf --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/special_tokens_map.json @@ -0,0 +1,7 @@ +{ + "cls_token": "[CLS]", + "mask_token": "[MASK]", + "pad_token": "[PAD]", + "sep_token": "[SEP]", + "unk_token": "[UNK]" +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/tokenizer_config.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..a14356073e11a885074a7cdbddc749463cefd911 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/tokenizer_config.json @@ -0,0 +1,16 @@ +{ + "cls_token": "[CLS]", + "do_basic_tokenize": true, + "do_lower_case": true, + "mask_token": "[MASK]", + "name_or_path": "hfl/chinese-roberta-wwm-ext", + "never_split": null, + "pad_token": "[PAD]", + "sep_token": "[SEP]", + "special_tokens_map_file": "/home/chenweifeng/.cache/huggingface/hub/models--hfl--chinese-roberta-wwm-ext/snapshots/5c58d0b8ec1d9014354d691c538661bf00bfdb44/special_tokens_map.json", + "strip_accents": null, + "tokenize_chinese_chars": true, + "tokenizer_class": "BertTokenizer", + "unk_token": "[UNK]", + "model_max_length": 77 +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab.txt b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab.txt new file mode 100644 index 0000000000000000000000000000000000000000..6246906805d02aca01714c71e4c8d77b69a7a131 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab.txt @@ -0,0 +1,47020 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 +[unused0] +[unused100] +[unused101] +[unused102] +[unused103] +[unused104] +[unused105] +[unused106] +[unused107] +[unused108] +[unused109] +[unused110] +[unused111] +[unused112] +[unused113] +[unused114] +[unused115] +[unused116] +[unused117] +[unused118] +[unused119] +[unused120] +[unused121] +[unused122] +[unused123] +[unused124] +[unused125] +[unused126] +[unused127] +[unused128] +[unused129] +[unused130] +[unused131] +[unused132] +[unused133] +[unused134] +[unused135] +[unused136] +[unused137] +[unused138] +[unused139] +[unused140] +[unused141] +[unused142] +[unused143] +[unused144] +[unused145] +[unused146] +[unused147] +[unused148] +[unused149] +[unused150] +[unused151] +[unused152] +[unused153] +[unused154] +[unused155] +[unused156] +[unused157] +[unused158] +[unused159] +[unused160] +[unused161] +[unused162] +[unused163] +[unused164] +[unused165] +[unused166] +[unused167] +[unused168] +[unused169] +[unused170] +[unused171] +[unused172] +[unused173] +[unused174] +[unused175] +[unused176] +[unused177] +[unused178] +[unused179] +[unused180] +[unused181] +[unused182] +[unused183] +[unused184] +[unused185] +[unused186] +[unused187] +[unused188] +[unused189] +[unused190] +[unused191] +[unused192] +[unused193] +[unused194] +[unused195] +[unused196] +[unused197] +[unused198] +[unused199] +[unused200] +[unused201] +[unused202] +[unused203] +[unused204] +[unused205] +[unused206] +[unused207] +[unused208] +[unused209] +[unused210] +[unused211] +[unused212] +[unused213] +[unused214] +[unused215] +[unused216] +[unused217] +[unused218] +[unused219] +[unused220] +[unused221] +[unused222] +[unused223] +[unused224] +[unused225] +[unused226] +[unused227] +[unused228] +[unused229] +[unused230] +[unused231] +[unused232] +[unused233] +[unused234] +[unused235] +[unused236] +[unused237] +[unused238] +[unused239] +[unused240] +[unused241] +[unused242] +[unused243] +[unused244] +[unused245] +[unused246] +[unused247] +[unused248] +[unused249] +[unused250] +[unused251] +[unused252] +[unused253] +[unused254] +[unused255] +[unused256] +[unused257] +[unused258] +[unused259] +[unused260] +[unused261] +[unused262] +[unused263] +[unused264] +[unused265] +[unused266] +[unused267] +[unused268] +[unused269] +[unused270] +[unused271] +[unused272] +[unused273] +[unused274] +[unused275] +[unused276] +[unused277] +[unused278] +[unused279] +[unused280] +[unused281] +[unused282] +[unused283] +[unused284] +[unused285] +[unused286] +[unused287] +[unused288] +[unused289] +[unused290] +[unused291] +[unused292] +[unused293] +[unused294] +[unused295] +[unused296] +[unused297] +[unused298] +[unused299] +[unused300] +[unused301] +[unused302] +[unused303] +[unused304] +[unused305] +[unused306] +[unused307] +[unused308] +[unused309] +[unused310] +[unused311] +[unused312] +[unused313] +[unused314] +[unused315] +[unused316] +[unused317] +[unused318] +[unused319] +[unused320] +[unused321] +[unused322] +[unused323] +[unused324] +[unused325] +[unused326] +[unused327] +[unused328] +[unused329] +[unused330] +[unused331] +[unused332] +[unused333] +[unused334] +[unused335] +[unused336] +[unused337] +[unused338] +[unused339] +[unused340] +[unused341] +[unused342] +[unused343] +[unused344] +[unused345] +[unused346] +[unused347] +[unused348] +[unused349] +[unused350] +[unused351] +[unused352] +[unused353] +[unused354] +[unused355] +[unused356] +[unused357] +[unused358] +[unused359] +[unused360] +[unused361] +[unused362] +[unused363] +[unused364] +[unused365] +[unused366] +[unused367] +[unused368] +[unused369] +[unused370] +[unused371] +[unused372] +[unused373] +[unused374] +[unused375] +[unused376] +[unused377] +[unused378] +[unused379] +[unused380] +[unused381] +[unused382] +[unused383] +[unused384] +[unused385] +[unused386] +[unused387] +[unused388] +[unused389] +[unused390] +[unused391] +[unused392] +[unused393] +[unused394] +[unused395] +[unused396] +[unused397] +[unused398] +[unused399] +[unused400] +[unused401] +[unused402] +[unused403] +[unused404] +[unused405] +[unused406] +[unused407] +[unused408] +[unused409] +[unused410] +[unused411] +[unused412] +[unused413] +[unused414] +[unused415] +[unused416] +[unused417] +[unused418] +[unused419] +[unused420] +[unused421] +[unused422] +[unused423] +[unused424] +[unused425] +[unused426] +[unused427] +[unused428] +[unused429] +[unused430] +[unused431] +[unused432] +[unused433] +[unused434] +[unused435] +[unused436] +[unused437] +[unused438] +[unused439] +[unused440] +[unused441] +[unused442] +[unused443] +[unused444] +[unused445] +[unused446] +[unused447] +[unused448] +[unused449] +[unused450] +[unused451] +[unused452] +[unused453] +[unused454] +[unused455] +[unused456] +[unused457] +[unused458] +[unused459] +[unused460] +[unused461] +[unused462] +[unused463] +[unused464] +[unused465] +[unused466] +[unused467] +[unused468] +[unused469] +[unused470] +[unused471] +[unused472] +[unused473] +[unused474] +[unused475] +[unused476] +[unused477] +[unused478] +[unused479] +[unused480] +[unused481] +[unused482] +[unused483] +[unused484] +[unused485] +[unused486] +[unused487] +[unused488] +[unused489] +[unused490] +[unused491] +[unused492] +[unused493] +[unused494] +[unused495] +[unused496] +[unused497] +[unused498] +[unused499] +[unused500] +[unused501] +[unused502] +[unused503] +[unused504] +[unused505] +[unused506] +[unused507] +[unused508] +[unused509] +[unused510] +[unused511] +[unused512] +[unused513] +[unused514] +[unused515] +[unused516] +[unused517] +[unused518] +[unused519] +[unused520] +[unused521] +[unused522] +[unused523] +[unused524] +[unused525] +[unused526] +[unused527] +[unused528] +[unused529] +[unused530] +[unused531] +[unused532] +[unused533] +[unused534] +[unused535] +[unused536] +[unused537] +[unused538] +[unused539] +[unused540] +[unused541] +[unused542] +[unused543] +[unused544] +[unused545] +[unused546] +[unused547] +[unused548] +[unused549] +[unused550] +[unused551] +[unused552] +[unused553] +[unused554] +[unused555] +[unused556] +[unused557] +[unused558] +[unused559] +[unused560] +[unused561] +[unused562] +[unused563] +[unused564] +[unused565] +[unused566] +[unused567] +[unused568] +[unused569] +[unused570] +[unused571] +[unused572] +[unused573] +[unused574] +[unused575] +[unused576] +[unused577] +[unused578] +[unused579] +[unused580] +[unused581] +[unused582] +[unused583] +[unused584] +[unused585] +[unused586] +[unused587] +[unused588] +[unused589] +[unused590] +[unused591] +[unused592] +[unused593] +[unused594] +[unused595] +[unused596] +[unused597] +[unused598] +[unused599] +[unused600] +[unused601] +[unused602] +[unused603] +[unused604] +[unused605] +[unused606] +[unused607] +[unused608] +[unused609] +[unused610] +[unused611] +[unused612] +[unused613] +[unused614] +[unused615] +[unused616] +[unused617] +[unused618] +[unused619] +[unused620] +[unused621] +[unused622] +[unused623] +[unused624] +[unused625] +[unused626] +[unused627] +[unused628] +[unused629] +[unused630] +[unused631] +[unused632] +[unused633] +[unused634] +[unused635] +[unused636] +[unused637] +[unused638] +[unused639] +[unused640] +[unused641] +[unused642] +[unused643] +[unused644] +[unused645] +[unused646] +[unused647] +[unused648] +[unused649] +[unused650] +[unused651] +[unused652] +[unused653] +[unused654] +[unused655] +[unused656] +[unused657] +[unused658] +[unused659] +[unused660] +[unused661] +[unused662] +[unused663] +[unused664] +[unused665] +[unused666] +[unused667] +[unused668] +[unused669] +[unused670] +[unused671] +[unused672] +[unused673] +[unused674] +[unused675] +[unused676] +[unused677] +[unused678] +[unused679] +[unused680] +[unused681] +[unused682] +[unused683] +[unused684] +[unused685] +[unused686] +[unused687] +[unused688] +[unused689] +[unused690] +[unused691] +[unused692] +[unused693] +[unused694] +[unused695] +[unused696] +[unused697] +[unused698] +[unused699] +[unused700] +[unused701] +[unused702] +[unused703] +[unused704] +[unused705] +[unused706] +[unused707] +[unused708] +[unused709] +[unused710] +[unused711] +[unused712] +[unused713] +[unused714] +[unused715] +[unused716] +[unused717] +[unused718] +[unused719] +[unused720] +[unused721] +[unused722] +[unused723] +[unused724] +[unused725] +[unused726] +[unused727] +[unused728] +[unused729] +[unused730] +[unused731] +[unused732] +[unused733] +[unused734] +[unused735] +[unused736] +[unused737] +[unused738] +[unused739] +[unused740] +[unused741] +[unused742] +[unused743] +[unused744] +[unused745] +[unused746] +[unused747] +[unused748] +[unused749] +[unused750] +[unused751] +[unused752] +[unused753] +[unused754] +[unused755] +[unused756] +[unused757] +[unused758] +[unused759] +[unused760] +[unused761] +[unused762] +[unused763] +[unused764] +[unused765] +[unused766] +[unused767] +[unused768] +[unused769] +[unused770] +[unused771] +[unused772] +[unused773] +[unused774] +[unused775] +[unused776] +[unused777] +[unused778] +[unused779] +[unused780] +[unused781] +[unused782] +[unused783] +[unused784] +[unused785] +[unused786] +[unused787] +[unused788] +[unused789] +[unused790] +[unused791] +[unused792] +[unused793] +[unused794] +[unused795] +[unused796] +[unused797] +[unused798] +[unused799] +[unused800] +[unused801] +[unused802] +[unused803] +[unused804] +[unused805] +[unused806] +[unused807] +[unused808] +[unused809] +[unused810] +[unused811] +[unused812] +[unused813] +[unused814] +[unused815] +[unused816] +[unused817] +[unused818] +[unused819] +[unused820] +[unused821] +[unused822] +[unused823] +[unused824] +[unused825] +[unused826] +[unused827] +[unused828] +[unused829] +[unused830] +[unused831] +[unused832] +[unused833] +[unused834] +[unused835] +[unused836] +[unused837] +[unused838] +[unused839] +[unused840] +[unused841] +[unused842] +[unused843] +[unused844] +[unused845] +[unused846] +[unused847] +[unused848] +[unused849] +[unused850] +[unused851] +[unused852] +[unused853] +[unused854] +[unused855] +[unused856] +[unused857] +[unused858] +[unused859] +[unused860] +[unused861] +[unused862] +[unused863] +[unused864] +[unused865] +[unused866] +[unused867] +[unused868] +[unused869] +[unused870] +[unused871] +[unused872] +[unused873] +[unused874] +[unused875] +[unused876] +[unused877] +[unused878] +[unused879] +[unused880] +[unused881] +[unused882] +[unused883] +[unused884] +[unused885] +[unused886] +[unused887] +[unused888] +[unused889] +[unused890] +[unused891] +[unused892] +[unused893] +[unused894] +[unused895] +[unused896] +[unused897] +[unused898] +[unused899] +[unused900] +[unused901] +[unused902] +[unused903] +[unused904] +[unused905] +[unused906] +[unused907] +[unused908] +[unused909] +[unused910] +[unused911] +[unused912] +[unused913] +[unused914] +[unused915] +[unused916] +[unused917] +[unused918] +[unused919] +[unused920] +[unused921] +[unused922] +[unused923] +[unused924] +[unused925] +[unused926] +[unused927] +[unused928] +[unused929] +[unused930] +[unused931] +[unused932] +[unused933] +[unused934] +[unused935] +[unused936] +[unused937] +[unused938] +[unused939] +[unused940] +[unused941] +[unused942] +[unused943] +[unused944] +[unused945] +[unused946] +[unused947] +[unused948] +[unused949] +[unused950] +[unused951] +[unused952] +[unused953] +[unused954] +[unused955] +[unused956] +[unused957] +[unused958] +[unused959] +[unused960] +[unused961] +[unused962] +[unused963] +[unused964] +[unused965] +[unused966] +[unused967] +[unused968] +[unused969] +[unused970] +[unused971] +[unused972] +[unused973] +[unused974] +[unused975] +[unused976] +[unused977] +[unused978] +[unused979] +[unused980] +[unused981] +[unused982] +[unused983] +[unused984] +[unused985] +[unused986] +[unused987] +[unused988] +[unused989] +[unused990] +[unused991] +[unused992] +[unused993] +` +¡ +¢ +¦ +¨ +ª +¬ +´ +¶ +½ +¾ +¿ +ð +þ +ħ +ı +ł +œ +ƒ +ɐ +ɑ +ɒ +ɕ +ɛ +ɣ +ɨ +ɪ +ɫ +ɬ +ɯ +ɲ +ɴ +ɹ +ɾ +ʀ +ʁ +ʂ +ʃ +ʉ +ʊ +ʋ +ʌ +ʎ +ʐ +ʑ +ʒ +ʔ +ʲ +ʳ +ʷ +ʸ +ʻ +ʼ +ʾ +ʿ +ˡ +ˣ +ˤ +ζ +ξ +щ +ъ +э +ю +ђ +є +ј +љ +њ +ћ +ӏ +ա +բ +գ +դ +ե +թ +ի +լ +կ +հ +մ +յ +ն +ո +պ +ս +վ +տ +ր +ւ +ք +־ +א +ב +ג +ד +ה +ו +ז +ח +ט +י +ך +כ +ל +ם +מ +ן +נ +ס +ע +ף +פ +ץ +צ +ק +ר +ש +ת +، +ء +ث +ج +ح +خ +ذ +ز +ش +ص +ض +ط +ظ +غ +ـ +ف +ق +ك +ى +ٹ +پ +چ +ک +گ +ں +ھ +ہ +ی +ے +अ +आ +उ +ए +क +ख +ग +च +ज +ट +ड +ण +त +थ +द +ध +न +प +ब +भ +म +य +र +ल +व +श +ष +स +ह +ा +ि +ी +ो +। +॥ +ং +অ +আ +ই +উ +এ +ও +ক +খ +গ +চ +ছ +জ +ট +ড +ণ +ত +থ +দ +ধ +ন +প +ব +ভ +ম +য +র +ল +শ +ষ +স +হ +া +ি +ী +ে +க +ச +ட +த +ந +ன +ப +ம +ய +ர +ல +ள +வ +ா +ி +ு +ே +ை +ನ +ರ +ಾ +ක +ය +ර +ල +ව +ා +ต +ท +พ +ล +ว +ส +། +ག +ང +ད +ན +པ +བ +མ +འ +ར +ལ +ས +မ +ა +ბ +გ +დ +ე +ვ +თ +ი +კ +ლ +მ +ნ +ო +რ +ს +ტ +უ +ᄊ +ᴬ +ᴮ +ᴰ +ᴵ +ᴺ +ᵀ +ᵇ +ᵈ +ᵖ +ᵗ +ᵢ +ᵣ +ᵤ +ᵥ +ᶜ +ᶠ +‐ +‑ +‒ +– +— +― +‘ +’ +‚ +“ +” +‡ +… +⁰ +⁴ +⁵ +⁶ +⁷ +⁸ +⁹ +⁻ +₀ +₅ +₆ +₇ +₈ +₉ +₊ +₍ +₎ +ₐ +ₑ +ₒ +ₓ +ₕ +ₖ +ₗ +ₘ +ₙ +ₚ +ₛ +ₜ +₤ +₩ +₱ +₹ +ℓ +ℝ +⅓ +⅔ +↦ +⇄ +⇌ +∂ +∅ +∆ +∇ +∈ +∗ +∘ +∧ +∨ +∪ +⊂ +⊆ +⊕ +⊗ +☉ +♭ +♯ +⟨ +⟩ +ⱼ +⺩ +⺼ +⽥ +亻 +宀 +彳 +忄 +扌 +氵 +疒 +糹 +訁 +辶 +阝 +龸 +fi +fl +had +were +which +him +their +been +would +then +them +could +during +through +between +while +later +around +did +such +being +used +against +many +both +these +known +until +even +didn +because +born +since +still +became +any +including +took +same +each +called +much +however +four +another +found +won +going +away +hand +several +following +released +played +began +district +those +held +own +early +league +government +came +based +thought +looked +along +went +few +father +former +located +got +though +every +century +without +within +building +large +named +started +once +should +built +british +death +moved +door +need +president +wasn +although +due +major +died +third +knew +asked +turned +wanted +together +received +son +served +different +behind +himself +felt +members +football +near +having +saw +mother +army +front +late +hands +put +division +across +told +often +ever +french +six +include +tell +among +species +really +according +half +original +gave +making +enough +opened +must +included +given +german +woman +community +might +million +court +short +round +seen +always +become +sure +almost +director +council +career +things +using +couldn +better +students +married +nothing +worked +others +record +anything +continued +give +military +established +returned +does +written +thing +feet +far +already +championship +western +department +role +various +production +television +produced +working +region +present +period +looking +least +total +england +wife +per +brother +soon +political +taken +created +further +able +reached +joined +upon +done +important +either +appeared +position +ground +lead +election +arms +police +instead +words +moment +someone +announced +less +wrote +past +followed +founded +finally +india +taking +records +considered +northern +toward +european +outside +described +track +playing +heard +professional +australia +miles +yet +trying +blood +southern +maybe +everything +mouth +race +recorded +above +daughter +points +middle +move +tried +elected +closed +ten +minister +chief +person +similar +brought +rest +formed +floor +doing +killed +training +needed +turn +finished +railway +rather +sent +example +ran +term +coming +currently +forces +despite +areas +fact +dead +originally +germany +probably +developed +pulled +stood +signed +songs +child +eventually +met +average +teams +minutes +current +kind +decided +usually +eastern +seemed +episode +bed +added +indian +route +available +throughout +addition +appointed +eight +construction +mean +remained +schools +sometimes +events +possible +australian +forward +debut +seat +performance +committee +features +character +herself +lot +russian +range +hours +sold +quickly +directed +guitar +performed +players +smile +myself +placed +province +towards +wouldn +leading +whole +designed +census +europe +attack +japanese +getting +alone +lower +wide +hospital +believe +changed +sister +gone +hadn +ship +studies +academy +shot +below +involved +kept +largest +especially +beginning +movement +section +female +professor +lord +longer +walked +actually +civil +families +thus +aircraft +completed +includes +captain +fight +vocals +featured +fourth +officer +hear +means +medical +groups +lips +competition +entire +lived +leaving +federal +tournament +passed +independent +kingdom +spent +fine +doesn +reported +fall +raised +itself +replaced +leader +theatre +whose +parents +spanish +canadian +degree +writing +awarded +higher +coast +provided +senior +organization +stopped +onto +countries +parts +conference +interest +saying +allowed +earlier +matter +winning +try +happened +moving +los +breath +nearly +mid +certain +italian +african +standing +fell +artist +shows +deal +mine +industry +everyone +republic +provide +student +primary +owned +older +heavy +1st +makes +attention +anyone +africa +stated +length +ended +fingers +command +staff +foreign +opening +governor +okay +medal +kill +introduced +chest +hell +feeling +success +meet +reason +meeting +novel +trade +buildings +guy +goal +native +husband +previously +entered +producer +operations +takes +covered +forced +roman +complete +successful +texas +cold +traditional +films +clear +approximately +nine +prince +question +tracks +ireland +regional +personal +operation +economic +holding +twenty +additional +hour +regular +historic +places +whom +shook +km² +secretary +prior +scored +units +ask +property +ready +immediately +month +listed +contract +themselves +lines +navy +writer +meant +runs +practice +championships +singer +commission +required +starting +generally +giving +attended +couple +stand +catholic +caught +executive +thinking +chair +quite +shoulder +hope +decision +plays +defeated +municipality +whether +offered +slowly +pain +direction +mission +mostly +noted +individual +managed +lives +plant +helped +except +studied +computer +figure +relationship +issue +significant +loss +smiled +gun +highest +male +bring +goals +mexico +problem +distance +commercial +completely +location +annual +famous +neck +caused +italy +understand +greek +highway +wrong +comes +appearance +issues +musical +companies +castle +income +assembly +bass +initially +parliament +artists +experience +particular +walk +foot +engineering +talking +dropped +boys +stars +remember +carried +train +stadium +angeles +evidence +becoming +assistant +soviet +upper +youth +reach +actor +numerous +nodded +arrived +minute +believed +complex +victory +associated +temple +chance +perhaps +bishop +launched +particularly +retired +subject +prize +contains +yeah +theory +empire +suddenly +waiting +trust +recording +terms +champion +religious +zealand +names +2nd +ancient +corner +represented +legal +justice +cause +watched +brothers +material +changes +simply +response +answer +historical +stories +straight +feature +increased +administration +virginia +activities +cultural +overall +winner +programs +basketball +legs +guard +cast +doctor +flight +results +remains +cost +effect +winter +larger +islands +problems +chairman +grew +commander +isn +failed +selected +hurt +fort +regiment +majority +plans +shown +pretty +irish +characters +directly +scene +likely +operated +allow +matches +looks +houses +fellow +marriage +rules +florida +expected +nearby +congress +peace +recent +wait +subsequently +variety +serving +agreed +poor +attempt +wood +democratic +rural +mile +appears +township +soldiers +##ized +pennsylvania +closer +fighting +claimed +score +physical +filled +genus +specific +sitting +mom +therefore +supported +status +fear +cases +meaning +wales +minor +spain +vice +parish +separate +horse +fifth +remaining +branch +presented +stared +uses +forms +baseball +exactly +choice +discovered +composed +truth +russia +dad +ring +referred +numbers +greater +metres +slightly +direct +increase +responsible +crew +rule +trees +troops +broke +goes +individuals +hundred +weight +creek +sleep +defense +provides +ordered +jewish +safe +judge +whatever +corps +realized +growing +cities +gaze +lies +spread +letter +showed +situation +mayor +transport +watching +workers +extended +expression +normal +chart +multiple +border +mrs +walls +piano +heat +cannot +earned +products +drama +era +authority +seasons +join +grade +difficult +territory +mainly +stations +squadron +stepped +iron +19th +serve +appear +speak +broken +charge +knowledge +kilometres +removed +ships +campus +pushed +britain +leaves +recently +boston +latter +acquired +poland +quality +officers +presence +planned +nations +mass +broadcast +influence +wild +emperor +electric +headed +ability +promoted +yellow +ministry +throat +smaller +politician +latin +spoke +cars +males +lack +acting +seeing +consists +estate +pressure +newspaper +olympics +conditions +beat +elements +walking +vote +needs +carolina +featuring +levels +francisco +purpose +females +dutch +duke +ahead +gas +safety +serious +turning +highly +lieutenant +firm +amount +mixed +proposed +perfect +agreement +affairs +3rd +seconds +contemporary +paid +prison +label +administrative +intended +constructed +academic +teacher +races +formerly +nation +issued +shut +drums +housing +seems +graduated +mentioned +picked +recognized +shortly +protection +picture +notable +elections +1980s +loved +percent +racing +elizabeth +volume +hockey +beside +settled +competed +replied +drew +actress +marine +scotland +steel +glanced +farm +risk +tonight +positive +singles +effects +gray +screen +residents +sides +none +secondary +literature +polish +destroyed +flying +founder +households +lay +reserve +industrial +younger +approach +appearances +ones +finish +powerful +fully +growth +honor +jersey +projects +revealed +infantry +pair +equipment +visit +evening +grant +effort +treatment +buried +republican +primarily +bottom +owner +1970s +israel +gives +remain +spot +produce +champions +accepted +ways +##ally +losing +split +capacity +basis +trial +questions +20th +guess +officially +memorial +naval +initial +##ization +whispered +median +engineer +sydney +columbia +strength +tears +senate +asian +draw +warm +supposed +transferred +leaned +candidate +escape +mountains +potential +activity +seem +traffic +murder +slow +orchestra +haven +agency +taught +website +comedy +unable +storm +planning +albums +rugby +environment +scientific +grabbed +protect +boat +typically +damage +principal +divided +dedicated +ohio +pick +fought +driver +empty +shoulders +sort +thank +berlin +prominent +account +freedom +necessary +efforts +headquarters +follows +alongside +suggested +operating +steps +technical +begin +easily +teeth +speaking +settlement +scale +renamed +enemy +semi +joint +compared +scottish +leadership +analysis +offers +georgia +pieces +captured +animal +deputy +organized +combined +method +challenge +1960s +huge +wants +battalion +sons +rise +crime +types +facilities +telling +platform +sit +1990s +tells +assigned +pull +commonly +alive +letters +concept +conducted +wearing +happen +bought +becomes +holy +gets +defeat +languages +purchased +occurred +titled +declared +applied +sciences +concert +sounds +jazz +brain +painting +fleet +tax +michigan +animals +leaders +episodes +birth +clubs +palace +critical +refused +fair +leg +laughed +returning +surrounding +participated +formation +lifted +pointed +connected +rome +medicine +laid +powers +tall +shared +focused +knowing +yards +entrance +falls +calling +sources +chosen +beneath +resources +yard +nominated +silence +defined +gained +thirty +bodies +adopted +christmas +widely +register +apart +iran +premier +serves +unknown +parties +generation +continues +fields +brigade +quiet +teaching +clothes +impact +weapons +partner +flat +theater +relations +plants +suffered +begins +seats +armed +models +worth +laws +communities +classes +background +knows +thanks +quarter +reaching +humans +carry +killing +format +setting +architecture +disease +railroad +possibly +arthur +thoughts +doors +density +crowd +illinois +stomach +tone +unique +reports +anyway +liberal +vehicle +thick +dry +drug +faced +largely +facility +theme +holds +creation +strange +colonel +revolution +politics +turns +silent +rail +relief +independence +combat +shape +determined +sales +learned +4th +finger +providing +heritage +fiction +situated +designated +allowing +hosted +sight +interview +estimated +reduced +toronto +footballer +keeping +guys +damn +claim +motion +sixth +stayed +rear +receive +handed +twelve +dress +audience +granted +brazil +spirit +##ated +noticed +olympic +representative +tight +trouble +reviews +drink +vampire +missing +roles +ranked +newly +household +finals +critics +phase +massachusetts +pilot +unlike +philadelphia +bright +guns +crown +organizations +roof +respectively +clearly +tongue +marked +circle +bronze +expanded +sexual +supply +yourself +inspired +labour +reference +draft +connection +reasons +driving +jesus +cells +entry +neither +trail +claims +atlantic +orders +labor +nose +afraid +identified +intelligence +calls +cancer +attacked +passing +positions +imperial +grey +swedish +avoid +extra +uncle +covers +allows +surprise +materials +fame +hunter +citizens +figures +environmental +confirmed +shit +titles +performing +difference +acts +attacks +existing +votes +opportunity +nor +entirely +trains +opposite +pakistan +develop +resulted +representatives +actions +reality +pressed +barely +conversation +faculty +northwest +ends +documentary +nuclear +stock +sets +eat +alternative +resulting +creating +surprised +cemetery +drop +finding +cricket +streets +tradition +ride +ear +explained +composer +injury +apartment +municipal +educational +occupied +netherlands +clean +billion +constitution +learn +maximum +classical +lose +opposition +ontario +hills +rolled +ending +drawn +permanent +lewis +sites +chamber +scoring +height +lyrics +staring +officials +snow +oldest +qualified +interior +apparently +succeeded +thousand +dinner +lights +existence +heavily +greatest +conservative +send +bowl +catch +duty +speech +authorities +princess +performances +versions +shall +graduate +pictures +effective +remembered +poetry +desk +crossed +starring +starts +passenger +sharp +acres +ass +weather +falling +rank +fund +supporting +adult +heads +southeast +lane +condition +transfer +prevent +regions +earl +federation +relatively +answered +besides +obtained +portion +reaction +liked +peak +counter +religion +chain +rare +convention +aid +lie +vehicles +perform +squad +wonder +lying +crazy +sword +attempted +centuries +weren +philosophy +interested +sweden +wolf +frequently +abandoned +literary +alliance +task +entitled +threw +promotion +tiny +soccer +visited +achieved +defence +internal +persian +methods +arrested +otherwise +programming +villages +elementary +districts +rooms +criminal +conflict +worry +trained +attempts +waited +signal +truck +subsequent +programme +communist +faith +sector +carrying +laugh +controlled +korean +showing +origin +fuel +evil +brief +identity +darkness +pool +missed +publication +wings +invited +briefly +standards +kissed +ideas +climate +causing +walter +worse +albert +winners +desire +aged +northeast +dangerous +gate +doubt +wooden +poet +rising +funding +communications +communication +violence +copies +prepared +investigation +skills +pulling +containing +ultimately +offices +singing +understanding +tomorrow +christ +ward +pope +stands +5th +flow +studios +aired +commissioned +contained +exist +americans +wrestling +approved +kid +employed +respect +suit +asking +increasing +frame +angry +selling +1950s +thin +finds +temperature +statement +ali +explain +inhabitants +towns +extensive +narrow +flowers +promise +somewhere +closely +bureau +cape +weekly +presidential +legislative +launch +founding +artillery +strike +un +institutions +roll +writers +landing +chose +anymore +attorney +billboard +receiving +agricultural +breaking +sought +dave +admitted +lands +mexican +##bury +specifically +hole +moscow +roads +accident +proved +struck +guards +stuff +slid +expansion +melbourne +opposed +sub +southwest +architect +failure +plane +tank +listen +regarding +wet +introduction +metropolitan +fighter +inch +grown +gene +anger +fixed +khan +domestic +worldwide +chapel +mill +functions +examples +developing +turkey +hits +pocket +antonio +papers +grow +unless +circuit +18th +concerned +attached +journalist +selection +journey +converted +provincial +painted +hearing +aren +bands +negative +aside +wondered +knight +lap +noise +billy +shooting +bedroom +priest +resistance +motor +homes +sounded +giant +scenes +equal +comic +patients +hidden +solid +actual +bringing +afternoon +touched +funds +consisted +marie +canal +treaty +turkish +recognition +residence +cathedral +broad +knees +incident +shaped +fired +norwegian +handle +cheek +contest +represent +representing +birds +advantage +emergency +wrapped +drawing +notice +broadcasting +somehow +bachelor +seventh +collected +registered +establishment +assumed +chemical +personnel +retirement +portuguese +wore +tied +device +threat +progress +advance +##ised +banks +hired +manchester +nfl +teachers +structures +forever +tennis +helping +saturday +applications +junction +incorporated +neighborhood +dressed +ceremony +influenced +hers +stairs +decades +inner +kansas +hung +hoped +gain +scheduled +downtown +engaged +austria +clock +norway +certainly +pale +victor +employees +plate +putting +surrounded +##ists +finishing +blues +tropical +minnesota +consider +philippines +accept +retrieved +concern +anderson +properties +institution +gordon +successfully +vietnam +backing +outstanding +muslim +crossing +folk +producing +usual +demand +occurs +observed +lawyer +educated +pleasure +budget +items +quietly +colorado +philip +typical +##worth +derived +survived +asks +mental +jake +jews +distinguished +sri +extremely +athletic +loud +thousands +worried +transportation +horses +weapon +arena +importance +users +objects +contributed +douglas +aware +senator +johnny +sisters +engines +flag +investment +samuel +shock +capable +clark +row +wheel +refers +familiar +biggest +wins +hate +maintained +drove +hamilton +expressed +injured +underground +churches +wars +tunnel +passes +stupid +agriculture +softly +cabinet +regarded +joining +indiana +dates +spend +behavior +woods +protein +gently +chase +morgan +mention +burning +wake +combination +occur +mirror +leads +indeed +impossible +paintings +covering +soldier +locations +attendance +sell +historian +wisconsin +invasion +argued +painter +diego +changing +egypt +experienced +inches +missouri +grounds +spoken +switzerland +reform +rolling +forget +massive +resigned +burned +tennessee +locked +values +improved +wounded +universe +sick +dating +facing +purchase +##pur +moments +merged +anniversary +coal +brick +understood +causes +dynasty +queensland +establish +stores +crisis +promote +hoping +cards +referee +extension +raise +arizona +improve +colonial +formal +charged +palm +hide +rescue +faces +feelings +candidates +juan +6th +courses +weekend +luke +cash +fallen +delivered +affected +installed +carefully +tries +hollywood +costs +lincoln +responsibility +shore +proper +normally +maryland +assistance +constant +offering +friendly +waters +persons +realize +contain +trophy +partnership +factor +musicians +bound +oregon +indicated +houston +medium +consisting +somewhat +cycle +beer +moore +frederick +gotten +worst +weak +approached +arranged +chin +loan +bond +fifteen +pattern +disappeared +translated +##zed +lip +arab +capture +interests +insurance +shifted +cave +prix +warning +sections +courts +coat +plot +smell +golf +favorite +maintain +knife +voted +degrees +finance +quebec +opinion +translation +manner +ruled +operate +productions +choose +musician +confused +tired +separated +stream +techniques +committed +attend +ranking +kings +throw +passengers +measure +horror +mining +sand +danger +salt +calm +decade +dam +require +runner +rush +associate +greece +rivers +consecutive +matthew +##ski +sighed +sq +documents +closing +tie +accused +islamic +distributed +directors +organisation +7th +breathing +mad +lit +arrival +concrete +taste +composition +shaking +faster +amateur +adjacent +stating +twin +flew +publications +obviously +ridge +storage +carl +pages +concluded +desert +driven +universities +ages +terminal +sequence +borough +constituency +cousin +economics +dreams +margaret +notably +reduce +montreal +17th +ears +saved +vocal +riding +roughly +threatened +meters +meanwhile +landed +compete +repeated +grass +czech +regularly +charges +sudden +appeal +solution +describes +classification +glad +parking +belt +physics +rachel +hungarian +participate +expedition +damaged +gift +childhood +fifty +mathematics +jumped +letting +defensive +mph +testing +hundreds +shoot +owners +matters +smoke +israeli +kentucky +dancing +mounted +grandfather +designs +profit +argentina +truly +lawrence +cole +begun +detroit +willing +branches +smiling +decide +miami +enjoyed +recordings +##dale +poverty +ethnic +arabic +accompanied +fishing +determine +residential +acid +returns +starred +strategy +forty +businesses +equivalent +commonwealth +distinct +ill +seriously +##ped +harris +replace +rio +imagine +formula +ensure +additionally +scheme +conservation +occasionally +purposes +feels +favor +1930s +contrast +hanging +hunt +movies +instruments +victims +danish +christopher +busy +demon +sugar +earliest +colony +studying +duties +belgium +slipped +carter +visible +stages +iraq +commune +forming +continuing +talked +counties +legend +bathroom +option +tail +clay +daughters +afterwards +severe +jaw +visitors +devices +aviation +entering +subjects +temporary +swimming +forth +smooth +bush +operates +rocks +movements +signs +eddie +voices +honorary +memories +dallas +measures +racial +promised +harvard +16th +parliamentary +indicate +benefit +flesh +dublin +louisiana +patient +sleeping +membership +coastal +medieval +wanting +element +scholars +rice +limit +survive +makeup +rating +definitely +collaboration +obvious +baron +birthday +linked +soil +diocese +ncaa +offensive +shouldn +waist +plain +ross +organ +resolution +manufacturing +adding +relative +kennedy +whilst +moth +gardens +crash +heading +partners +credited +carlos +moves +cable +marshall +depending +bottle +represents +rejected +responded +existed +denmark +##ating +treated +graham +routes +talent +commissioner +drugs +secure +tests +reign +restored +photography +contributions +oklahoma +designer +disc +grin +seattle +robin +paused +atlanta +unusual +praised +las +laughing +satellite +hungary +visiting +interesting +factors +deck +poems +norman +##water +stuck +speaker +rifle +premiered +comics +actors +reputation +eliminated +8th +ceiling +prisoners +leather +austin +mississippi +rapidly +admiral +parallel +charlotte +guilty +tools +gender +divisions +fruit +laboratory +nelson +marry +rapid +aunt +tribe +requirements +aspects +suicide +amongst +adams +bone +ukraine +kick +sees +edinburgh +clothing +column +rough +gods +hunting +broadway +gathered +concerns +spending +ty +12th +snapped +requires +solar +bones +cavalry +iowa +drinking +waste +franklin +charity +thompson +stewart +tip +landscape +enjoy +singh +poem +listening +eighth +fred +differences +adapted +bomb +ukrainian +surgery +corporate +masters +anywhere +waves +odd +portugal +orleans +dick +debate +kent +eating +puerto +cleared +expect +cinema +guitarist +blocks +electrical +agree +involving +depth +dying +panel +struggle +peninsula +adults +novels +emerged +vienna +debuted +shoes +tamil +songwriter +meets +prove +beating +instance +heaven +scared +sending +marks +artistic +passage +superior +significantly +retained +##izing +technique +cheeks +warren +maintenance +destroy +extreme +allied +appearing +fill +advice +alabama +qualifying +policies +cleveland +hat +battery +authors +10th +soundtrack +acted +dated +lb +glance +equipped +coalition +funny +outer +ambassador +roy +possibility +couples +campbell +loose +ethan +supplies +gonna +monster +shake +agents +frequency +springs +dogs +practices +gang +plastic +easier +suggests +gulf +blade +exposed +colors +industries +markets +nervous +electoral +charts +legislation +ownership +##idae +appointment +shield +assault +socialist +abbey +monument +license +throne +employment +replacement +charter +suffering +accounts +oak +connecticut +strongly +wright +colour +13th +context +welsh +networks +voiced +gabriel +forehead +manage +schedule +totally +remix +forests +occupation +print +nicholas +brazilian +strategic +vampires +engineers +roots +seek +correct +instrumental +und +alfred +backed +stanley +robinson +traveled +wayne +austrian +achieve +exit +rates +strip +whereas +sing +deeply +adventure +bobby +jamie +careful +components +cap +useful +personality +knee +pushing +hosts +protest +ottoman +symphony +boundary +processes +considering +considerable +tons +cooper +trading +conduct +illegal +revolutionary +definition +harder +jacob +circumstances +destruction +popularity +grip +classified +liverpool +baltimore +flows +seeking +honour +approval +mechanical +till +happening +statue +critic +increasingly +immediate +describe +commerce +stare +indonesia +meat +rounds +boats +baker +orthodox +depression +formally +worn +naked +muttered +sentence +11th +document +criticism +wished +vessel +spiritual +bent +virgin +minimum +murray +lunch +danny +printed +compilation +keyboards +blow +belonged +raising +cutting +pittsburgh +9th +shadows +hated +indigenous +jon +15th +barry +scholar +oliver +stick +susan +meetings +attracted +spell +romantic +ye +demanded +customers +logan +revival +keys +modified +commanded +jeans +upset +phil +detective +hiding +resident +##bly +experiences +diamond +defeating +coverage +lucas +external +parks +franchise +helen +bible +successor +percussion +celebrated +lift +clan +romania +##ied +mills +nobody +achievement +shrugged +fault +rhythm +initiative +breakfast +carbon +lasted +violent +wound +killer +gradually +filmed +°c +processing +remove +criticized +guests +sang +chemistry +legislature +##bridge +uniform +escaped +integrated +proposal +purple +denied +liquid +influential +morris +nights +stones +intense +experimental +twisted +pace +nazi +mitchell +ny +blind +reporter +newspapers +14th +centers +burn +basin +forgotten +surviving +filed +collections +monastery +losses +manual +couch +description +appropriate +merely +missions +sebastian +restoration +replacing +triple +elder +julia +warriors +benjamin +julian +convinced +stronger +amazing +declined +versus +merchant +happens +output +finland +bare +barbara +absence +ignored +dawn +injuries +producers +luis +##ities +kw +admit +expensive +electricity +exception +symbol +ladies +shower +sheriff +characteristics +##je +aimed +button +ratio +effectively +summit +angle +jury +bears +foster +vessels +pants +executed +evans +dozen +advertising +kicked +patrol +competitions +lifetime +principles +athletics +birmingham +sponsored +rob +nomination +acoustic +creature +longest +credits +harbor +dust +josh +territories +milk +infrastructure +completion +thailand +indians +leon +archbishop +assist +pitch +blake +arrangement +girlfriend +serbian +operational +hence +sad +scent +fur +sessions +refer +rarely +exists +1892 +scientists +dirty +penalty +burst +portrait +seed +pole +limits +rival +stable +grave +constitutional +alcohol +arrest +flower +mystery +devil +architectural +relationships +greatly +habitat +##istic +larry +progressive +remote +cotton +preserved +reaches +cited +vast +scholarship +decisions +teach +editions +knocked +eve +searching +partly +participation +animated +fate +excellent +alternate +saints +youngest +climbed +suggest +discussion +staying +choir +lakes +jacket +revenue +nevertheless +peaked +instrument +wondering +annually +managing +neil +1891 +signing +terry +apply +clinical +brooklyn +aim +catherine +fuck +farmers +figured +ninth +pride +hugh +ordinary +involvement +comfortable +shouted +encouraged +representation +sharing +panic +exact +cargo +competing +fat +cried +1920s +occasions +cabin +borders +utah +marcus +##isation +badly +muscles +victorian +transition +warner +bet +permission +slave +terrible +similarly +shares +seth +uefa +possession +medals +benefits +colleges +lowered +perfectly +transit +##kar +publisher +##ened +harrison +deaths +elevation +asleep +machines +sigh +ash +hardly +argument +occasion +parent +decline +contribution +concentration +opportunities +hispanic +guardian +extent +emotions +hips +mason +volumes +bloody +controversy +diameter +steady +mistake +phoenix +identify +violin +departure +richmond +spin +funeral +enemies +1864 +literally +connor +random +sergeant +grab +confusion +1865 +transmission +informed +leaning +sacred +suspended +thinks +gates +portland +luck +agencies +yours +hull +expert +muscle +layer +practical +sculpture +jerusalem +latest +lloyd +statistics +deeper +recommended +warrior +arkansas +mess +supports +greg +eagle +recovered +rated +concerts +rushed +stops +eggs +premiere +keith +delhi +turner +pit +affair +belief +paint +##zing +victim +withdrew +bonus +styles +fled +glasgow +technologies +funded +adaptation +portrayed +cooperation +supporters +judges +bernard +hallway +ralph +graduating +controversial +distant +continental +spider +bite +recognize +intention +mixing +egyptian +bow +tourism +suppose +claiming +dominated +participants +nurse +partially +tape +psychology +essential +touring +duo +voting +civilian +emotional +channels +apparent +hebrew +1887 +tommy +carrier +intersection +beast +hudson +bench +discuss +costa +##ered +detailed +behalf +drivers +unfortunately +obtain +rocky +##dae +siege +friendship +1861 +hang +governments +collins +respond +wildlife +preferred +operator +laura +pregnant +videos +dennis +suspected +boots +instantly +weird +automatic +businessman +alleged +placing +throwing +mood +1862 +perry +venue +jet +remainder +passion +biological +boyfriend +1863 +dirt +buffalo +ron +segment +abuse +genre +thrown +stroke +colored +stress +exercise +displayed +struggled +abroad +dramatic +wonderful +thereafter +madrid +component +widespread +##sed +tale +citizen +todd +vancouver +overseas +forcing +crying +descent +discussed +substantial +ranks +regime +provinces +drum +zane +tribes +proof +researchers +volunteer +manor +silk +milan +donated +allies +venture +principle +delivery +enterprise +bars +traditionally +witch +reminded +copper +pete +inter +colin +grinned +elsewhere +competitive +frequent +scream +tension +texts +submarine +finnish +defending +defend +pat +detail +affiliated +stuart +themes +periods +tool +belgian +ruling +crimes +answers +folded +licensed +demolished +hans +lucy +1881 +lion +traded +photographs +writes +craig +trials +generated +beth +noble +debt +percentage +yorkshire +erected +viewed +grades +confidence +ceased +islam +telephone +retail +chile +m² +roberts +sixteen +commented +hampshire +innocent +dual +pounds +checked +regulations +afghanistan +sung +rico +liberty +assets +bigger +options +angels +relegated +tribute +wells +attending +leaf +romanian +monthly +patterns +gmina +madison +hurricane +rev +##ians +bristol +elite +valuable +disaster +democracy +awareness +germans +freyja +loop +absolutely +paying +populations +maine +sole +prayer +spencer +releases +doorway +bull +lover +midnight +conclusion +thirteen +mediterranean +nhl +proud +sample +##hill +drummer +guinea +murphy +climb +instant +attributed +horn +ain +railways +autumn +ferry +opponent +traveling +secured +corridor +stretched +tales +sheet +trinity +cattle +helps +indicates +manhattan +murdered +fitted +gentle +grandmother +mines +shocked +vegas +produces +caribbean +belong +continuous +desperate +drunk +historically +trio +waved +raf +dealing +nathan +murmured +interrupted +residing +scientist +pioneer +harold +aaron +delta +attempting +minority +believes +chorus +tend +lots +eyed +indoor +load +shots +updated +jail +concerning +connecting +wealth +slaves +arrive +rangers +sufficient +rebuilt +##wick +cardinal +flood +muhammad +whenever +relation +runners +moral +repair +viewers +arriving +revenge +punk +assisted +bath +fairly +breathe +lists +innings +illustrated +whisper +nearest +voters +clinton +ties +ultimate +screamed +beijing +lions +andre +fictional +gathering +comfort +radar +suitable +dismissed +hms +ban +pine +wrist +atmosphere +voivodeship +bid +timber +##ned +giants +cameron +recovery +uss +identical +categories +switched +serbia +laughter +noah +ensemble +therapy +peoples +touching +##off +locally +pearl +platforms +everywhere +ballet +tables +lanka +herbert +outdoor +toured +derek +1883 +spaces +contested +swept +1878 +exclusive +slight +connections +winds +prisoner +collective +bangladesh +tube +publicly +wealthy +isolated +insisted +fortune +ticket +spotted +reportedly +animation +enforcement +tanks +decides +wider +lowest +owen +nod +hitting +gregory +furthermore +magazines +fighters +solutions +pointing +requested +peru +reed +chancellor +knights +mask +worker +eldest +flames +reduction +volunteers +reporting +wire +advisory +endemic +origins +settlers +pursue +knock +consumer +1876 +eu +compound +creatures +mansion +sentenced +ivan +deployed +guitars +frowned +involves +mechanism +kilometers +perspective +shops +terminus +duncan +alien +fist +bridges +##pers +heroes +derby +swallowed +patent +sara +illness +characterized +adventures +slide +hawaii +jurisdiction +organised +adelaide +walks +biology +rogers +swing +tightly +boundaries +prepare +implementation +stolen +certified +colombia +edwards +garage +recalled +rage +harm +nigeria +breast +furniture +pupils +settle +cuba +balls +alaska +21st +linear +thrust +celebration +latino +genetic +terror +##ening +lightning +fee +witness +lodge +establishing +skull +earning +hood +rebellion +sporting +warned +missile +devoted +activist +porch +worship +fourteen +package +decorated +##shire +housed +chess +sailed +doctors +oscar +joan +treat +garcia +harbour +jeremy +traditions +dominant +jacques +##gon +relocated +1879 +amendment +sized +companion +simultaneously +volleyball +spun +acre +increases +stopping +loves +belongs +affect +drafted +tossed +scout +battles +1875 +filming +shoved +munich +tenure +vertical +romance +argue +craft +ranging +opens +honest +tyler +yesterday +muslims +reveal +snake +immigrants +radical +screaming +speakers +firing +saving +belonging +ease +lighting +prefecture +blame +farmer +hungry +grows +rubbed +beam +sur +subsidiary +armenian +dropping +conventional +qualify +spots +sweat +festivals +immigration +physician +discover +exposure +sandy +explanation +isaac +implemented +##fish +hart +initiated +stakes +presents +heights +householder +pleased +tourist +regardless +slip +closest +surely +sultan +brings +riley +preparation +aboard +slammed +baptist +experiment +ongoing +interstate +organic +playoffs +1877 +hindu +tours +tier +plenty +arrangements +talks +trapped +excited +sank +athens +1872 +denver +welfare +suburb +athletes +trick +diverse +belly +exclusively +yelled +conversion +1874 +internationally +computers +conductor +abilities +sensitive +dispute +measured +globe +rocket +prices +amsterdam +flights +tigers +municipalities +emotion +references +explains +airlines +manufactured +archaeological +1873 +interpretation +devon +##ites +settlements +kissing +absolute +improvement +impressed +barcelona +sullivan +jefferson +towers +jesse +julie +grandson +gauge +regard +rings +interviews +trace +raymond +thumb +departments +burns +serial +bulgarian +scores +demonstrated +1866 +kyle +alberta +underneath +romanized +relieved +acquisition +phrase +cliff +reveals +cuts +merger +custom +nee +gilbert +graduation +assessment +difficulty +demands +swung +democrat +commons +1940s +grove +completing +focuses +sum +substitute +bearing +stretch +reception +reflected +essentially +destination +pairs +##ched +survival +resource +##bach +promoting +doubles +messages +tear +##fully +parade +florence +harvey +incumbent +partial +pedro +frozen +procedure +olivia +controls +shelter +personally +temperatures +brisbane +tested +sits +marble +comprehensive +oxygen +leonard +##kov +inaugural +iranian +referring +quarters +attitude +mainstream +lined +mars +dakota +norfolk +unsuccessful +explosion +helicopter +congressional +##sing +inspector +bitch +seal +departed +divine +coaching +examination +punishment +manufacturer +sink +columns +unincorporated +signals +nevada +squeezed +dylan +dining +martial +manuel +eighteen +elevator +brushed +plates +ministers +congregation +slept +specialized +taxes +restricted +negotiations +likes +statistical +arnold +inspiration +execution +bold +intermediate +significance +margin +ruler +wheels +gothic +intellectual +dependent +listened +eligible +buses +widow +syria +earn +cincinnati +collapsed +recipient +secrets +accessible +philippine +maritime +goddess +clerk +surrender +breaks +playoff +ideal +beetle +aspect +soap +regulation +strings +expand +anglo +shorter +crosses +retreat +tough +coins +wallace +directions +pressing +shipping +locomotives +comparison +topics +nephew +distinction +honors +travelled +sierra +ibn +fortress +recognised +carved +1869 +clients +intent +coaches +describing +bread +##ington +beaten +northwestern +merit +collapse +challenges +historians +objective +submitted +virus +attacking +drake +assume +diseases +stem +leeds +farming +glasses +visits +nowhere +fellowship +relevant +carries +restaurants +experiments +constantly +bases +targets +shah +tenth +opponents +verse +territorial +writings +corruption +instruction +inherited +reverse +emphasis +employee +arch +keeps +rabbi +watson +payment +uh +nancy +##tre +venice +fastest +sexy +banned +adrian +properly +ruth +touchdown +dollar +boards +metre +circles +edges +favour +travels +liberation +scattered +firmly +holland +permitted +diesel +kenya +den +originated +demons +resumed +dragged +rider +servant +blinked +extend +torn +##sey +input +meal +everybody +cylinder +kinds +camps +bullet +logic +croatian +evolved +healthy +fool +wise +preserve +pradesh +respective +artificial +gross +corresponding +convicted +cage +caroline +dialogue +##dor +narrative +stranger +mario +christianity +failing +trent +commanding +buddhist +1848 +maurice +focusing +yale +bike +altitude +mouse +revised +##sley +veteran +pulls +theology +crashed +campaigns +legion +##ability +drag +excellence +customer +cancelled +intensity +excuse +liga +participating +contributing +printing +##burn +variable +curious +legacy +renaissance +symptoms +binding +vocalist +dancer +grammar +gospel +democrats +enters +diplomatic +hitler +clouds +mathematical +quit +defended +oriented +##heim +fundamental +hardware +impressive +equally +convince +confederate +guilt +chuck +sliding +magnetic +narrowed +petersburg +bulgaria +otto +phd +skill +hopes +pitcher +reservoir +hearts +automatically +expecting +mysterious +bennett +extensively +imagined +seeds +monitor +fix +##ative +journalism +struggling +signature +ranch +encounter +photographer +observation +protests +influences +calendar +cruz +croatia +locomotive +hughes +naturally +shakespeare +basement +hook +uncredited +faded +theories +approaches +dare +phillips +filling +fury +obama +efficient +arc +deliver +breeding +inducted +leagues +efficiency +axis +montana +eagles +##ked +supplied +instructions +karen +picking +indicating +trap +anchor +practically +christians +tomb +vary +occasional +electronics +lords +readers +newcastle +faint +innovation +collect +situations +engagement +claude +mixture +##feld +peer +tissue +lean +°f +floors +architects +reducing +rope +1859 +ottawa +##har +samples +banking +declaration +proteins +resignation +francois +saudi +advocate +exhibited +armor +twins +divorce +##ras +abraham +reviewed +temporarily +matrix +physically +pulse +curled +difficulties +bengal +usage +##ban +riders +certificate +holes +warsaw +distinctive +mutual +1857 +customs +circular +eugene +removal +loaded +mere +vulnerable +depicted +generations +dame +heir +enormous +lightly +climbing +pitched +lessons +pilots +nepal +preparing +brad +louise +renowned +liam +##ably +shaw +brilliant +bills +##nik +fucking +mainland +pleasant +seized +veterans +jerked +fail +brush +radiation +stored +warmth +southeastern +nate +sin +raced +berkeley +joke +athlete +designation +trunk +roland +qualification +heels +artwork +receives +judicial +reserves +##bed +woke +installation +abu +floating +fake +lesser +excitement +interface +concentrated +addressed +characteristic +amanda +saxophone +monk +releasing +egg +dies +interaction +defender +outbreak +glory +loving +sequel +consciousness +awake +ski +enrolled +handling +rookie +brow +somebody +biography +warfare +amounts +contracts +presentation +fabric +dissolved +challenged +meter +psychological +elevated +rally +accurate +##tha +hospitals +undergraduate +specialist +venezuela +exhibit +shed +nursing +protestant +fluid +structural +footage +jared +consistent +prey +##ska +succession +reflect +exile +lebanon +wiped +suspect +shanghai +resting +integration +preservation +marvel +variant +pirates +sheep +rounded +capita +sailing +colonies +manuscript +deemed +variations +clarke +functional +emerging +boxing +relaxed +curse +azerbaijan +heavyweight +nickname +editorial +rang +grid +tightened +earthquake +flashed +miguel +rushing +##ches +improvements +boxes +brooks +consumption +molecular +felix +societies +repeatedly +variation +aids +civic +graphics +professionals +realm +autonomous +receiver +delayed +workshop +militia +chairs +canyon +harsh +extending +lovely +happiness +##jan +stake +eyebrows +embassy +wellington +hannah +corners +bishops +swear +cloth +contents +namely +commenced +1854 +stanford +nashville +courage +graphic +commitment +garrison +hamlet +clearing +rebels +attraction +literacy +cooking +ruins +temples +jenny +humanity +celebrate +hasn +freight +sixty +rebel +bastard +newton +deer +##ges +##ching +smiles +delaware +singers +approaching +assists +flame +boulevard +barrel +planted +pursuit +consequences +shallow +invitation +rode +depot +ernest +kane +rod +concepts +preston +topic +chambers +striking +blast +arrives +descendants +montgomery +ranges +worlds +chaos +praise +fewer +1855 +sanctuary +mud +programmes +maintaining +harper +bore +handsome +closure +tournaments +nebraska +linda +facade +puts +satisfied +argentine +dale +cork +dome +panama +##yl +1858 +tasks +experts +##ates +feeding +equation +engage +bryan +um +quartet +disbanded +sheffield +blocked +gasped +delay +kisses +connects +##non +sts +poured +creator +publishers +guided +ellis +extinct +hug +gaining +##ord +complicated +poll +clenched +investigate +thereby +quantum +spine +cdp +humor +kills +administered +semifinals +encountered +ignore +commentary +##maker +bother +roosevelt +plains +halfway +flowing +cultures +crack +imprisoned +neighboring +airline +gather +wolves +marathon +transformed +cruise +organisations +punch +exhibitions +numbered +alarm +ratings +daddy +silently +##stein +queens +colours +impression +guidance +tactical +##rat +marshal +della +arrow +rested +feared +tender +owns +bitter +advisor +escort +##ides +spare +farms +grants +dragons +encourage +colleagues +cameras +sucked +pile +spirits +prague +statements +suspension +landmark +fence +torture +recreation +bags +permanently +survivors +pond +spy +predecessor +bombing +coup +protecting +transformation +glow +##lands +dug +priests +andrea +feat +barn +jumping +##ologist +casualties +stern +auckland +pipe +serie +revealing +trevor +mercy +spectrum +consist +governing +collaborated +possessed +epic +comprises +blew +shane +lopez +honored +magical +sacrifice +judgment +perceived +hammer +baronet +tune +das +missionary +sheets +neutral +oral +threatening +attractive +shade +aims +seminary +estates +1856 +michel +wounds +refugees +manufacturers +mercury +syndrome +porter +##iya +##din +hamburg +identification +upstairs +purse +widened +pause +cared +breathed +affiliate +santiago +prevented +celtic +fisher +recruited +byzantine +reconstruction +farther +diet +sake +spite +sensation +blank +separation +##hon +vladimir +armies +anime +accommodate +orbit +cult +sofia +##ify +founders +sustained +disorder +honours +northeastern +mia +crops +violet +threats +blanket +fires +canton +followers +southwestern +prototype +voyage +assignment +altered +moderate +protocol +pistol +questioned +brass +lifting +1852 +math +authored +doug +dimensional +dynamic +1851 +pronounced +grateful +quest +uncomfortable +boom +presidency +stevens +relating +politicians +barrier +quinn +diana +mosque +tribal +palmer +portions +sometime +chester +treasure +bend +millions +reforms +registration +consequently +monitoring +ate +preliminary +brandon +invented +eaten +exterior +intervention +ports +documented +displays +lecture +sally +favourite +vermont +invisible +isle +breed +journalists +relay +speaks +backward +explore +midfielder +actively +stefan +procedures +cannon +blond +kenneth +centered +servants +chains +libraries +malcolm +essex +henri +slavery +##hal +facts +fairy +coached +cassie +cats +washed +cop +announcement +2000s +vinyl +activated +marco +frontier +growled +curriculum +##das +loyal +accomplished +leslie +ritual +kenny +vii +napoleon +hollow +hybrid +jungle +stationed +friedrich +counted +##ulated +platinum +theatrical +seated +col +rubber +glen +diversity +healing +extends +provisions +administrator +columbus +tributary +assured +##uous +prestigious +examined +lectures +grammy +ronald +associations +bailey +allan +essays +flute +believing +consultant +proceedings +travelling +1853 +kerala +yugoslavia +buddy +methodist +burial +centres +batman +discontinued +dock +stockholm +lungs +severely +citing +manga +steal +mumbai +iraqi +robot +celebrity +bride +broadcasts +abolished +pot +joel +overhead +franz +packed +reconnaissance +johann +acknowledged +introduce +handled +doctorate +developments +drinks +alley +palestine +##aki +proceeded +recover +bradley +grain +patch +afford +infection +nationalist +legendary +interchange +virtually +gen +gravity +exploration +amber +vital +wishes +powell +doctrine +elbow +screenplay +##bird +contribute +indonesian +creates +enzyme +kylie +discipline +drops +manila +hunger +layers +suffer +fever +bits +monica +keyboard +manages +##hood +searched +appeals +##bad +testament +grande +reid +##war +beliefs +congo +requiring +casey +1849 +regret +streak +rape +depends +syrian +sprint +pound +tourists +upcoming +pub +tense +##els +practiced +nationwide +guild +motorcycle +liz +##zar +chiefs +desired +elena +precious +absorbed +relatives +booth +pianist +##mal +citizenship +exhausted +wilhelm +##ceae +##hed +noting +quarterback +urge +hectares +##gue +holly +blonde +davies +parked +sustainable +stepping +twentieth +airfield +nest +chip +##nell +shaft +paulo +requirement +paradise +tobacco +trans +renewed +vietnamese +suggesting +catching +holmes +enjoying +trips +colt +holder +butterfly +nerve +reformed +cherry +bowling +trailer +carriage +goodbye +appreciate +toy +joshua +interactive +enabled +involve +##kan +collar +determination +bunch +recall +shorts +superintendent +episcopal +frustration +giovanni +nineteenth +laser +privately +array +circulation +##ovic +armstrong +deals +painful +permit +discrimination +aires +retiring +cottage +horizon +ellen +jamaica +ripped +fernando +chapters +patron +lecturer +behaviour +genes +georgian +export +solomon +rivals +seventeen +rodriguez +princeton +independently +sox +1847 +arguing +entity +casting +hank +criteria +oakland +geographic +milwaukee +reflection +expanding +conquest +dubbed +halt +brave +brunswick +arched +curtis +divorced +predominantly +somerset +streams +ugly +zoo +horrible +curved +buenos +fierce +dictionary +vector +theological +unions +handful +stability +punjab +segments +altar +ignoring +gesture +monsters +pastor +thighs +unexpected +operators +abruptly +coin +compiled +associates +improving +migration +compact +collegiate +quarterfinals +roster +restore +assembled +hurry +oval +##cies +1846 +flags +martha +victories +sharply +##rated +argues +deadly +drawings +symbols +performer +griffin +restrictions +editing +andrews +journals +arabia +compositions +dee +pierce +removing +hindi +casino +runway +civilians +minds +##zation +refuge +rent +retain +potentially +conferences +suburban +conducting +descended +massacre +ammunition +terrain +fork +souls +counts +chelsea +durham +drives +cab +perth +realizing +palestinian +finn +simpson +##dal +betty +moreover +particles +cardinals +tent +evaluation +extraordinary +inscription +wednesday +chloe +maintains +panels +ashley +trucks +##nation +cluster +sunlight +strikes +zhang +dialect +tucked +collecting +##mas +##sville +quoted +evan +franco +aria +buying +cleaning +closet +provision +apollo +clinic +rat +necessarily +##ising +venues +flipped +cent +spreading +trustees +checking +authorized +disappointed +##ado +notion +duration +trumpet +hesitated +topped +brussels +rolls +theoretical +hint +define +aggressive +repeat +wash +peaceful +optical +width +allegedly +mcdonald +strict +##illa +investors +jam +witnesses +sounding +miranda +michelle +hugo +harmony +valid +lynn +glared +nina +headquartered +diving +boarding +gibson +albanian +marsh +routine +dealt +enhanced +intelligent +substance +targeted +enlisted +discovers +spinning +observations +pissed +smoking +capitol +varied +costume +seemingly +indies +compensation +surgeon +thursday +arsenal +westminster +suburbs +rid +anglican +##ridge +knots +foods +alumni +lighter +fraser +whoever +portal +scandal +gavin +advised +instructor +flooding +terrorist +teenage +interim +senses +duck +teen +thesis +abby +eager +overcome +newport +glenn +rises +shame +prompted +priority +forgot +bomber +nicolas +protective +cartoon +katherine +breeze +lonely +trusted +henderson +richardson +relax +palms +remarkable +legends +cricketer +essay +ordained +edmund +rifles +trigger +##uri +##away +sail +alert +1830 +audiences +penn +sussex +siblings +pursued +indianapolis +resist +rosa +consequence +succeed +avoided +1845 +##ulation +inland +##tie +##nna +counsel +profession +chronicle +hurried +##una +eyebrow +eventual +bleeding +innovative +cure +committees +accounting +scope +hardy +heather +tenor +gut +herald +codes +tore +scales +wagon +luxury +tin +prefer +fountain +triangle +bonds +darling +convoy +dried +traced +beings +troy +accidentally +slam +findings +smelled +joey +lawyers +outcome +steep +bosnia +configuration +shifting +toll +brook +performers +lobby +philosophical +construct +shrine +aggregate +cox +phenomenon +savage +insane +solely +reynolds +nationally +holdings +consideration +enable +edgar +fights +relegation +chances +atomic +hub +conjunction +awkward +reactions +currency +finale +kumar +underwent +steering +elaborate +gifts +comprising +melissa +veins +reasonable +sunshine +solve +trails +inhabited +elimination +ethics +huh +ana +molly +consent +apartments +layout +marines +hunters +bulk +##oma +hometown +##wall +##mont +cracked +reads +neighbouring +withdrawn +admission +wingspan +damned +anthology +lancashire +brands +batting +forgive +cuban +awful +##lyn +dimensions +imagination +dante +tracking +desperately +goalkeeper +##yne +groaned +workshops +confident +burton +gerald +milton +circus +uncertain +slope +copenhagen +sophia +fog +philosopher +portraits +accent +cycling +varying +gripped +larvae +garrett +specified +scotia +mature +luther +kurt +rap +##kes +aerial +ferdinand +heated +transported +##shan +safely +nonetheless +##orn +##gal +motors +demanding +##sburg +startled +##brook +ally +generate +caps +ghana +stained +mentions +beds +afterward +##bling +utility +##iro +richards +1837 +conspiracy +conscious +shining +footsteps +observer +cyprus +urged +loyalty +developer +probability +olive +upgraded +gym +miracle +insects +graves +1844 +ourselves +hydrogen +katie +tickets +poets +planes +prevention +witnessed +dense +jin +randy +tang +warehouse +monroe +archived +elderly +investigations +alec +granite +mineral +conflicts +controlling +aboriginal +mechanics +stan +stark +rhode +skirt +est +bombs +respected +##horn +imposed +limestone +deny +nominee +memphis +grabbing +disabled +amusement +frankfurt +corn +referendum +varies +slowed +disk +firms +unconscious +incredible +clue +sue +##zhou +twist +##cio +joins +idaho +chad +developers +computing +destroyer +mortal +tucker +kingston +choices +carson +whitney +geneva +pretend +dimension +staged +plateau +maya +##une +freestyle +rovers +##ids +tristan +classroom +prospect +##hus +honestly +diploma +lied +thermal +auxiliary +feast +unlikely +iata +morocco +pounding +treasury +lithuania +considerably +1841 +dish +1812 +geological +matching +stumbled +destroying +marched +brien +advances +nicole +settling +measuring +directing +##mie +tuesday +bassist +capabilities +stunned +fraud +torpedo +##phone +anton +wisdom +surveillance +ruined +##ulate +lawsuit +healthcare +theorem +halls +trend +aka +horizontal +dozens +acquire +lasting +swim +hawk +gorgeous +fees +vicinity +decrease +adoption +tactics +##ography +pakistani +##ole +draws +##hall +willie +burke +heath +algorithm +integral +powder +elliott +brigadier +jackie +tate +varieties +darker +##cho +lately +cigarette +specimens +adds +##ensis +##inger +exploded +finalist +murders +wilderness +arguments +nicknamed +acceptance +onwards +manufacture +robertson +jets +tampa +enterprises +loudly +composers +nominations +1838 +malta +inquiry +automobile +hosting +viii +rays +tilted +grief +museums +strategies +furious +euro +equality +cohen +poison +surrey +wireless +governed +ridiculous +moses +##esh +vanished +barnes +attract +morrison +istanbul +##iness +absent +rotation +petition +janet +##logical +satisfaction +custody +deliberately +observatory +comedian +surfaces +pinyin +novelist +strictly +canterbury +oslo +monks +embrace +jealous +photograph +continent +dorothy +marina +excess +holden +allegations +explaining +stack +avoiding +lance +storyline +majesty +poorly +spike +bradford +raven +travis +classics +proven +voltage +pillow +fists +butt +1842 +interpreted +1839 +gage +telegraph +lens +promising +expelled +casual +collector +zones +silly +nintendo +##kh +downstairs +chef +suspicious +afl +flies +vacant +uganda +pregnancy +condemned +lutheran +estimates +cheap +decree +saxon +proximity +stripped +idiot +deposits +contrary +presenter +magnus +glacier +offense +edwin +##ori +upright +##long +bolt +##ois +toss +geographical +##izes +environments +delicate +marking +abstract +xavier +nails +windsor +plantation +occurring +equity +saskatchewan +fears +drifted +sequences +vegetation +revolt +##stic +1843 +sooner +fusion +opposing +nato +skating +1836 +secretly +ruin +lease +flora +anxiety +##ological +##mia +bout +taxi +emmy +frost +rainbow +compounds +foundations +rainfall +assassination +nightmare +dominican +achievements +deserve +orlando +intact +armenia +##nte +calgary +valentine +marion +proclaimed +theodore +bells +courtyard +thigh +gonzalez +console +troop +minimal +everyday +supporter +terrorism +buck +openly +presbyterian +activists +carpet +##iers +rubbing +uprising +cute +conceived +legally +##cht +millennium +cello +velocity +rescued +cardiff +1835 +rex +concentrate +senators +beard +rendered +glowing +battalions +scouts +competitors +sculptor +catalogue +arctic +ion +raja +bicycle +glancing +lawn +##woman +gentleman +lighthouse +publish +predicted +calculated +variants +##gne +strain +winston +deceased +touchdowns +brady +caleb +sinking +echoed +crush +hon +blessed +protagonist +hayes +endangered +magnitude +editors +##tine +estimate +responsibilities +##mel +backup +laying +consumed +sealed +zurich +lovers +frustrated +##eau +ahmed +kicking +treasurer +1832 +biblical +refuse +terrified +pump +agrees +genuine +imprisonment +refuses +plymouth +lou +##nen +tara +trembling +antarctic +ton +learns +##tas +crap +crucial +faction +atop +##borough +wrap +lancaster +odds +hopkins +erik +lyon +##eon +bros +snap +locality +empress +crowned +cal +acclaimed +chuckled +clara +sends +mild +towel +wishing +assuming +interviewed +##bal +interactions +eden +cups +helena +indie +beck +##fire +batteries +filipino +wizard +parted +traces +##born +rows +idol +albany +delegates +##ees +##sar +discussions +notre +instructed +belgrade +highways +suggestion +lauren +possess +orientation +alexandria +abdul +beats +salary +reunion +ludwig +alright +wagner +intimate +pockets +slovenia +hugged +brighton +merchants +cruel +stole +trek +slopes +repairs +enrollment +politically +underlying +promotional +counting +boeing +isabella +naming +keen +bacteria +listing +separately +belfast +ussr +lithuanian +anybody +ribs +sphere +martinez +cock +embarrassed +proposals +fragments +nationals +##wski +premises +fin +alpine +matched +freely +bounded +jace +sleeve +pier +populated +evident +##like +frances +flooded +##dle +frightened +pour +trainer +framed +visitor +challenging +pig +wickets +##fold +infected +##pes +arose +reward +ecuador +oblast +vale +shuttle +##usa +bach +rankings +forbidden +cornwall +accordance +salem +consumers +bruno +fantastic +toes +machinery +resolved +julius +remembering +propaganda +iceland +bombardment +tide +contacts +wives +##rah +concerto +macdonald +albania +implement +daisy +tapped +sudan +helmet +mistress +crop +sunk +finest +##craft +hostile +boxer +fr +paths +adjusted +habit +ballot +supervision +soprano +bullets +wicked +sunset +regiments +disappear +lamp +performs +##gia +rabbit +digging +incidents +entries +##cion +dishes +introducing +##ati +##fied +freshman +slot +jill +tackles +baroque +backs +##iest +lone +sponsor +destiny +altogether +convert +##aro +consensus +shapes +demonstration +basically +feminist +auction +artifacts +##bing +strongest +halifax +allmusic +mighty +smallest +precise +alexandra +viola +##los +##ille +manuscripts +##illo +dancers +ari +managers +monuments +blades +barracks +springfield +maiden +consolidated +electron +berry +airing +wheat +nobel +inclusion +blair +payments +geography +bee +eleanor +react +##hurst +afc +manitoba +lineup +fitness +recreational +investments +airborne +disappointment +##dis +edmonton +viewing +renovation +infant +bankruptcy +roses +aftermath +pavilion +carpenter +withdrawal +ladder +discussing +popped +reliable +agreements +rochester +##abad +curves +bombers +rao +reverend +decreased +choosing +stiff +consulting +naples +crawford +tracy +ribbon +cops +crushed +deciding +unified +teenager +accepting +flagship +poles +sanchez +inspection +revived +skilled +induced +exchanged +flee +locals +tragedy +swallow +hanna +demonstrate +##ela +salvador +flown +contestants +civilization +##ines +wanna +rhodes +fletcher +hector +knocking +considers +nash +mechanisms +sensed +mentally +walt +unclear +##eus +renovated +madame +crews +governmental +undertaken +monkey +##ben +##ato +fatal +armored +copa +caves +governance +grasp +perception +certification +froze +damp +tugged +wyoming +##rg +##ero +newman +nerves +curiosity +graph +##ami +withdraw +tunnels +dull +meredith +moss +exhibits +neighbors +communicate +accuracy +explored +raiders +republicans +secular +kat +superman +penny +criticised +freed +conviction +ham +likewise +delegation +gotta +doll +promises +technological +myth +nationality +resolve +convent +sharon +dig +sip +coordinator +entrepreneur +fold +##dine +capability +councillor +synonym +blown +swan +cursed +1815 +jonas +haired +sofa +canvas +keeper +rivalry +##hart +rapper +speedway +swords +postal +maxwell +estonia +potter +recurring +errors +##oni +cognitive +1834 +claws +nadu +roberto +bce +wrestler +ellie +infinite +ink +##tia +presumably +finite +staircase +noel +patricia +nacional +chill +eternal +tu +preventing +prussia +fossil +limbs +##logist +ernst +frog +perez +rene +prussian +##ios +molecules +regulatory +answering +opinions +sworn +lengths +supposedly +hypothesis +upward +habitats +seating +ancestors +drank +yield +synthesis +researcher +modest +##var +mothers +peered +voluntary +homeland +acclaim +##igan +static +valve +luxembourg +alto +carroll +receptor +norton +ambulance +##tian +johnston +catholics +depicting +jointly +elephant +gloria +mentor +badge +ahmad +distinguish +remarked +councils +precisely +allison +advancing +detection +crowded +cooperative +ankle +mercedes +dagger +surrendered +pollution +commit +subway +jeffrey +lesson +sculptures +provider +##fication +membrane +timothy +rectangular +fiscal +heating +teammate +basket +particle +anonymous +deployment +missiles +courthouse +proportion +shoe +sec +complaints +forbes +blacks +abandon +remind +sizes +overwhelming +autobiography +natalie +##awa +risks +contestant +countryside +babies +scorer +invaded +enclosed +proceed +hurling +disorders +##cu +reflecting +continuously +cruiser +graduates +freeway +investigated +ore +deserved +maid +blocking +phillip +jorge +shakes +dove +mann +variables +lacked +burden +accompanying +que +consistently +organizing +provisional +complained +endless +tubes +juice +georges +krishna +mick +thriller +laps +arcade +sage +snail +shannon +laurence +seoul +vacation +presenting +hire +churchill +surprisingly +prohibited +savannah +technically +##oli +##lessly +testimony +suited +speeds +toys +romans +flowering +measurement +talented +kay +settings +charleston +expectations +shattered +achieving +triumph +ceremonies +portsmouth +lanes +mandatory +loser +stretching +cologne +realizes +seventy +cornell +careers +webb +##ulating +americas +budapest +ava +suspicion +yo +conrad +sterling +jessie +rector +##az +1831 +transform +organize +loans +christine +volcanic +warrant +slender +summers +subfamily +newer +danced +dynamics +rhine +proceeds +heinrich +gastropod +commands +sings +facilitate +easter +positioned +responses +expense +fruits +yanked +imported +25th +velvet +vic +primitive +tribune +baldwin +neighbourhood +donna +rip +hay +##uro +1814 +espn +welcomed +##aria +qualifier +glare +highland +timing +##cted +shells +eased +geometry +louder +exciting +slovakia +##iz +savings +prairie +marching +rafael +tonnes +##lled +curtain +preceding +shy +heal +greene +worthy +##pot +detachment +bury +sherman +##eck +reinforced +seeks +bottles +contracted +duchess +outfit +walsh +mickey +geoffrey +archer +squeeze +dawson +eliminate +invention +##enberg +neal +##eth +stance +dealer +coral +maple +retire +simplified +1833 +hid +watts +backwards +jules +##oke +genesis +frames +rebounds +burma +woodland +moist +santos +whispers +drained +subspecies +streaming +ulster +burnt +correspondence +maternal +gerard +denis +stealing +genius +duchy +##oria +inaugurated +momentum +suits +placement +sovereign +clause +thames +##hara +confederation +reservation +sketch +yankees +lets +rotten +charm +hal +verses +commercially +dot +salon +citation +adopt +winnipeg +mist +allocated +cairo +jenkins +interference +objectives +##wind +1820 +portfolio +armoured +sectors +initiatives +integrity +exercises +robe +tap +gazed +##tones +distracted +rulers +favorable +jerome +tended +cart +factories +##eri +diplomat +valued +gravel +charitable +calvin +exploring +shepherd +terrace +pupil +##ural +reflects +##rch +governors +shelf +depths +##nberg +trailed +crest +tackle +##nian +hatred +##kai +clare +makers +ethiopia +longtime +detected +embedded +lacking +slapped +rely +thomson +anticipation +morton +successive +agnes +screenwriter +straightened +philippe +playwright +haunted +licence +iris +intentions +sutton +logical +correctly +##weight +branded +licked +tipped +silva +ricky +narrator +requests +##ents +greeted +supernatural +cow +##wald +lung +refusing +employer +strait +gaelic +liner +##piece +zoe +sabha +##mba +driveway +harvest +prints +bates +reluctantly +threshold +algebra +ira +wherever +coupled +assumption +picks +designers +raids +gentlemen +roller +blowing +leipzig +locks +screw +dressing +strand +##lings +scar +dwarf +depicts +##nu +nods +differ +boris +##eur +yuan +flip +##gie +mob +invested +questioning +applying +shout +##sel +gameplay +blamed +illustrations +bothered +weakness +rehabilitation +##zes +envelope +rumors +miners +leicester +subtle +kerry +ferguson +premiership +bengali +prof +catches +remnants +dana +##rily +shouting +presidents +baltic +ought +ghosts +dances +sailors +shirley +fancy +dominic +##bie +madonna +##rick +bark +buttons +gymnasium +ashes +liver +toby +oath +providence +doyle +evangelical +nixon +cement +carnegie +embarked +hatch +surroundings +guarantee +needing +pirate +essence +filter +crane +hammond +projected +immune +percy +twelfth +regent +doctoral +damon +mikhail +##ichi +critically +elect +realised +abortion +acute +screening +mythology +steadily +frown +nottingham +kirk +wa +minneapolis +##rra +module +algeria +nautical +encounters +surprising +statues +availability +shirts +pie +alma +brows +munster +mack +soup +crater +tornado +sanskrit +cedar +explosive +bordered +dixon +planets +stamp +exam +happily +##bble +carriers +kidnapped +accommodation +emigrated +##met +knockout +correspondent +violation +profits +peaks +lang +specimen +agenda +ancestry +pottery +spelling +equations +obtaining +ki +linking +1825 +debris +asylum +buddhism +##ants +gazette +dental +eligibility +fathers +averaged +zimbabwe +francesco +coloured +hissed +translator +lynch +mandate +humanities +mackenzie +uniforms +##iana +asset +fitting +samantha +genera +rim +beloved +shark +riot +entities +expressions +indo +carmen +slipping +owing +abbot +neighbor +sidney +rats +recommendations +encouraging +squadrons +anticipated +commanders +conquered +donations +diagnosed +divide +##iva +guessed +decoration +vernon +auditorium +revelation +conversations +##kers +##power +herzegovina +dash +alike +protested +lateral +herman +accredited +##gent +freeman +mel +fiji +crow +crimson +##rine +livestock +##pped +humanitarian +bored +oz +whip +##lene +##ali +legitimate +alter +grinning +spelled +anxious +oriental +wesley +##nin +##hole +carnival +controller +detect +##ssa +bowed +educator +kosovo +macedonia +##sin +occupy +mastering +stephanie +janeiro +para +unaware +nurses +noon +hopefully +ranger +combine +sociology +polar +rica +##eer +neill +##sman +holocaust +doubled +lust +1828 +decent +cooling +unveiled +1829 +nsw +homer +chapman +meyer +dive +mae +reagan +expertise +##gled +darwin +brooke +sided +prosecution +investigating +comprised +petroleum +genres +reluctant +differently +trilogy +johns +vegetables +corpse +highlighted +lounge +pension +unsuccessfully +elegant +aided +ivory +beatles +amelia +cain +dubai +immigrant +babe +underwater +combining +mumbled +atlas +horns +accessed +ballad +physicians +homeless +gestured +rpm +freak +louisville +corporations +patriots +prizes +rational +warn +modes +decorative +overnight +din +troubled +phantom +monarch +sheer +##dorf +generals +guidelines +organs +addresses +enhance +curling +parishes +cord +##kie +caesar +deutsche +bavaria +coleman +cyclone +##eria +bacon +petty +##yama +##old +hampton +diagnosis +1824 +throws +complexity +rita +disputed +pablo +marketed +trafficking +##ulus +examine +plague +formats +vault +faithful +##bourne +webster +highlights +##ient +phones +vacuum +sandwich +modeling +##gated +bolivia +clergy +qualities +isabel +##nas +##ars +wears +screams +reunited +annoyed +bra +##ancy +##rate +differential +transmitter +tattoo +container +poker +##och +excessive +resides +cowboys +##tum +augustus +trash +providers +statute +retreated +balcony +reversed +void +storey +preceded +masses +leap +laughs +neighborhoods +wards +schemes +falcon +santo +battlefield +ronnie +lesbian +venus +##dian +beg +sandstone +daylight +punched +gwen +analog +stroked +wwe +acceptable +measurements +toxic +##kel +adequate +surgical +economist +parameters +varsity +##sberg +quantity +##chy +##rton +countess +generating +precision +diamonds +expressway +##ı +1821 +uruguay +talents +galleries +expenses +scanned +colleague +outlets +ryder +lucien +##ila +paramount +syracuse +dim +fangs +gown +sweep +##sie +missionaries +websites +sentences +adviser +val +trademark +spells +##plane +patience +starter +slim +##borg +toe +incredibly +shoots +elliot +nobility +##wyn +cowboy +endorsed +gardner +tendency +persuaded +organisms +emissions +kazakhstan +amused +boring +chips +themed +##hand +constantinople +chasing +systematic +guatemala +borrowed +erin +carey +##hard +highlands +struggles +1810 +##ifying +##ced +exceptions +develops +enlarged +kindergarten +castro +##rina +leigh +zombie +juvenile +##most +consul +sailor +hyde +clarence +intensive +pinned +nasty +useless +jung +clayton +stuffed +exceptional +ix +apostolic +transactions +exempt +swinging +cove +religions +shields +dairy +bypass +pursuing +joyce +bombay +chassis +southampton +chat +interact +redesignated +##pen +nascar +pray +salmon +rigid +regained +malaysian +grim +publicity +constituted +capturing +toilet +delegate +purely +tray +drift +loosely +striker +weakened +trinidad +mitch +itv +defines +transmitted +scarlet +nodding +fitzgerald +narrowly +tooth +standings +virtue +##wara +##cting +chateau +gloves +lid +hurting +conservatory +##pel +sinclair +reopened +sympathy +nigerian +strode +advocated +optional +chronic +discharge +suck +compatible +laurel +stella +fails +wage +dodge +informal +sorts +levi +buddha +villagers +chronicles +heavier +summoned +gateway +eleventh +jewelry +translations +accordingly +seas +##ency +fiber +pyramid +cubic +dragging +##ista +caring +##ops +contacted +lunar +lisbon +patted +1826 +sacramento +theft +madagascar +subtropical +disputes +holidays +piper +willow +mare +cane +newfoundland +benny +companions +dong +raj +observe +roar +charming +plaque +tibetan +fossils +enacted +manning +bubble +tanzania +##eda +##hir +funk +swamp +deputies +cloak +ufc +scenario +par +scratch +metals +anthem +guru +engaging +specially +##boat +dialects +nineteen +cecil +duet +disability +unofficial +##lies +defunct +moonlight +drainage +surname +puzzle +switching +conservatives +mammals +knox +broadcaster +sidewalk +cope +##ried +benson +princes +peterson +##sal +bedford +sharks +eli +wreck +alberto +gasp +archaeology +lgbt +teaches +securities +madness +compromise +waving +coordination +davidson +visions +leased +possibilities +eighty +fernandez +enthusiasm +assassin +sponsorship +reviewer +kingdoms +estonian +laboratories +##fy +##nal +applies +verb +celebrations +##zzo +rowing +lightweight +sadness +submit +balanced +dude +explicitly +metric +magnificent +mound +brett +mohammad +mistakes +irregular +sanders +betrayed +shipped +surge +##enburg +reporters +termed +georg +pity +verbal +bulls +abbreviated +enabling +appealed +sicily +sting +heel +sweetheart +bart +spacecraft +brutal +monarchy +aberdeen +cameo +diane +survivor +clyde +##aries +complaint +##makers +clarinet +delicious +chilean +karnataka +coordinates +1818 +panties +##rst +pretending +dramatically +kiev +tends +distances +catalog +launching +instances +telecommunications +portable +lindsay +vatican +##eim +angles +aliens +marker +stint +screens +bolton +##rne +judy +wool +benedict +plasma +europa +imaging +filmmaker +swiftly +contributor +opted +stamps +apologize +financing +butter +gideon +sophisticated +alignment +avery +chemicals +yearly +speculation +prominence +professionally +immortal +institutional +inception +wrists +identifying +tribunal +derives +gains +papal +preference +linguistic +vince +operative +brewery +##ont +unemployment +boyd +##ured +##outs +albeit +prophet +1813 +##rad +quarterly +asteroid +cleaned +radius +temper +##llen +telugu +jerk +viscount +##ote +glimpse +##aya +yacht +hawaiian +baden +laptop +readily +##gu +monetary +offshore +scots +watches +##yang +##arian +upgrade +needle +lea +encyclopedia +flank +fingertips +delight +teachings +confirm +roth +beaches +midway +winters +##iah +teasing +daytime +beverly +gambling +##backs +regulated +clement +hermann +tricks +knot +##shing +##uring +##vre +detached +ecological +owed +specialty +byron +inventor +bats +stays +screened +unesco +midland +trim +affection +##ander +jess +thoroughly +feedback +chennai +strained +heartbeat +wrapping +overtime +pleaded +##sworth +leisure +oclc +##tate +##ele +feathers +angelo +thirds +nuts +surveys +clever +gill +commentator +##dos +darren +rides +gibraltar +dissolution +dedication +shin +meals +saddle +elvis +reds +chaired +taller +appreciation +functioning +niece +favored +advocacy +robbie +criminals +suffolk +yugoslav +passport +constable +congressman +hastings +##rov +consecrated +sparks +ecclesiastical +confined +##ovich +muller +floyd +nora +1822 +paved +1827 +cumberland +ned +saga +spiral +appreciated +collaborative +treating +similarities +feminine +finishes +##ib +jade +import +##hot +champagne +mice +securing +celebrities +helsinki +attributes +##gos +cousins +phases +ache +lucia +gandhi +submission +vicar +spear +shine +tasmania +biting +detention +constitute +tighter +seasonal +##gus +terrestrial +matthews +effectiveness +parody +philharmonic +##onic +1816 +strangers +encoded +consortium +guaranteed +regards +shifts +tortured +collision +supervisor +inform +broader +insight +theaters +armour +emeritus +blink +incorporates +mapping +handball +flexible +##nta +substantially +generous +thief +carr +loses +1793 +prose +ucla +romeo +generic +metallic +realization +damages +commissioners +zach +default +helicopters +lengthy +stems +partnered +spectators +rogue +indication +penalties +teresa +1801 +sen +##tric +dalton +##wich +irving +photographic +##vey +deaf +peters +excluded +unsure +##vable +patterson +crawled +##zio +resided +whipped +latvia +slower +ecole +pipes +employers +maharashtra +comparable +textile +pageant +##gel +alphabet +binary +irrigation +chartered +choked +antoine +offs +waking +supplement +quantities +demolition +regain +locate +urdu +folks +scary +andreas +whites +##ava +classrooms +mw +aesthetic +publishes +valleys +guides +cubs +johannes +bryant +conventions +affecting +##itt +drain +awesome +isolation +prosecutor +ambitious +apology +captive +downs +atmospheric +lorenzo +aisle +beef +foul +##onia +kidding +composite +disturbed +illusion +natives +##ffer +rockets +riverside +wartime +painters +adolf +melted +uncertainty +simulation +hawks +progressed +meantime +builder +spray +breach +unhappy +regina +russians +determining +tram +1806 +##quin +aging +1823 +garion +rented +mister +diaz +terminated +clip +1817 +depend +nervously +disco +owe +defenders +shiva +notorious +disbelief +shiny +worcester +##gation +##yr +trailing +undertook +islander +belarus +limitations +watershed +fuller +overlooking +utilized +raphael +1819 +synthetic +breakdown +klein +##nate +moaned +memoir +lamb +practicing +##erly +cellular +arrows +exotic +witches +charted +rey +hut +hierarchy +subdivision +freshwater +giuseppe +aloud +reyes +qatar +marty +sideways +utterly +sexually +jude +prayers +mccarthy +softball +blend +damien +##gging +##metric +wholly +erupted +lebanese +negro +revenues +tasted +comparative +teamed +transaction +labeled +maori +sovereignty +parkway +trauma +gran +malay +advancement +descendant +buzz +salvation +inventory +symbolic +##making +antarctica +mps +##bro +mohammed +myanmar +holt +submarines +tones +##lman +locker +patriarch +bangkok +emerson +remarks +predators +kin +afghan +confession +norwich +rental +emerge +advantages +##zel +rca +##hold +shortened +storms +aidan +##matic +autonomy +compliance +##quet +dudley +##osis +1803 +motto +documentation +summary +professors +spectacular +christina +archdiocese +flashing +innocence +remake +##dell +psychic +reef +scare +employ +sticks +meg +gus +leans +accompany +bergen +tomas +doom +wages +pools +##bes +breasts +scholarly +alison +outline +brittany +breakthrough +willis +realistic +##cut +##boro +competitor +##stan +pike +picnic +designing +commercials +washing +villain +skiing +costumes +auburn +halted +executives +logistics +cycles +vowel +applicable +barrett +exclaimed +eurovision +eternity +ramon +##umi +modifications +sweeping +disgust +torch +aviv +ensuring +rude +dusty +sonic +donovan +outskirts +cu +pathway +##band +##gun +disciplines +acids +cadet +paired +sketches +##sive +marriages +folding +peers +slovak +implies +admired +##beck +1880s +leopold +instinct +attained +weston +megan +horace +##ination +dorsal +ingredients +evolutionary +complications +deity +lethal +brushing +levy +deserted +institutes +posthumously +delivering +telescope +coronation +motivated +rapids +luc +flicked +pays +volcano +tanner +weighed +##nica +crowds +frankie +gifted +addressing +granddaughter +winding +##rna +constantine +gomez +##front +landscapes +rudolf +anthropology +slate +werewolf +astronomy +circa +rouge +dreaming +sack +knelt +drowned +naomi +prolific +tracked +freezing +herb +agony +randall +twisting +wendy +deposit +touches +vein +wheeler +##bbled +batted +retaining +tire +presently +compare +specification +daemon +nigel +##grave +merry +recommendation +czechoslovakia +sandra +roma +##sts +lambert +inheritance +sheikh +winchester +cries +examining +##yle +comeback +cuisine +nave +##iv +retrieve +tomatoes +barker +polished +defining +irene +lantern +personalities +begging +tract +swore +1809 +##gic +omaha +brotherhood +haiti +##ots +exeter +##ete +##zia +steele +dumb +pearson +surveyed +elisabeth +trends +fritz +bugs +fraction +calmly +viking +##birds +tug +inserted +unusually +##ield +confronted +distress +crashing +brent +turks +resign +##olo +cambodia +gabe +sauce +##kal +evelyn +extant +clusters +quarry +teenagers +luna +##lers +##ister +affiliation +drill +##ashi +panthers +scenic +libya +anita +strengthen +inscriptions +##cated +lace +sued +judith +riots +##uted +mint +##eta +preparations +midst +dub +challenger +##vich +mock +displaced +wicket +breaths +enables +schmidt +analyst +##lum +highlight +automotive +axe +josef +newark +sufficiently +resembles +50th +##pal +flushed +mum +traits +##ante +commodore +incomplete +warming +titular +ceremonial +ethical +celebrating +eighteenth +cao +lima +medalist +mobility +strips +snakes +miniature +zagreb +barton +escapes +umbrella +automated +doubted +differs +cooled +georgetown +dresden +cooked +fade +wyatt +jacobs +carlton +abundant +stereo +madras +inning +spur +malayalam +begged +osaka +groan +escaping +charging +dose +##aj +bud +papa +communists +advocates +edged +tri +resemble +peaking +necklace +fried +montenegro +saxony +goose +glances +stuttgart +curator +recruit +grocery +sympathetic +##tting +##fort +lotus +randolph +ancestor +##rand +succeeding +jupiter +1798 +macedonian +##heads +hiking +1808 +handing +fischer +##itive +garbage +##pies +prone +singular +papua +inclined +attractions +italia +pouring +motioned +grandma +garnered +jacksonville +corp +ego +ringing +aluminum +##hausen +ordering +##foot +drawer +traders +synagogue +##kawa +resistant +wandering +fragile +fiona +teased +hardcore +soaked +jubilee +decisive +exposition +mercer +poster +valencia +hale +kuwait +1811 +##ises +##wr +##eed +tavern +gamma +johan +##uer +airways +amino +gil +vocational +domains +torres +generator +folklore +outcomes +##keeper +canberra +shooter +fl +beams +confrontation +##gram +aligned +forestry +pipeline +jax +motorway +conception +decay +coffin +##cott +stalin +1805 +escorted +minded +##nam +sitcom +purchasing +twilight +veronica +additions +passive +tensions +straw +frequencies +1804 +refugee +cultivation +##iate +christie +clary +bulletin +crept +disposal +##rich +##zong +processor +crescent +##rol +emphasized +whale +nazis +aurora +dwelling +hauled +sponsors +toledo +ideology +theatres +tessa +cerambycidae +saves +turtle +cone +suspects +kara +rusty +yelling +greeks +mozart +shades +cocked +participant +shire +spit +freeze +necessity +##cos +inmates +nielsen +councillors +loaned +uncommon +omar +peasants +botanical +offspring +daniels +formations +jokes +1794 +pioneers +sigma +licensing +##sus +wheelchair +polite +1807 +liquor +pratt +trustee +##uta +forewings +balloon +kilometre +camping +explicit +casually +shawn +foolish +teammates +nm +hassan +carrie +judged +satisfy +vanessa +knives +selective +flowed +##lice +stressed +eliza +mathematician +cease +cultivated +##roy +commissions +browns +##ania +destroyers +sheridan +meadow +##rius +minerals +##cial +downstream +clash +gram +memoirs +ventures +baha +seymour +archie +midlands +edith +fare +flynn +invite +canceled +tiles +stabbed +boulder +incorporate +amended +camden +facial +mollusk +unreleased +descriptions +grabs +raises +ramp +shiver +##rose +coined +pioneering +tunes +qing +warwick +tops +melanie +giles +##rous +wandered +##inal +annexed +30th +unnamed +##ished +organizational +airplane +normandy +stoke +whistle +blessing +violations +chased +holders +shotgun +##ctic +reactor +##vik +tires +tearing +shores +fortified +mascot +constituencies +columnist +productive +tibet +##rta +lineage +hooked +tapes +judging +cody +##gger +hansen +kashmir +triggered +##eva +solved +cliffs +##tree +resisted +anatomy +protesters +transparent +implied +##iga +injection +mattress +excluding +##mbo +defenses +helpless +devotion +##elli +growl +liberals +weber +phenomena +atoms +plug +##iff +mortality +apprentice +howe +convincing +swimmer +barber +leone +promptly +sodium +def +nowadays +arise +##oning +gloucester +corrected +dignity +norm +erie +##ders +elders +evacuated +compression +##yar +hartford +backpack +reasoning +accepts +24th +wipe +millimetres +marcel +##oda +dodgers +albion +1790 +overwhelmed +aerospace +oaks +1795 +showcase +acknowledge +recovering +nolan +ashe +hurts +geology +fashioned +disappearance +farewell +swollen +shrug +marquis +wimbledon +rue +1792 +commemorate +reduces +experiencing +inevitable +calcutta +##court +murderer +sticking +fisheries +imagery +bloom +##inus +gustav +hesitation +memorable +viral +beans +accidents +tunisia +antenna +spilled +consort +treatments +aye +perimeter +##gard +donation +hostage +migrated +banker +addiction +apex +lil +trout +##ously +conscience +##nova +rams +sands +genome +passionate +troubles +##lets +amid +##ibility +##ret +higgins +exceed +vikings +##vie +payne +##zan +muscular +defendant +sucking +##wal +ibrahim +fuselage +claudia +vfl +europeans +snails +interval +##garh +preparatory +statewide +tasked +lacrosse +viktor +##lation +angola +##hra +flint +implications +employs +teens +patrons +stall +weekends +barriers +scrambled +nucleus +tehran +jenna +parsons +lifelong +robots +displacement +##bles +precipitation +knuckles +clutched +1802 +marrying +ecology +marx +accusations +declare +scars +kolkata +mat +meadows +bermuda +skeleton +finalists +vintage +crawl +coordinate +affects +subjected +orchestral +mistaken +mirrors +dipped +relied +arches +candle +##nick +incorporating +wildly +fond +basilica +owl +fringe +rituals +whispering +stirred +feud +tertiary +slick +goat +honorable +whereby +ricardo +stripes +parachute +adjoining +submerged +synthesizer +##gren +intend +positively +ninety +phi +beaver +partition +fellows +alexis +prohibition +carlisle +bizarre +fraternity +doubts +icy +aquatic +sneak +sonny +combines +airports +crude +supervised +spatial +merge +alfonso +##bic +corrupt +scan +undergo +##ams +disabilities +colombian +comparing +dolphins +perkins +reprinted +unanimous +bounced +hairs +underworld +midwest +semester +bucket +paperback +miniseries +coventry +demise +##leigh +demonstrations +sensor +rotating +yan +##hler +arrange +soils +##idge +hyderabad +labs +brakes +grandchildren +##nde +negotiated +rover +ferrari +continuation +directorate +augusta +stevenson +counterpart +gore +##rda +nursery +rican +ave +collectively +broadly +pastoral +repertoire +asserted +discovering +nordic +styled +fiba +cunningham +harley +middlesex +survives +tumor +tempo +zack +aiming +lok +urgent +##nto +devils +contractor +turin +##wl +bliss +repaired +simmons +moan +astronomical +negotiate +lyric +1890s +lara +bred +clad +angus +pbs +engineered +posed +hernandez +possessions +elbows +psychiatric +strokes +confluence +electorate +lifts +campuses +lava +alps +##ution +##date +physicist +woody +##ographic +##itis +juliet +reformation +sparhawk +complement +suppressed +jewel +##½ +floated +##kas +continuity +sadly +##ische +inability +melting +scanning +paula +flour +judaism +safer +vague +solving +curb +##stown +financially +gable +bees +expired +miserable +cassidy +dominion +1789 +cupped +robbery +facto +amos +warden +resume +tallest +marvin +pounded +declaring +gasoline +##aux +darkened +sophomore +##mere +erection +gossip +televised +risen +dial +##eu +pillars +passages +profound +arabian +ashton +silicon +nail +##lated +##hardt +fleming +firearms +ducked +circuits +blows +waterloo +titans +fireplace +cheshire +financed +activation +algorithms +constituent +catcher +cherokee +partnerships +sexuality +platoon +tragic +vivian +guarded +whiskey +meditation +poetic +##nga +porto +listeners +dominance +kendra +mona +chandler +factions +22nd +salisbury +attitudes +derivative +##ido +##haus +intake +paced +javier +illustrator +barrels +bias +cockpit +burnett +dreamed +ensuing +receptors +someday +hawkins +mattered +##lal +slavic +1799 +jesuit +cameroon +wasted +wax +lowering +victorious +freaking +outright +hancock +librarian +sensing +bald +calcium +myers +tablet +announcing +barack +shipyard +pharmaceutical +greenwich +flush +medley +patches +wolfgang +speeches +acquiring +exams +nikolai +hayden +kannada +reilly +waitress +abdomen +devastated +capped +pseudonym +pharmacy +fulfill +paraguay +1796 +clicked +##trom +archipelago +syndicated +##hman +lumber +orgasm +rejection +clifford +lorraine +advent +mafia +rodney +brock +##used +##elia +cassette +chamberlain +despair +mongolia +sensors +developmental +upstream +##alis +spanning +trombone +basque +seeded +interred +renewable +rhys +leapt +revision +molecule +##ages +chord +vicious +nord +shivered +23rd +arlington +debts +corpus +sunrise +bays +blackburn +centimetres +##uded +shuddered +strangely +gripping +cartoons +isabelle +orbital +##ppa +seals +proving +refusal +strengthened +bust +assisting +baghdad +batsman +portrayal +mara +pushes +spears +og +##cock +reside +nathaniel +brennan +1776 +confirmation +caucus +##worthy +markings +yemen +nobles +ku +lazy +viewer +catalan +encompasses +sawyer +##fall +sparked +substances +patents +braves +arranger +evacuation +sergio +persuade +dover +tolerance +penguin +cum +jockey +insufficient +townships +occupying +declining +plural +processed +projection +puppet +flanders +introduces +liability +##yon +gymnastics +antwerp +hobart +candles +jeep +wes +observers +chaplain +bundle +glorious +##hine +hazel +flung +sol +excavations +dumped +stares +bangalore +triangular +icelandic +intervals +expressing +turbine +##vers +songwriting +crafts +##igo +jasmine +ditch +rite +entertaining +comply +sorrow +wrestlers +basel +emirates +marian +rivera +helpful +##some +caution +downward +networking +##atory +##tered +darted +genocide +emergence +replies +specializing +spokesman +convenient +unlocked +fading +augustine +concentrations +resemblance +elijah +investigator +andhra +##uda +promotes +##rrell +fleeing +simone +announcer +lydia +weaver +residency +modification +##fest +stretches +alternatively +nat +lowe +lacks +##ented +pam +tile +concealed +inferior +abdullah +residences +tissues +vengeance +##ided +moisture +peculiar +groove +bologna +jennings +ninja +oversaw +zombies +pumping +batch +livingston +emerald +installations +1797 +peel +nitrogen +rama +##fying +schooling +strands +responding +werner +lime +casa +accurately +targeting +##rod +underway +##uru +hemisphere +lester +##yard +occupies +griffith +angrily +reorganized +##owing +courtney +deposited +estadio +##ifies +dunn +exiled +##ying +checks +##combe +successes +unexpectedly +blu +assessed +##flower +observing +sacked +spiders +kn +nodes +prosperity +audrey +divisional +broncos +tangled +adjust +feeds +erosion +paolo +surf +directory +snatched +humid +admiralty +screwed +reddish +##nese +modules +trench +lamps +bind +leah +bucks +competes +##nz +transcription +isles +violently +clutching +pga +cyclist +inflation +flats +ragged +unnecessary +##hian +stubborn +coordinated +harriet +baba +disqualified +insect +wolfe +##fies +reinforcements +rocked +duel +winked +embraced +bricks +##raj +hiatus +defeats +pending +brightly +jealousy +##xton +##uki +lena +colorful +##dley +stein +kidney +##shu +underwear +wanderers +##haw +##icus +guardians +m³ +roared +habits +##wise +permits +uranium +punished +disguise +bundesliga +elise +dundee +erotic +partisan +collectors +float +individually +rendering +behavioral +bucharest +ser +hare +valerie +corporal +nutrition +proportional +immense +##kis +pavement +##zie +##eld +sutherland +crouched +1775 +suzuki +trades +endurance +operas +crosby +prayed +priory +rory +socially +gujarat +walton +cube +pasha +privilege +lennon +floods +thorne +waterfall +nipple +scouting +approve +##lov +minorities +voter +dwight +extensions +assure +ballroom +slap +dripping +privileges +rejoined +confessed +demonstrating +patriotic +yell +investor +##uth +pagan +slumped +squares +confront +bert +embarrassment +aston +urging +sweater +starr +yuri +brains +williamson +commuter +mortar +structured +selfish +exports +##jon +cds +##him +unfinished +##rre +mortgage +destinations +##nagar +canoe +solitary +buchanan +delays +magistrate +fk +##pling +motivation +##lier +##vier +recruiting +assess +##mouth +malik +antique +1791 +pius +rahman +reich +tub +zhou +smashed +airs +galway +xii +conditioning +honduras +discharged +dexter +##pf +lionel +debates +lemon +volunteered +dioxide +procession +devi +sic +tremendous +advertisements +colts +transferring +verdict +hanover +decommissioned +utter +relate +pac +racism +beacon +limp +similarity +terra +occurrence +ant +becky +capt +updates +armament +richie +pal +##graph +halloween +mayo +##ssen +##bone +cara +serena +fcc +dolls +obligations +##dling +violated +lafayette +jakarta +exploitation +infamous +iconic +##lah +##park +moody +reginald +dread +spill +crystals +olivier +modeled +bluff +equilibrium +separating +notices +ordnance +extinction +onset +cosmic +attachment +sammy +expose +privy +anchored +##bil +abbott +admits +bending +baritone +emmanuel +policeman +vaughan +winged +climax +dresses +denny +polytechnic +mohamed +burmese +authentic +nikki +genetics +grandparents +homestead +gaza +postponed +metacritic +una +##sby +unstable +dissertation +##cian +curls +obscure +uncovered +bronx +praying +disappearing +##hoe +prehistoric +coke +turret +mutations +nonprofit +pits +monaco +##usion +prominently +dispatched +podium +##mir +uci +##uation +fortifications +birthplace +kendall +##lby +##oll +preacher +rack +goodman +persistent +##ott +countless +jaime +recorder +lexington +persecution +jumps +renewal +wagons +crushing +##holder +decorations +##lake +abundance +wrath +laundry +£1 +garde +jeanne +beetles +peasant +splitting +caste +sergei +##rer +##ema +scripts +##ively +rub +satellites +##vor +inscribed +verlag +scrapped +gale +packages +chick +potato +slogan +kathleen +arabs +##culture +counterparts +reminiscent +choral +##tead +rand +retains +bushes +dane +accomplish +courtesy +closes +##oth +slaughter +hague +krakow +lawson +tailed +elias +ginger +##ttes +canopy +betrayal +rebuilding +turf +##hof +frowning +allegiance +brigades +kicks +rebuild +polls +alias +nationalism +rowan +audition +bowie +fortunately +recognizes +harp +dillon +horrified +##oro +renault +ropes +presumed +rewarded +infrared +wiping +accelerated +illustration +presses +practitioners +badminton +##iard +detained +##tera +recognizing +relates +misery +##sies +##tly +reproduction +piercing +potatoes +thornton +esther +manners +hbo +##aan +ours +bullshit +ernie +perennial +sensitivity +illuminated +rupert +##iss +rfc +nassau +##dock +staggered +socialism +##haven +appointments +nonsense +prestige +sharma +haul +solidarity +##rata +igor +pedestrian +##uit +baxter +tenants +wires +medication +unlimited +guiding +impacts +diabetes +##rama +sasha +pas +clive +extraction +continually +constraints +##bilities +sonata +hunted +sixteenth +chu +planting +quote +mayer +pretended +spat +ceramic +##cci +curtains +pigs +pitching +##dad +latvian +sore +dayton +##sted +patrols +slice +playground +##nted +shone +stool +apparatus +inadequate +mates +treason +##ija +desires +##liga +##croft +somalia +laurent +mir +grape +obliged +chevrolet +thirteenth +stunning +enthusiastic +##ede +accounted +concludes +currents +basil +##kovic +drought +##rica +mai +##aire +shove +posting +##shed +pilgrimage +humorous +packing +fry +pencil +wines +smells +marilyn +aching +newest +clung +bon +neighbours +sanctioned +##pie +mug +##stock +drowning +hydraulic +##vil +hiring +reminder +lilly +investigators +##ncies +sour +##eous +compulsory +packet +##rion +##graphic +##elle +cannes +##inate +depressed +##rit +heroic +importantly +theresa +##tled +conway +saturn +marginal +rae +##xia +corresponds +royce +pact +jasper +explosives +packaging +aluminium +##ttered +denotes +rhythmic +spans +assignments +hereditary +outlined +originating +sundays +lad +reissued +greeting +beatrice +##dic +pillar +marcos +plots +handbook +alcoholic +judiciary +avant +slides +extract +masculine +blur +##eum +homage +trembled +owens +hymn +trey +signaling +socks +accumulated +reacted +attic +theo +lining +angie +distraction +primera +talbot +creativity +billed +##hey +deacon +eduardo +identifies +proposition +dizzy +gunner +hogan +##yam +##pping +##hol +ja +##chan +jensen +reconstructed +##berger +clearance +darius +##nier +abe +harlem +plea +dei +circled +emotionally +notation +fascist +neville +exceeded +upwards +viable +ducks +workforce +racer +limiting +shri +##lson +possesses +kerr +moths +devastating +laden +disturbing +locking +gal +fearing +accreditation +flavor +aide +1870s +mountainous +##baum +melt +##ures +texture +servers +soda +herd +##nium +erect +puzzled +hum +peggy +examinations +gould +testified +geoff +ren +devised +sacks +##law +denial +posters +grunted +cesar +tutor +gerry +offerings +byrne +falcons +combinations +incoming +pardon +rocking +26th +avengers +flared +mankind +seller +uttar +loch +nadia +stroking +exposing +fertile +ancestral +instituted +##has +noises +prophecy +taxation +eminent +vivid +pol +##bol +dart +indirect +multimedia +notebook +upside +displaying +adrenaline +referenced +geometric +##iving +progression +##ddy +blunt +announce +##far +implementing +##lav +aggression +liaison +cooler +cares +headache +plantations +gorge +dots +impulse +thickness +ashamed +averaging +kathy +obligation +precursor +fowler +symmetry +thee +hears +##rai +undergoing +butcher +bowler +##lip +cigarettes +subscription +goodness +##ically +browne +##hos +kyoto +donor +##erty +damaging +friction +drifting +expeditions +hardened +prostitution +fauna +blankets +claw +tossing +snarled +butterflies +recruits +investigative +coated +healed +communal +hai +xiii +academics +boone +psychologist +restless +lahore +stephens +brendan +foreigners +printer +ached +explode +27th +deed +scratched +dared +##pole +cardiac +1780 +okinawa +proto +commando +compelled +oddly +electrons +replica +thanksgiving +##rist +sheila +deliberate +stafford +tidal +representations +hercules +ou +##path +##iated +kidnapping +lenses +##tling +deficit +samoa +mouths +consuming +computational +maze +granting +smirk +razor +fixture +ideals +inviting +aiden +nominal +issuing +julio +pitt +ramsey +docks +##oss +exhaust +##owed +bavarian +draped +anterior +mating +ethiopian +explores +noticing +##nton +discarded +convenience +hoffman +endowment +beasts +cartridge +mormon +paternal +probe +sleeves +interfere +lump +deadline +jenks +bulldogs +scrap +alternating +justified +reproductive +nam +seize +descending +secretariat +kirby +grouped +smash +panther +sedan +tapping +lola +cheer +germanic +unfortunate +##eter +unrelated +##fan +subordinate +##sdale +suzanne +advertisement +##ility +horsepower +##lda +cautiously +discourse +luigi +##mans +##fields +noun +prevalent +mao +schneider +everett +surround +governorate +kira +##avia +westward +##take +misty +rails +sustainability +unused +##rating +packs +toast +unwilling +regulate +thy +suffrage +nile +awe +assam +definitions +travelers +affordable +##rb +conferred +sells +undefeated +beneficial +torso +basal +repeating +remixes +bahrain +cables +fang +##itated +excavated +numbering +statutory +deluxe +##lian +forested +ramirez +derbyshire +zeus +slamming +transfers +astronomer +banana +lottery +berg +histories +bamboo +##uchi +resurrection +posterior +bowls +vaguely +##thi +thou +preserving +tensed +offence +##inas +meyrick +callum +ridden +watt +langdon +tying +lowland +snorted +daring +truman +##hale +##girl +aura +overly +filing +weighing +goa +infections +philanthropist +saunders +eponymous +##owski +latitude +perspectives +reviewing +mets +commandant +radial +##kha +flashlight +reliability +koch +vowels +amazed +ada +elaine +supper +##encies +predator +debated +soviets +cola +##boards +##nah +compartment +crooked +arbitrary +fourteenth +havana +majors +steelers +clips +profitable +ambush +exited +packers +##tile +nude +cracks +fungi +limb +trousers +josie +shelby +tens +frederic +##ος +definite +smoothly +constellation +insult +baton +discs +lingering +##nco +conclusions +lent +staging +becker +grandpa +shaky +##tron +einstein +obstacles +adverse +economically +##moto +mccartney +thor +dismissal +motions +readings +nostrils +treatise +##pace +squeezing +evidently +prolonged +1783 +venezuelan +je +marguerite +beirut +takeover +shareholders +##vent +denise +digit +airplay +norse +##bbling +imaginary +pills +hubert +blaze +vacated +eliminating +vine +mansfield +retrospective +barrow +borne +clutch +bail +forensic +weaving +##nett +##witz +desktop +citadel +promotions +worrying +dorset +subdivided +##iating +manned +expeditionary +pickup +synod +chuckle +barney +##rz +##ffin +functionality +karachi +litigation +meanings +lick +anders +##ffed +execute +curl +oppose +ankles +typhoon +##ache +linguistics +compassion +pressures +grazing +perfection +##iting +immunity +monopoly +muddy +backgrounds +namibia +francesca +monitors +attracting +stunt +tuition +##ии +vegetable +##mates +##quent +mgm +jen +complexes +forts +cellar +bites +seventeenth +royals +flemish +failures +mast +charities +##cular +peruvian +capitals +macmillan +ipswich +outward +frigate +postgraduate +folds +employing +##ouse +concurrently +fiery +##tai +contingent +nightmares +monumental +nicaragua +##kowski +lizard +mal +fielding +gig +reject +harding +##ipe +coastline +##cin +beethoven +humphrey +innovations +##tam +norris +doris +solicitor +obey +niagara +shelves +bourbon +nightclub +specifications +hilton +##ndo +centennial +dispersed +worm +neglected +briggs +kuala +uneasy +##nstein +##bound +##aking +##burgh +awaiting +pronunciation +##bbed +##quest +eh +optimal +zhu +raped +greens +presided +brenda +worries +venetian +marxist +turnout +##lius +refined +braced +sins +grasped +sunderland +nickel +speculated +lowell +cyrillic +communism +fundraising +resembling +colonists +mutant +freddie +usc +##mos +gratitude +##run +mural +##lous +chemist +reminds +28th +steals +tess +pietro +##ingen +promoter +ri +microphone +honoured +rai +sant +##qui +feather +##nson +burlington +kurdish +terrorists +deborah +sickness +##wed +hazard +irritated +desperation +veil +clarity +##rik +jewels +xv +##gged +##ows +##cup +berkshire +unfair +mysteries +orchid +winced +exhaustion +renovations +stranded +obe +infinity +##nies +adapt +redevelopment +thanked +registry +olga +domingo +noir +tudor +ole +commenting +behaviors +##ais +crisp +pauline +probable +stirling +wigan +paralympics +panting +surpassed +##rew +luca +barred +famed +##sters +cassandra +waiter +carolyn +exported +##orted +andres +destructive +deeds +jonah +castles +vacancy +##glass +1788 +orchard +yep +famine +belarusian +sprang +##forth +skinny +##mis +administrators +rotterdam +zambia +zhao +boiler +discoveries +##ride +##physics +lucius +disappointing +outreach +spoon +##frame +qualifications +unanimously +enjoys +regency +##iidae +stade +realism +veterinary +rodgers +dump +alain +chestnut +castile +censorship +rumble +gibbs +communion +reggae +inactivated +logs +loads +##houses +homosexual +##iano +ale +informs +##cas +phrases +plaster +linebacker +ambrose +kaiser +fascinated +limerick +recruitment +forge +mastered +##nding +leinster +rooted +threaten +##strom +borneo +##hes +suggestions +scholarships +propeller +documentaries +patronage +coats +constructing +invest +neurons +comet +entirety +shouts +identities +annoying +unchanged +wary +##antly +##ogy +neat +oversight +##kos +phillies +replay +constance +##kka +incarnation +humble +skies +minus +##acy +smithsonian +guerrilla +jar +cadets +##plate +surplus +audit +##aru +cracking +joanna +louisa +pacing +##lights +intentionally +##iri +diner +nwa +imprint +australians +tong +unprecedented +bunker +naive +specialists +ark +nichols +railing +leaked +pedal +##uka +shrub +longing +roofs +captains +neural +tuned +##ntal +##jet +emission +medina +frantic +codex +definitive +sid +abolition +intensified +stocks +enrique +sustain +genoa +oxide +##written +clues +cha +##gers +tributaries +fragment +venom +##ente +##sca +muffled +vain +sire +laos +##ingly +##hana +hastily +snapping +surfaced +sentiment +motive +##oft +contests +approximate +mesa +luckily +dinosaur +exchanges +propelled +accord +bourne +relieve +tow +masks +offended +##ues +cynthia +##mmer +rains +bartender +zinc +reviewers +lois +##sai +legged +arrogant +rafe +comprise +handicap +blockade +inlet +lagoon +copied +drilling +shelley +petals +##inian +mandarin +obsolete +##inated +onward +arguably +productivity +praising +seldom +busch +discusses +raleigh +shortage +ranged +stanton +encouragement +firstly +conceded +overs +temporal +##uke +cbe +##bos +woo +certainty +pumps +##pton +stalked +##uli +lizzie +periodic +thieves +weaker +gases +shoving +chooses +wc +##chemical +prompting +weights +##kill +robust +flanked +sticky +tuberculosis +##eb +##eal +christchurch +resembled +wallet +reese +inappropriate +pictured +distract +fixing +fiddle +giggled +burger +heirs +hairy +mechanic +torque +obsessed +chiefly +cheng +logging +extracted +meaningful +numb +##vsky +gloucestershire +reminding +unite +##lit +breeds +diminished +clown +glove +1860s +archibald +focal +freelance +sliced +depiction +##yk +organism +switches +sights +stray +crawling +##ril +lever +leningrad +interpretations +loops +anytime +reel +alicia +delighted +##ech +inhaled +xiv +suitcase +bernie +vega +licenses +northampton +exclusion +induction +monasteries +racecourse +homosexuality +##sfield +##rky +dimitri +michele +alternatives +ions +commentators +genuinely +objected +pork +hospitality +fencing +stephan +warships +peripheral +wit +drunken +wrinkled +quentin +spends +departing +chung +numerical +spokesperson +johannesburg +caliber +killers +##udge +assumes +neatly +demographic +abigail +bloc +mounting +##lain +bentley +slightest +xu +recipients +##jk +merlin +##writer +seniors +prisons +blinking +hindwings +flickered +kappa +##hel +80s +strengthening +appealing +brewing +gypsy +mali +lashes +hulk +unpleasant +harassment +bio +treaties +predict +instrumentation +pulp +troupe +boiling +mantle +##ffe +##vn +dividing +handles +verbs +##onal +coconut +senegal +thorough +gum +momentarily +##sto +cocaine +panicked +destined +##turing +teatro +denying +weary +captained +mans +##hawks +wakefield +bollywood +thankfully +cyril +amendments +##bahn +consultation +stud +reflections +kindness +1787 +internally +##ovo +tex +mosaic +distribute +paddy +seeming +##hic +piers +##mura +popularly +winger +kang +sentinel +mccoy +##anza +covenant +##bag +verge +fireworks +suppress +thrilled +dominate +##jar +swansea +reconciliation +stiffened +cue +dorian +##uf +damascus +amor +ida +foremost +##aga +porsche +unseen +dir +##had +##azi +stony +lexi +melodies +##nko +angular +integer +podcast +ants +inherent +jaws +justify +persona +##olved +josephine +##nr +##ressed +customary +flashes +gala +cyrus +glaring +backyard +ariel +physiology +greenland +stir +avon +atletico +finch +methodology +ked +mas +catholicism +townsend +branding +quincy +fits +containers +1777 +ashore +aragon +forearm +poisoning +adopting +conquer +grinding +amnesty +keller +finances +evaluate +forged +lankan +instincts +##uto +guam +bosnian +photographed +workplace +desirable +protector +allocation +intently +encourages +willy +##sten +bodyguard +electro +brighter +bihar +##chev +lasts +opener +amphibious +sal +verde +arte +##cope +captivity +vocabulary +yields +##tted +agreeing +desmond +pioneered +##chus +strap +campaigned +railroads +##ович +emblem +##dre +stormed +##ulous +marijuana +northumberland +##nath +bowen +landmarks +beaumont +##qua +danube +##bler +attorneys +th +flyers +critique +villains +cass +mutation +acc +##0s +colombo +mckay +motif +sampling +concluding +syndicate +##rell +neon +stables +warnings +clint +mourning +wilkinson +##tated +merrill +leopard +evenings +exhaled +emil +sonia +ezra +discrete +stove +farrell +fifteenth +prescribed +superhero +##rier +worms +helm +wren +##duction +expo +##rator +hq +unfamiliar +antony +prevents +acceleration +fiercely +mari +painfully +calculations +cheaper +ign +clifton +irvine +davenport +mozambique +pierced +##evich +wonders +##wig +##cate +##iling +crusade +ware +enzymes +reasonably +mls +##coe +mater +ambition +bunny +eliot +kernel +##fin +asphalt +headmaster +torah +aden +lush +pins +waived +##yas +joao +substrate +enforce +##grad +##ules +alvarez +selections +epidemic +tempted +bremen +translates +ensured +waterfront +29th +forrest +manny +malone +kramer +reigning +simpler +absorption +engraved +##ffy +evaluated +1778 +haze +comforting +crossover +##abe +thorn +##rift +##imo +suppression +fatigue +cutter +wurttemberg +##orf +enforced +hovering +proprietary +samurai +syllable +ascent +lacey +tick +lars +tractor +merchandise +rep +bouncing +defendants +##yre +huntington +##oko +standardized +##hor +##hima +assassinated +predecessors +rainy +liar +assurance +lyrical +##uga +secondly +flattened +parameter +undercover +##mity +bordeaux +punish +ridges +markers +exodus +inactive +hesitate +debbie +nyc +pledge +savoy +nagar +offset +organist +##tium +hesse +marin +converting +##iver +diagram +propulsion +validity +reverted +supportive +ministries +clans +responds +proclamation +##inae +ein +pleading +patriot +birch +islanders +strauss +hates +##dh +brandenburg +concession +1900s +killings +textbook +antiquity +cinematography +wharf +embarrassing +setup +creed +farmland +inequality +centred +signatures +fallon +##ingham +##uts +ceylon +gazing +directive +laurie +##tern +globally +##uated +##dent +allah +excavation +threads +##cross +frantically +icc +utilize +determines +respiratory +thoughtful +receptions +##dicate +merging +chandra +seine +builders +builds +diagnostic +dev +visibility +goddamn +analyses +dhaka +proves +chancel +concurrent +curiously +canadians +pumped +restoring +1850s +turtles +jaguar +sinister +spinal +declan +vows +1784 +glowed +capitalism +swirling +universidad +##lder +##oat +soloist +##genic +##oor +coincidence +beginnings +nissan +dip +resorts +caucasus +combustion +infectious +##eno +pigeon +serpent +##itating +conclude +masked +salad +jew +##gr +surreal +toni +##wc +harmonica +##gins +##etic +##coat +fishermen +intending +bravery +##wave +klaus +titan +wembley +taiwanese +ransom +40th +incorrect +hussein +eyelids +cooke +dramas +utilities +##etta +##print +eisenhower +principally +granada +lana +##rak +openings +concord +##bl +bethany +connie +morality +sega +##mons +##nard +earnings +##kara +##cine +communes +##rel +coma +composing +softened +severed +grapes +nguyen +analyzed +warlord +hubbard +heavenly +behave +slovenian +##hit +##ony +hailed +filmmakers +trance +caldwell +skye +unrest +coward +likelihood +##aging +bern +taliban +honolulu +propose +browser +imagining +cobra +contributes +dukes +instinctively +conan +violinist +##ores +accessories +gradual +##amp +quotes +sioux +##dating +undertake +intercepted +sparkling +compressed +fungus +tombs +haley +imposing +rests +degradation +lincolnshire +retailers +wetlands +tulsa +distributor +dungeon +nun +greenhouse +convey +atlantis +aft +exits +oman +dresser +lyons +##sti +joking +eddy +judgement +omitted +digits +##game +juniors +##rae +cents +stricken +une +##ngo +wizards +weir +breton +nan +technician +fibers +liking +royalty +persia +terribly +magician +##rable +##unt +vance +cafeteria +booker +camille +warmer +##static +consume +cavern +gaps +compass +contemporaries +foyer +soothing +graveyard +maj +plunged +blush +##wear +cascade +demonstrates +ordinance +##nov +boyle +##lana +rockefeller +shaken +banjo +izzy +##ense +breathless +vines +##eman +alterations +chromosome +dwellings +feudal +mole +catalonia +relics +tenant +mandated +##fm +fridge +hats +honesty +patented +raul +heap +cruisers +accusing +enlightenment +infants +wherein +chatham +contractors +affinity +hc +osborne +piston +traps +maturity +##rana +lagos +##zal +peering +##nay +attendant +dealers +protocols +subset +prospects +biographical +##cre +artery +##zers +insignia +nuns +endured +##eration +recommend +schwartz +serbs +berger +cromwell +crossroads +enduring +clasped +grounded +##bine +marseille +twitched +abel +choke +catalyst +moldova +italians +##tist +disastrous +wee +##oured +##nti +wwf +nope +##piration +##asa +expresses +thumbs +##nza +coca +1781 +cheating +##ption +skipped +sensory +heidelberg +spies +satan +dangers +semifinal +bohemia +whitish +confusing +shipbuilding +relies +surgeons +landings +ravi +baku +moor +suffix +alejandro +##yana +litre +upheld +##unk +rajasthan +##rek +coaster +insists +posture +scenarios +etienne +favoured +appoint +transgender +elephants +poked +greenwood +defences +fulfilled +militant +somali +1758 +chalk +potent +##ucci +migrants +wink +assistants +nos +restriction +activism +niger +##ario +colon +shaun +##sat +daphne +##erated +swam +congregations +reprise +considerations +magnet +playable +xvi +overthrow +tobias +knob +chavez +coding +##mers +propped +katrina +orient +newcomer +##suke +temperate +##pool +farmhouse +interrogation +committing +##vert +forthcoming +strawberry +joaquin +macau +ponds +shocking +siberia +##cellular +chant +contributors +##nant +##ologists +sped +absorb +hail +1782 +spared +##hore +barbados +karate +opus +originates +saul +##xie +evergreen +leaped +##rock +correlation +exaggerated +weekday +unification +bump +tracing +brig +afb +pathways +utilizing +disturbance +kneeling +##stad +##guchi +100th +pune +##thy +decreasing +manipulation +miriam +academia +ecosystem +occupational +rbi +##lem +rift +rotary +stacked +incorporation +awakening +generators +guerrero +racist +##omy +cyber +derivatives +culminated +allie +annals +panzer +sainte +pops +zu +austro +##vate +algerian +politely +nicholson +mornings +educate +tastes +thrill +dartmouth +##gating +##jee +regan +differing +concentrating +choreography +divinity +pledged +alexandre +routing +gregor +madeline +##idal +apocalypse +##hora +gunfire +culminating +elves +fined +liang +lam +programmed +tar +guessing +transparency +gabrielle +##gna +cancellation +flexibility +##lining +accession +shea +stronghold +nets +specializes +##rgan +abused +hasan +sgt +exceeding +admiration +supermarket +photographers +specialised +tilt +resonance +hmm +perfume +sami +threatens +garland +botany +guarding +boiled +greet +puppy +russo +supplier +wilmington +vibrant +vijay +##bius +paralympic +grumbled +paige +faa +licking +margins +hurricanes +##gong +fest +grenade +ripping +##uz +counseling +weigh +##sian +needles +wiltshire +edison +costly +##not +fulton +tramway +redesigned +staffordshire +gasping +watkins +sleepy +candidacy +monkeys +timeline +throbbing +##bid +##sos +berth +uzbekistan +vanderbilt +bothering +overturned +ballots +gem +##iger +sunglasses +subscribers +hooker +compelling +ang +exceptionally +saloon +stab +##rdi +carla +terrifying +##vision +coil +##oids +satisfying +vendors +31st +mackay +deities +overlooked +ambient +bahamas +felipe +olympia +whirled +botanist +advertised +tugging +disciples +morales +unionist +rites +foley +morse +motives +creepy +##₀ +soo +##sz +bargain +highness +frightening +turnpike +tory +reorganization +depict +biographer +unopposed +manifesto +##gles +institut +emile +accidental +kapoor +##dam +kilkenny +cortex +lively +romanesque +jain +shan +cannons +##ske +petrol +echoing +amalgamated +disappears +cautious +proposes +sanctions +trenton +flotilla +aus +contempt +tor +canary +cote +theirs +##hun +conceptual +deleted +fascinating +paso +blazing +elf +honourable +hutchinson +##eiro +##outh +##zin +surveyor +amidst +wooded +reissue +intro +##ono +cobb +shelters +newsletter +hanson +brace +encoding +confiscated +dem +caravan +marino +scroll +melodic +cows +imam +##adi +##aneous +northward +searches +biodiversity +cora +roaring +##bers +connell +theologian +halo +compose +pathetic +unmarried +dynamo +az +calculation +toulouse +deserves +humour +nr +forgiveness +tam +undergone +martyr +pamela +myths +whore +counselor +hicks +heavens +battleship +electromagnetic +stellar +establishments +presley +hopped +##chin +temptation +90s +wills +##yuan +nhs +##nya +seminars +##yev +adaptations +gong +asher +lex +indicator +sikh +tobago +cites +goin +##yte +satirical +##gies +characterised +correspond +bubbles +lure +participates +##vid +eruption +skate +therapeutic +1785 +canals +wholesale +defaulted +sac +petit +##zzled +virgil +leak +ravens +portraying +##yx +ghetto +creators +dams +portray +vicente +##rington +fae +namesake +bounty +##arium +joachim +##ota +##iser +aforementioned +axle +snout +depended +dismantled +reuben +##ibly +gallagher +##lau +earnest +##ieu +##iary +inflicted +objections +##llar +asa +gritted +##athy +jericho +##sea +##was +flick +underside +ceramics +undead +substituted +eastward +undoubtedly +wheeled +chimney +##iche +guinness +siding +traitor +baptiste +disguised +inauguration +tipperary +choreographer +perched +warmed +stationary +##ntes +bacterial +##aurus +flores +phosphate +attacker +invaders +alvin +intersects +indirectly +immigrated +businessmen +cornelius +valves +narrated +pill +sober +nationale +monastic +applicants +scenery +##jack +motifs +constitutes +##osh +jurisdictions +tuning +irritation +woven +##uddin +fertility +gao +##erie +antagonist +impatient +glacial +hides +boarded +denominations +interception +##jas +nicola +algebraic +marquess +bahn +parole +buyers +bait +turbines +paperwork +bestowed +natasha +renee +oceans +purchases +vaccine +##tock +fixtures +playhouse +integrate +jai +oswald +intellectuals +booked +nests +mortimer +##isi +obsession +sept +##gler +##sum +scrutiny +simultaneous +squinted +##shin +collects +oven +shankar +penned +remarkably +slips +luggage +spectral +1786 +collaborations +louie +consolidation +##ailed +##ivating +hoover +blackpool +harness +ignition +vest +tails +belmont +mongol +skinner +##nae +visually +mage +derry +##tism +##unce +stevie +transitional +##rdy +redskins +drying +prep +prospective +annoyance +oversee +##loaded +fills +##books +announces +scowled +respects +prasad +mystic +tucson +##vale +revue +springer +bankrupt +1772 +aristotle +habsburg +##geny +dal +natal +nut +pod +chewing +darts +moroccan +walkover +rosario +lenin +punjabi +##ße +grossed +scattering +wired +invasive +hui +polynomial +corridors +wakes +gina +portrays +##cratic +arid +retreating +erich +irwin +sniper +##dha +linen +lindsey +maneuver +butch +shutting +socio +bounce +commemorative +postseason +jeremiah +pines +mystical +beads +abbas +furnace +bidding +consulted +assaulted +empirical +rubble +enclosure +sob +weakly +cancel +polly +yielded +##emann +curly +prediction +battered +70s +vhs +jacqueline +render +sails +barked +detailing +grayson +riga +sloane +raging +##yah +herbs +bravo +##athlon +alloy +giggle +imminent +suffers +assumptions +waltz +##itate +accomplishments +##ited +bathing +remixed +deception +##emia +deepest +##eis +balkan +frogs +##rong +slab +##pate +philosophers +peterborough +grains +imports +dickinson +rwanda +##atics +1774 +dirk +tablets +##rove +clone +##rice +caretaker +hostilities +mclean +##gre +regimental +treasures +norms +impose +tsar +tango +diplomacy +variously +complain +recognise +arrests +1779 +celestial +pulitzer +##dus +libretto +##moor +adele +splash +expectation +lds +confronts +##izer +spontaneous +harmful +wedge +entrepreneurs +buyer +bilingual +translate +rugged +conner +circulated +uae +eaton +##gra +##zzle +lingered +lockheed +vishnu +reelection +alonso +##oom +joints +yankee +headline +cooperate +heinz +laureate +invading +##sford +echoes +scandinavian +##dham +hugging +vitamin +salute +micah +hind +trader +##sper +radioactive +##ndra +militants +poisoned +ratified +remark +campeonato +deprived +wander +prop +##dong +##tani +##eye +chiang +darcy +##oping +mandolin +spice +statesman +babylon +walled +forgetting +afro +##cap +giorgio +buffer +##polis +planetary +##gis +overlap +terminals +kinda +centenary +##bir +arising +manipulate +elm +ke +1770 +##tad +chrysler +mapped +moose +pomeranian +quad +macarthur +assemblies +shoreline +recalls +stratford +##rted +noticeable +##evic +imp +##rita +##sque +accustomed +supplying +tents +disgusted +sipped +filters +khz +reno +selecting +luftwaffe +mcmahon +tyne +masterpiece +carriages +collided +dunes +exercised +flare +remembers +muzzle +heck +##rson +burgess +lunged +middleton +boycott +bilateral +##sity +hazardous +lumpur +multiplayer +spotlight +jackets +goldman +liege +porcelain +rag +waterford +attracts +hopeful +battling +ottomans +kensington +baked +hymns +cheyenne +lattice +levine +borrow +polymer +clashes +michaels +monitored +commitments +denounced +##von +cavity +##oney +hobby +akin +##holders +futures +intricate +cornish +patty +##oned +illegally +dolphin +##lag +barlow +yellowish +maddie +apologized +luton +plagued +##puram +##rds +sway +fanny +łodz +##rino +psi +suspicions +hanged +##eding +initiate +charlton +##por +nak +competent +analytical +annex +wardrobe +reservations +sect +fairfax +hedge +piled +buckingham +uneven +bauer +simplicity +snyder +interpret +accountability +donors +moderately +byrd +continents +##cite +disciple +jamaican +nominees +##uss +mongolian +diver +attackers +eagerly +ideological +pillows +miracles +apartheid +revolver +sulfur +clinics +moran +##enko +ile +katy +rhetoric +##icated +chronology +recycling +##hrer +elongated +mughal +pascal +profiles +vibration +databases +domination +##fare +matthias +digest +rehearsal +polling +weiss +initiation +reeves +clinging +flourished +impress +##hoff +buckley +symposium +rhythms +weed +emphasize +transforming +##taking +##yman +accountant +analyze +flicker +foil +priesthood +voluntarily +decreases +##hya +slater +sv +charting +mcgill +##lde +moreno +besieged +zur +robes +##phic +admitting +deported +turmoil +peyton +earthquakes +##ares +nationalists +beau +clair +brethren +interrupt +welch +curated +galerie +requesting +##ested +impending +steward +viper +##vina +complaining +beautifully +brandy +foam +nl +1660 +alessandro +punches +laced +explanations +##lim +attribute +clit +reggie +discomfort +##cards +smoothed +whales +##cene +adler +countered +duffy +disciplinary +widening +recipe +reliance +conducts +goats +gradient +preaching +##shaw +matilda +quasi +striped +meridian +cannabis +cordoba +certificates +##agh +##tering +graffiti +hangs +pilgrims +repeats +##ych +revive +urine +etat +##hawk +fueled +belts +fuzzy +susceptible +mauritius +salle +sincere +beers +hooks +##cki +arbitration +entrusted +advise +sniffed +seminar +junk +donnell +processors +principality +strapped +celia +mendoza +everton +fortunes +prejudice +starving +reassigned +steamer +##lund +tuck +evenly +foreman +##ffen +dans +envisioned +slit +baseman +liberia +rosemary +##weed +electrified +periodically +potassium +stride +contexts +sperm +slade +mariners +influx +bianca +subcommittee +##rane +spilling +icao +estuary +##nock +delivers +##ulata +isa +mira +bohemian +dessert +##sbury +welcoming +proudly +slowing +##chs +musee +ascension +russ +##vian +waits +##psy +africans +exploit +##morphic +eccentric +crab +peck +entrances +formidable +marketplace +groom +bolted +metabolism +patton +robbins +courier +payload +endure +##ifier +andes +refrigerator +ornate +##uca +ruthless +illegitimate +masonry +strasbourg +bikes +apples +quintet +willingly +niche +bakery +corpses +energetic +##cliffe +##sser +##ards +centimeters +centro +fuscous +cretaceous +rancho +##yde +andrei +telecom +tottenham +oasis +ordination +vulnerability +presiding +corey +penguins +sims +##pis +malawi +piss +correction +##cked +##ffle +##ryn +countdown +detectives +psychiatrist +psychedelic +dinosaurs +blouse +choi +vowed +randomly +##pol +49ers +scrub +blanche +bruins +dusseldorf +##using +unwanted +##ums +dominique +elevations +headlights +om +laguna +##oga +1750 +famously +ignorance +shrewsbury +breuning +che +confederacy +greco +overhaul +##screen +paz +skirts +disagreement +cruelty +jagged +phoebe +shifter +hovered +viruses +##wes +##lined +landlord +squirrel +dashed +ornamental +gag +wally +grange +literal +spurs +undisclosed +proceeding +billie +orphan +spanned +humidity +indy +weighted +presentations +explosions +lucian +##tary +vaughn +hindus +##anga +##hell +psycho +daytona +protects +efficiently +rematch +sly +tandem +##oya +rebranded +impaired +hee +metropolis +peach +godfrey +diaspora +ethnicity +prosperous +gleaming +dar +grossing +playback +##rden +stripe +pistols +##tain +births +labelled +##cating +rudy +alba +##onne +aquarium +hostility +##tase +shudder +sumatra +hardest +lakers +consonant +creeping +demos +homicide +capsule +zeke +liberties +expulsion +pueblo +##comb +trait +transporting +##ddin +##neck +##yna +depart +gregg +mold +ledge +hangar +oldham +playboy +termination +analysts +gmbh +romero +##itic +insist +cradle +filthy +brightness +slash +shootout +deposed +bordering +##truct +microwave +tumbled +sheltered +cathy +werewolves +messy +andersen +convex +clapped +clinched +satire +wasting +edo +rufus +##jak +mont +##etti +poznan +##keeping +restructuring +transverse +##rland +azerbaijani +slovene +gestures +roommate +choking +shear +##quist +vanguard +oblivious +##hiro +disagreed +baptism +##lich +coliseum +##aceae +salvage +societe +cory +locke +relocation +relying +versailles +ahl +swelling +##elo +cheerful +##edes +gin +sarajevo +obstacle +diverted +##nac +messed +thoroughbred +fluttered +utrecht +chewed +acquaintance +assassins +dispatch +mirza +##wart +salzburg +swell +yen +##gee +idle +ligue +samson +##nds +##igh +playful +spawned +##cise +tease +##case +burgundy +stirring +skeptical +interceptions +marathi +##dies +bedrooms +aroused +pinch +##lik +preferences +tattoos +buster +digitally +projecting +rust +##ital +kitten +priorities +addison +pseudo +##guard +dusk +icons +sermon +##psis +##iba +##lift +ju +truce +rink +##dah +##wy +defects +psychiatry +offences +calculate +glucose +##iful +##rized +##unda +francaise +##hari +richest +warwickshire +carly +1763 +purity +redemption +lending +##cious +muse +bruises +cerebral +aero +carving +preface +terminology +invade +monty +anarchist +blurred +##iled +rossi +treats +guts +shu +foothills +ballads +undertaking +premise +cecilia +affiliates +blasted +conditional +wilder +minors +drone +rudolph +buffy +swallowing +horton +attested +rutherford +howell +primetime +livery +penal +##bis +minimize +hydro +wrecked +wrought +palazzo +##gling +cans +vernacular +friedman +nobleman +shale +walnut +danielle +##ection +##tley +sears +##kumar +chords +lend +flipping +streamed +por +dracula +gallons +sacrifices +gamble +orphanage +##iman +mckenzie +##gible +boxers +daly +##balls +##ان +##ific +##rative +##iq +exploited +slated +##uity +circling +hillary +pinched +goldberg +provost +campaigning +piles +ironically +jong +mohan +successors +usaf +##tem +##ught +autobiographical +haute +preserves +##ending +acquitted +comparisons +hydroelectric +gangs +cypriot +torpedoes +rushes +derive +bumps +instability +fiat +pets +##mbe +silas +dye +reckless +settler +##itation +heats +##writing +canonical +maltese +fins +mushroom +stacy +aspen +avid +##kur +##loading +vickers +gaston +hillside +statutes +wilde +gail +kung +sabine +comfortably +motorcycles +##rgo +pneumonia +fetch +##sonic +axel +faintly +parallels +##oop +mclaren +spouse +compton +interdisciplinary +miner +##eni +clamped +##chal +##llah +separates +versa +##mler +scarborough +labrador +##lity +##osing +rutgers +hurdles +como +burt +divers +wichita +cade +coincided +bruised +mla +vineyard +##ili +##brush +notch +mentioning +jase +hearted +kits +doe +##acle +pomerania +##ady +ronan +seizure +pavel +problematic +##zaki +domenico +##ulin +catering +penelope +dependence +parental +emilio +ministerial +atkinson +##bolic +clarkson +chargers +colby +grill +peeked +arises +summon +##aged +fools +##grapher +faculties +qaeda +##vial +garner +refurbished +##hwa +geelong +disasters +nudged +bs +shareholder +lori +algae +reinstated +rot +##ades +##nous +invites +stainless +inclusive +##itude +diocesan +til +##icz +denomination +##xa +benton +floral +registers +##erman +##kell +absurd +brunei +guangzhou +hitter +retaliation +##uled +##eve +blanc +nh +consistency +contamination +##eres +dire +palermo +broadcasters +diaries +inspire +vols +brewer +tightening +mixtape +hormone +##tok +stokes +##color +##dly +##ssi +##ometer +##lington +sanitation +##tility +intercontinental +##adt +¹⁄₂ +cylinders +economies +favourable +unison +croix +gertrude +odyssey +vanity +dangling +##logists +upgrades +dice +middleweight +practitioner +henrik +parlor +orion +angered +lac +blurted +##rri +sensual +intends +swings +angled +##phs +husky +attain +peerage +precinct +textiles +cheltenham +shuffled +dai +confess +tasting +bhutan +##riation +tyrone +segregation +abrupt +ruiz +##rish +smirked +blackwell +confidential +browning +amounted +vase +scarce +fabulous +raided +staple +guyana +unemployed +glider +shay +##tow +carmine +troll +intervene +squash +superstar +cylindrical +len +roadway +researched +handy +##rium +##jana +lao +declares +##rring +##tadt +##elin +##kova +willem +shrubs +napoleonic +realms +skater +volkswagen +##ł +tad +hara +archaeologist +awkwardly +eerie +##kind +wiley +##heimer +titus +organizers +cfl +crusaders +lama +vent +enraged +thankful +occupants +maximilian +##gaard +possessing +textbooks +##oran +collaborator +quaker +##ulo +avalanche +mono +silky +straits +isaiah +mustang +surged +resolutions +potomac +descend +kilograms +plato +strains +saturdays +##olin +bernstein +##ype +holstein +ponytail +belize +conversely +heroine +perpetual +##ylus +charcoal +piedmont +glee +negotiating +backdrop +prologue +##jah +pasadena +climbs +ramos +sunni +##holm +##tner +##tri +anand +deficiency +hertfordshire +stout +##avi +aperture +orioles +##irs +doncaster +intrigued +bombed +coating +otis +##mat +cocktail +##jit +##eto +amir +arousal +sar +##proof +dixie +pots +whereabouts +##fted +drains +bullying +cottages +scripture +coherent +fore +poe +appetite +##uration +sampled +##ators +derrick +rotor +jays +peacock +installment +##rro +advisors +##coming +rodeo +scotch +##mot +##fen +##vant +ensued +rodrigo +dictatorship +martyrs +twenties +towed +incidence +marta +rainforest +sai +scaled +##cles +oceanic +qualifiers +symphonic +mcbride +dislike +generalized +aubrey +colonization +##iation +##lion +##ssing +disliked +lublin +salesman +##ulates +spherical +whatsoever +sweating +avalon +contention +punt +severity +alderman +atari +##dina +##grant +##rop +scarf +seville +vertices +annexation +fairfield +fascination +inspiring +launches +palatinate +regretted +##rca +feral +##iom +elk +nap +olsen +reddy +yong +##leader +##iae +garment +transports +feng +gracie +outrage +viceroy +insides +##esis +breakup +grady +organizer +softer +grimaced +murals +galicia +arranging +vectors +##rsten +##sb +##cens +sloan +##eka +bitten +ara +fender +nausea +bumped +kris +banquet +comrades +detector +persisted +##llan +adjustment +endowed +cinemas +sellers +##uman +peek +epa +kindly +neglect +simpsons +talon +mausoleum +runaway +hangul +lookout +##cic +coughed +acquainted +chloride +quicker +accordion +neolithic +##qa +artemis +coefficient +lenny +pandora +tx +##xed +ecstasy +litter +segunda +chairperson +gemma +hiss +rumor +vow +nasal +antioch +compensate +patiently +transformers +##eded +judo +morrow +penis +posthumous +bandits +husbands +denote +flaming +##any +##phones +langley +yorker +1760 +walters +##kle +gubernatorial +fatty +leroy +outlaw +##nine +unpublished +poole +jakob +##ᵢ +##ₙ +crete +distorted +superiority +##dhi +intercept +crust +mig +claus +crashes +stallion +frontal +armistice +##estinal +elton +aj +encompassing +camel +commemorated +malaria +woodward +calf +cigar +penetrate +##oso +willard +##rno +##uche +illustrate +amusing +convergence +noteworthy +##lma +##rva +journeys +realise +manfred +##sable +##vocation +hearings +fiance +##posed +educators +provoked +adjusting +##cturing +modular +stockton +paterson +vlad +rejects +electors +selena +maureen +##tres +##rce +swirled +##num +proportions +nanny +pawn +naturalist +parma +apostles +awoke +ethel +wen +##bey +monsoon +overview +##inating +mccain +rendition +risky +adorned +##ih +equestrian +germain +nj +conspicuous +confirming +##yoshi +shivering +##imeter +milestone +rumours +flinched +bounds +smacked +token +##bei +lectured +automobiles +##shore +impacted +##iable +nouns +nero +##leaf +ismail +prostitute +trams +bridget +sud +stimulus +impressions +reins +revolves +##gned +giro +honeymoon +##swell +criterion +##sms +##uil +libyan +prefers +##osition +preview +sucks +accusation +bursts +metaphor +diffusion +tolerate +faye +betting +cinematographer +liturgical +specials +bitterly +humboldt +##ckle +flux +rattled +##itzer +archaeologists +odor +authorised +marshes +discretion +##ов +alarmed +archaic +inverse +##leton +explorers +##pine +drummond +tsunami +woodlands +##minate +##tland +booklet +insanity +owning +insert +crafted +calculus +receivers +stung +##eca +##nched +prevailing +travellers +eyeing +lila +graphs +##borne +julien +##won +morale +adaptive +therapist +erica +cw +libertarian +bowman +pitches +vita +##ional +crook +##entation +caledonia +mutiny +##sible +1840s +automation +flock +##pia +ironic +pathology +##imus +remarried +joker +withstand +energies +##att +shropshire +hostages +madeleine +tentatively +conflicting +mateo +recipes +euros +mercenaries +nico +##ndon +albuquerque +augmented +mythical +bel +freud +##child +cough +##lica +freddy +lillian +genetically +nuremberg +calder +bonn +outdoors +paste +suns +urgency +vin +restraint +tyson +##cera +##selle +barrage +bethlehem +kahn +##par +mounts +nippon +barony +happier +ryu +makeshift +sheldon +blushed +castillo +barking +listener +taped +bethel +fluent +headlines +pornography +rum +disclosure +sighing +mace +doubling +gunther +manly +##plex +interventions +physiological +forwards +emerges +##tooth +##gny +compliment +rib +recession +visibly +barge +faults +connector +exquisite +prefect +##rlin +patio +##cured +elevators +italics +pena +wasp +satin +botswana +graceful +respectable +##jima +##rter +##oic +franciscan +generates +##dl +alfredo +disgusting +##olate +##iously +sherwood +warns +cod +promo +cheryl +sino +##escu +twitch +##zhi +brownish +thom +ortiz +##dron +densely +##beat +carmel +reinforce +##bana +anastasia +downhill +vertex +contaminated +remembrance +harmonic +homework +fiancee +gears +olds +angelica +ramsay +quiz +colliery +sevens +##cape +autism +##hil +walkway +##boats +ruben +abnormal +ounce +khmer +##bbe +zachary +bedside +morphology +punching +##olar +sparrow +convinces +hewitt +queer +remastered +rods +mabel +solemn +notified +lyricist +symmetric +##xide +encore +passports +wildcats +##uni +baja +##pac +mildly +##ease +bleed +commodity +mounds +glossy +orchestras +##omo +damian +prelude +ambitions +##vet +awhile +remotely +##aud +asserts +imply +##iques +distinctly +modelling +remedy +##dded +windshield +dani +xiao +##endra +audible +powerplant +invalid +elemental +acquisitions +##hala +immaculate +libby +plata +smuggling +ventilation +denoted +minh +##morphism +differed +dion +kelley +lore +mocking +sabbath +spikes +hygiene +drown +runoff +stylized +tally +liberated +aux +interpreter +righteous +aba +siren +reaper +pearce +millie +##cier +##yra +gaius +##iso +captures +##ttering +dorm +claudio +##sic +benches +knighted +blackness +##ored +discount +fumble +oxidation +routed +novak +perpendicular +spoiled +fracture +splits +pads +topology +##cats +axes +fortunate +offenders +protestants +esteem +broadband +convened +frankly +hound +prototypes +isil +facilitated +keel +##sher +sahara +awaited +bubba +orb +prosecutors +hem +##xing +relaxing +remnant +romney +sorted +slalom +stefano +ulrich +##active +exemption +folder +pauses +foliage +hitchcock +epithet +criticisms +##aca +ballistic +brody +hinduism +chaotic +youths +equals +##pala +pts +thicker +analogous +capitalist +improvised +overseeing +sinatra +ascended +beverage +straightforward +##kon +curran +bois +induce +surveying +emperors +sax +unpopular +cartoonist +fused +##mble +unto +##yuki +localities +##cko +##ln +darlington +slain +academie +lobbying +sediment +puzzles +##grass +defiance +dickens +manifest +tongues +alumnus +arbor +coincide +appalachian +mustafa +examiner +cabaret +traumatic +yves +bracelet +draining +heroin +magnum +baths +odessa +consonants +mitsubishi +##gua +kellan +vaudeville +joked +straps +probation +##ław +ceded +interfaces +##pas +##zawa +blinding +viet +rothschild +museo +huddersfield +tactic +##storm +brackets +dazed +incorrectly +##vu +reg +glazed +fearful +manifold +benefited +irony +stumbling +##rte +willingness +balkans +mei +wraps +##aba +injected +##lea +gu +syed +harmless +##hammer +bray +takeoff +poppy +timor +cardboard +astronaut +purdue +weeping +southbound +cursing +stalls +diagonal +##neer +lamar +bryce +comte +weekdays +harrington +##uba +negatively +##see +lays +grouping +##cken +##henko +affirmed +halle +modernist +##lai +hodges +smelling +aristocratic +baptized +dismiss +justification +oilers +coupling +qin +snack +healer +##qing +gardener +layla +battled +formulated +stephenson +gravitational +##gill +1768 +granny +coordinating +suites +##ioned +monarchs +##cote +##hips +blended +barrister +deposition +fia +mina +policemen +paranoid +##pressed +churchyard +covert +crumpled +creep +abandoning +tr +transmit +conceal +barr +understands +readiness +spire +##cology +##enia +startling +unlock +vida +bowled +slots +##nat +##islav +spaced +trusting +admire +rig +slack +casualty +classmates +##odes +##rar +##rked +amherst +furnished +evolve +foundry +menace +mead +##lein +flu +wesleyan +##kled +monterey +webber +##vos +wil +##mith +##на +bartholomew +justices +restrained +##cke +amenities +mediated +sewage +trenches +mainz +##thus +1800s +##cula +##inski +caine +bonding +converts +spheres +superseded +marianne +crypt +sweaty +ensign +historia +##br +spruce +##ask +forks +thoughtfully +yukon +pamphlet +ames +##uter +karma +##yya +bryn +negotiation +sighs +incapable +##mbre +##ntial +actresses +taft +##mill +luce +prevailed +##amine +1773 +motionless +envoy +testify +investing +sculpted +instructors +provence +kali +cullen +horseback +##while +goodwin +##jos +gaa +norte +##ldon +modify +wavelength +abd +skinned +sprinter +forecast +scheduling +marries +squared +tentative +##chman +boer +##isch +bolts +swap +fisherman +assyrian +impatiently +guthrie +martins +murdoch +tanya +nicely +dolly +lacy +med +syn +decks +fashionable +millionaire +surfing +heaved +tammy +consulate +attendees +routinely +fuse +saxophonist +backseat +malaya +##lord +scowl +tau +##ishly +sighted +steaming +##rks +##holes +##hong +ching +##wife +bless +conserved +jurassic +stacey +zion +chunk +rigorous +blaine +peabody +slayer +dismay +brewers +nz +##jer +det +##glia +glover +postwar +penetration +sylvester +imitation +vertically +airlift +heiress +knoxville +viva +##uin +macon +##rim +##fighter +##gonal +janice +##orescence +##wari +marius +belongings +leicestershire +blanco +inverted +preseason +sanity +sobbing +##due +##elt +##dled +collingwood +regeneration +flickering +shortest +##mount +##osi +feminism +##lat +sherlock +cabinets +fumbled +northbound +precedent +snaps +##mme +researching +##akes +guillaume +insights +manipulated +vapor +neighbour +gangster +frey +stalking +scarcely +callie +barnett +tendencies +doomed +assessing +slung +panchayat +ambiguous +bartlett +##etto +distributing +violating +wolverhampton +##hetic +swami +histoire +##urus +liable +pounder +groin +hussain +larsen +popping +surprises +##atter +vie +curt +##station +mute +relocate +musicals +authorization +richter +##sef +immortality +tna +bombings +deteriorated +yiddish +##acious +robbed +colchester +ao +verified +balancing +apostle +swayed +recognizable +oxfordshire +retention +nottinghamshire +contender +judd +invitational +shrimp +uhf +##icient +cleaner +longitudinal +tanker +##mur +acronym +broker +koppen +sundance +suppliers +##gil +clipped +fuels +petite +##anne +landslide +helene +diversion +populous +landowners +auspices +melville +quantitative +##xes +ferries +nicky +##llus +doo +haunting +roche +carver +downed +unavailable +##pathy +approximation +hiroshima +##hue +garfield +valle +comparatively +keyboardist +traveler +##eit +congestion +calculating +subsidiaries +##bate +serb +modernization +fairies +deepened +ville +averages +##lore +inflammatory +tonga +##itch +co₂ +squads +##hea +gigantic +serum +enjoyment +retailer +verona +35th +cis +##phobic +magna +technicians +##vati +arithmetic +##sport +levin +##dation +amtrak +chow +sienna +##eyer +backstage +entrepreneurship +##otic +learnt +tao +##udy +worcestershire +formulation +baggage +hesitant +bali +sabotage +##kari +barren +enhancing +murmur +pl +freshly +putnam +syntax +aces +medicines +resentment +bandwidth +##sier +grins +chili +guido +##sei +framing +implying +gareth +lissa +genevieve +pertaining +admissions +geo +thorpe +proliferation +sato +bela +analyzing +parting +##gor +awakened +##isman +huddled +secrecy +##kling +hush +gentry +dungeons +##ego +coasts +##utz +sacrificed +##chule +landowner +mutually +prevalence +programmer +adolescent +disrupted +seaside +gee +trusts +vamp +georgie +##nesian +##iol +schedules +sindh +##market +etched +hm +sparse +bey +beaux +scratching +gliding +unidentified +collaborating +gems +jesuits +oro +accumulation +shaping +mbe +anal +##xin +enthusiasts +newscast +##egan +janata +dewey +parkinson +ankara +biennial +towering +inconsistent +##chet +thriving +terminate +cabins +furiously +eats +advocating +donkey +marley +muster +phyllis +leiden +##user +grassland +glittering +iucn +loneliness +memorandum +armenians +##ddle +popularized +rhodesia +60s +lame +##illon +sans +bikini +header +orbits +##finger +##ulator +sharif +spines +biotechnology +strolled +naughty +yates +##wire +fremantle +milo +##mour +abducted +removes +##atin +humming +##chrome +##ester +hume +pivotal +##rates +armand +grams +believers +elector +rte +apron +bis +scraped +##yria +endorsement +initials +##llation +dotted +hints +buzzing +emigration +nearer +indicators +##ulu +coarse +neutron +protectorate +##uze +directional +exploits +pains +loire +1830s +proponents +guggenheim +rabbits +ritchie +hectare +inputs +hutton +##raz +verify +##ako +boilers +longitude +##lev +skeletal +yer +emilia +citrus +compromised +##gau +prescription +paragraph +eduard +cadillac +attire +categorized +kenyan +weddings +charley +##bourg +entertain +monmouth +##lles +nutrients +davey +mesh +incentive +practised +ecosystems +kemp +subdued +overheard +##rya +bodily +maxim +##nius +apprenticeship +ursula +##fight +lodged +rug +silesian +unconstitutional +patel +inspected +coyote +unbeaten +##hak +34th +disruption +convict +parcel +##nham +collier +implicated +mallory +##iac +susannah +winkler +##rber +shia +phelps +sediments +graphical +robotic +##sner +adulthood +mart +smoked +##isto +kathryn +clarified +##aran +divides +convictions +oppression +pausing +burying +##mt +federico +mathias +eileen +##tana +kite +hunched +##acies +##atz +disadvantage +liza +kinetic +greedy +paradox +yokohama +dowager +trunks +ventured +##gement +gupta +vilnius +olaf +##thest +crimean +hopper +##ej +progressively +arturo +mouthed +arrondissement +##fusion +rubin +simulcast +oceania +##orum +##stra +##rred +busiest +intensely +navigator +cary +##vine +##hini +##bies +fife +rowe +rowland +posing +insurgents +shafts +lawsuits +activate +conor +inward +culturally +garlic +##eering +eclectic +##hui +##kee +##nl +furrowed +vargas +meteorological +rendezvous +##aus +culinary +commencement +##dition +quota +##notes +mommy +salaries +overlapping +mule +##iology +##mology +sums +wentworth +##isk +##zione +mainline +subgroup +##illy +hack +plaintiff +verdi +bulb +differentiation +engagements +multinational +supplemented +bertrand +caller +regis +##naire +##sler +##arts +##imated +blossom +propagation +kilometer +viaduct +vineyards +##uate +beckett +optimization +golfer +songwriters +seminal +semitic +thud +volatile +evolving +ridley +##wley +trivial +distributions +scandinavia +jiang +wrestled +insistence +emphasizes +napkin +##ods +adjunct +rhyme +##ricted +##eti +hopeless +surrounds +tremble +32nd +smoky +##ntly +oils +medicinal +padded +steer +wilkes +concessions +hue +uniquely +blinded +landon +##lane +hendrix +commemorating +dex +specify +chicks +##ggio +intercity +morley +##torm +highlighting +##oting +pang +oblique +stalled +##liner +flirting +newborn +1769 +bishopric +shaved +currie +dharma +spartan +##ooped +favorites +smug +novella +sirens +abusive +creations +espana +##lage +paradigm +semiconductor +sheen +##rdo +##yen +##zak +nrl +renew +##pose +##tur +adjutant +marches +norma +##enity +ineffective +weimar +grunt +##gat +lordship +plotting +expenditure +infringement +lbs +refrain +mimi +mistakenly +postmaster +1771 +##bara +ras +motorsports +tito +subjective +##zza +bully +stew +##kaya +prescott +##raphic +##zam +bids +styling +paranormal +reeve +sneaking +exploding +katz +akbar +migrant +syllables +indefinitely +##ogical +destroys +replaces +applause +##phine +pest +##fide +articulated +bertie +##cars +##ptic +courtroom +crowley +aesthetics +cummings +tehsil +hormones +titanic +dangerously +##ibe +stadion +jaenelle +auguste +ciudad +##chu +mysore +partisans +lucan +philipp +##aly +debating +henley +interiors +##rano +##tious +homecoming +beyonce +usher +henrietta +prepares +weeds +ely +plucked +##pire +##dable +luxurious +##aq +artifact +password +pasture +juno +maddy +minsk +##dder +##ologies +##rone +assessments +martian +royalist +1765 +examines +##mani +nino +parry +scooped +relativity +##eli +##uting +##cao +congregational +noisy +traverse +##agawa +strikeouts +nickelodeon +obituary +transylvania +binds +depictions +polk +trolley +##yed +##lard +breeders +##under +dryly +hokkaido +1762 +strengths +stacks +bonaparte +neared +prostitutes +stamped +anaheim +gutierrez +sinai +##zzling +bram +fresno +madhya +proton +##lena +##llum +##phon +reelected +wanda +##anus +##lb +ample +distinguishing +##yler +grasping +sermons +tomato +bland +stimulation +avenues +##eux +spreads +scarlett +fern +pentagon +assert +baird +chesapeake +calmed +distortion +fatalities +##olis +correctional +pricing +##astic +##gina +prom +dammit +ying +collaborate +##chia +welterweight +33rd +pointer +substitution +bonded +umpire +communicating +multitude +paddle +##obe +federally +intimacy +##insky +betray +ssr +##lett +##lves +##therapy +airbus +##tery +functioned +ud +bearer +biomedical +##hire +##nca +condom +brink +ik +##nical +macy +flap +gma +experimented +jelly +lavender +##icles +##ulia +munro +##mian +##tial +rye +##rle +60th +gigs +hottest +rotated +predictions +fuji +bu +##erence +##omi +barangay +##fulness +##sas +clocks +##rwood +##liness +cereal +roe +wight +decker +uttered +babu +onion +forcibly +##df +petra +sarcasm +hartley +peeled +storytelling +##xley +##ysis +##ffa +fibre +kiel +auditor +fig +harald +greenville +##berries +geographically +nell +quartz +##athic +cemeteries +crossings +nah +holloway +reptiles +chun +sichuan +snowy +corrections +##ivo +zheng +ambassadors +blacksmith +fielded +fluids +hardcover +turnover +medications +melvin +academies +##erton +roach +absorbing +spaniards +colton +##founded +outsider +espionage +kelsey +edible +##ulf +dora +establishes +##sham +##tries +contracting +##tania +cinematic +costello +nesting +##uron +connolly +duff +##nology +mma +##mata +fergus +sexes +optics +spectator +woodstock +banning +##hee +##fle +differentiate +outfielder +refinery +gerhard +horde +lair +drastically +##udi +landfall +##cheng +motorsport +odi +##achi +predominant +quay +skins +##ental +edna +harshly +complementary +murdering +##aves +wreckage +ono +outstretched +lennox +munitions +galen +reconcile +scalp +bicycles +gillespie +questionable +rosenberg +guillermo +jarvis +kabul +opium +yd +##twined +abuses +decca +outpost +##cino +sensible +neutrality +ponce +anchorage +atkins +turrets +inadvertently +disagree +libre +vodka +reassuring +weighs +##yal +glide +jumper +ceilings +repertory +outs +stain +##bial +envy +##ucible +smashing +heightened +policing +hyun +mixes +lai +prima +##ples +celeste +##bina +lucrative +intervened +kc +manually +##rned +stature +staffed +bun +bastards +nairobi +priced +##auer +thatcher +##kia +tripped +comune +##ogan +##pled +brasil +incentives +emanuel +hereford +musica +##kim +benedictine +biennale +##lani +eureka +gardiner +rb +knocks +sha +##ael +##elled +##onate +efficacy +ventura +masonic +sanford +maize +leverage +##feit +capacities +santana +##aur +novelty +vanilla +##cter +##tour +benin +##oir +neptune +drafting +tallinn +##cable +humiliation +##boarding +schleswig +fabian +bernardo +liturgy +spectacle +sweeney +pont +routledge +cosmos +ut +hilt +sleek +universally +##eville +##gawa +typed +##dry +favors +allegheny +glaciers +##rly +recalling +aziz +parasite +requiem +auf +##berto +##llin +illumination +##breaker +##issa +festivities +bows +govern +vibe +vp +sprawled +larson +pilgrim +bwf +leaping +##rts +##ssel +alexei +greyhound +hoarse +##dler +##oration +seneca +##cule +gaping +##ulously +##pura +cinnamon +##gens +##rricular +craven +fantasies +houghton +engined +reigned +dictator +supervising +##oris +bogota +commentaries +unnatural +fingernails +spirituality +tighten +canadiens +protesting +intentional +cheers +sparta +##ytic +##iere +##zine +widen +belgarath +controllers +dodd +iaaf +navarre +##ication +defect +squire +steiner +whisky +##mins +inevitably +tome +##gold +chew +##lid +elastic +##aby +streaked +alliances +jailed +regal +##ined +##phy +czechoslovak +narration +absently +##uld +bluegrass +guangdong +quran +criticizing +hose +hari +##liest +##owa +skier +streaks +deploy +##lom +raft +bose +dialed +huff +##eira +haifa +simplest +bursting +endings +sultanate +##titled +franks +whitman +ensures +sven +##ggs +collaborators +forster +organising +banished +napier +injustice +teller +layered +thump +##otti +roc +battleships +evidenced +fugitive +sadie +robotics +##roud +equatorial +geologist +##iza +yielding +##bron +##sr +internationale +mecca +##diment +skyline +toad +uploaded +reflective +undrafted +lal +leafs +bayern +##dai +lakshmi +shortlisted +##stick +##wicz +camouflage +donate +christi +lau +##acio +disclosed +nemesis +1761 +assemble +straining +northamptonshire +tal +##asi +bernardino +premature +heidi +42nd +coefficients +galactic +reproduce +buzzed +sensations +zionist +monsieur +myrtle +archery +strangled +musically +viewpoint +antiquities +bei +trailers +seahawks +cured +pee +preferring +tasmanian +lange +sul +##working +colder +overland +lucivar +massey +gatherings +haitian +##smith +disapproval +flaws +##cco +##enbach +1766 +npr +##icular +boroughs +creole +forums +techno +1755 +dent +abdominal +streetcar +##eson +##stream +procurement +gemini +predictable +##tya +acheron +christoph +feeder +fronts +vendor +bernhard +jammu +tumors +slang +##uber +goaltender +twists +curving +manson +vuelta +mer +peanut +confessions +pouch +unpredictable +allowance +theodor +vascular +##factory +bala +authenticity +metabolic +coughing +nanjing +##cea +pembroke +##bard +splendid +36th +hourly +##ahu +elmer +handel +##ivate +awarding +thrusting +experimentation +##hesion +caressed +entertained +steak +##rangle +biologist +orphans +baroness +oyster +stepfather +##dridge +mirage +reefs +speeding +barons +1764 +inhabit +preached +repealed +##tral +honoring +boogie +captives +administer +johanna +##imate +gel +suspiciously +1767 +sobs +##dington +backbone +hayward +garry +##folding +##nesia +maxi +##oof +##ppe +ellison +galileo +##stand +crimea +frenzy +amour +bumper +matrices +natalia +baking +garth +palestinians +##grove +smack +conveyed +ensembles +gardening +##manship +##rup +##stituting +1640 +harvesting +topography +shifters +dormitory +##carriage +##lston +ist +skulls +##stadt +dolores +jewellery +sarawak +##wai +##zier +fences +christy +confinement +tumbling +credibility +fir +stench +##bria +##plication +##nged +##sam +virtues +##belt +marjorie +pba +##eem +##made +celebrates +schooner +agitated +barley +fulfilling +anthropologist +restrict +novi +regulating +##nent +padres +##rani +##hesive +loyola +tabitha +milky +olson +proprietor +crambidae +guarantees +intercollegiate +ljubljana +hilda +##sko +ignorant +hooded +sardinia +##lidae +##vation +frontman +privileged +witchcraft +jammed +laude +poking +##than +bracket +amazement +yunnan +##erus +maharaja +linnaeus +commissioning +milano +peacefully +##logies +akira +rani +regulator +grasses +##rance +luzon +crows +compiler +gretchen +seaman +edouard +buccaneers +ellington +hamlets +whig +socialists +##anto +directorial +easton +mythological +##kr +##vary +rhineland +semantic +taut +dune +inventions +succeeds +##iter +replication +branched +##pired +prosecuted +kangaroo +penetrated +##avian +middlesbrough +doses +bleak +madam +predatory +relentless +##vili +reluctance +##vir +hailey +crore +silvery +1759 +monstrous +swimmers +transmissions +hawthorn +informing +##eral +toilets +caracas +crouch +##sett +cartel +hadley +##aling +alexia +yvonne +##biology +cinderella +eton +superb +blizzard +stabbing +industrialist +maximus +##orus +groves +maud +clade +oversized +comedic +##bella +rosen +nomadic +fulham +montane +beverages +galaxies +redundant +swarm +##rot +##folia +##llis +buckinghamshire +fen +bearings +bahadur +##rom +gilles +phased +dynamite +faber +benoit +##ount +fractured +tailored +anya +spices +westwood +cairns +auditions +inflammation +steamed +##rocity +##acion +##urne +skyla +thereof +watford +torment +archdeacon +transforms +demeanor +fucked +serge +##sor +mckenna +minas +entertainer +##icide +caress +originate +residue +##sty +1740 +##ilised +##org +beech +##wana +subsidies +##ghton +emptied +gladstone +firefighters +voodoo +het +nightingale +tamara +edmond +ingredient +weaknesses +silhouette +compatibility +withdrawing +hampson +##mona +anguish +giggling +bookstore +southernmost +tilting +##vance +bai +economical +briefcase +dreadful +hinted +projections +shattering +totaling +##rogate +analogue +indicted +periodical +fullback +##dman +haynes +##tenberg +##ffs +##ishment +1745 +thirst +stumble +penang +vigorous +##ddling +##kor +##lium +octave +##ove +##enstein +##inen +##ones +siberian +##uti +cbn +repeal +swaying +##vington +khalid +tanaka +unicorn +otago +plastered +lobe +riddle +##rella +perch +##ishing +croydon +filtered +graeme +tripoli +##ossa +crocodile +##chers +sufi +mined +##tung +inferno +lsu +##phi +swelled +utilizes +£2 +cale +periodicals +styx +hike +informally +coop +lund +##tidae +ala +hen +qui +transformations +disposed +sheath +chickens +##cade +fitzroy +silesia +unacceptable +odisha +1650 +sabrina +spokane +ratios +athena +massage +shen +dilemma +##drum +##riz +##hul +corona +doubtful +niall +##pha +##bino +fines +cite +acknowledging +bangor +ballard +bathurst +##resh +huron +mustered +alzheimer +garments +kinase +tyre +warship +flashback +pulmonary +braun +cheat +kamal +cyclists +constructions +grenades +ndp +traveller +excuses +stomped +signalling +trimmed +futsal +mosques +relevance +##wine +wta +##vah +hoc +##riding +optimistic +##´s +deco +interacting +rejecting +moniker +waterways +##ieri +##oku +mayors +gdansk +outnumbered +pearls +##ended +##hampton +fairs +totals +dominating +notions +stairway +compiling +pursed +commodities +grease +yeast +##jong +carthage +griffiths +residual +amc +contraction +laird +sapphire +##marine +##ivated +amalgamation +dissolve +inclination +lyle +packaged +altitudes +suez +canons +graded +lurched +narrowing +boasts +guise +enrico +##ovsky +rower +scarred +bree +cub +iberian +protagonists +bargaining +proposing +trainers +voyages +fishes +##aea +##ivist +##verance +encryption +artworks +kazan +sabre +cleopatra +hepburn +rotting +supremacy +mecklenburg +##brate +burrows +hazards +outgoing +flair +organizes +##ctions +scorpion +##usions +boo +chevalier +dunedin +slapping +ineligible +pensions +##omic +manufactures +emails +bismarck +weakening +blackish +ding +mcgee +quo +##rling +northernmost +manpower +greed +sampson +clicking +##ange +##horpe +##inations +##roving +torre +##eptive +##moral +symbolism +38th +asshole +meritorious +outfits +splashed +biographies +sprung +astros +##tale +filly +raoul +nw +tokugawa +linden +clubhouse +##apa +tracts +romano +##pio +putin +chained +dickson +gunshot +moe +gunn +rashid +##tails +zipper +##bas +##nea +contrasted +##ply +##udes +plum +pharaoh +##pile +aw +comedies +ingrid +sandwiches +subdivisions +mariana +kamen +hz +delaney +veto +herring +##words +possessive +outlines +##roup +siemens +stairwell +gallantry +messiah +palais +yells +zeppelin +bolivar +##cede +smackdown +mckinley +##mora +##yt +muted +geologic +finely +unitary +avatar +hamas +maynard +rees +bog +contrasting +##rut +liv +chico +disposition +##erate +becca +dmitry +yeshiva +narratives +##lva +##ulton +mercenary +sharpe +tempered +navigate +stealth +amassed +keynes +##lini +untouched +##rrie +havoc +lithium +##fighting +abyss +graf +southward +wolverine +balloons +implements +ngos +transitions +##icum +ambushed +concacaf +dormant +economists +##dim +costing +csi +rana +universite +boulders +verity +##llon +collin +mellon +misses +cypress +fluorescent +lifeless +spence +##ulla +crewe +shepard +pak +revelations +jolly +gibbons +paw +##dro +##quel +freeing +shack +fries +palatine +##hiko +accompaniment +cruising +recycled +##aver +erwin +sorting +synthesizers +dyke +realities +strides +enslaved +wetland +##ghan +competence +gunpowder +grassy +maroon +reactors +objection +##oms +carlson +gearbox +macintosh +radios +shelton +##sho +clergyman +prakash +mongols +trophies +oricon +stimuli +twenty20 +cantonese +cortes +mirrored +##saurus +bhp +cristina +melancholy +##lating +enjoyable +nuevo +##wny +downfall +schumacher +##ind +banging +lausanne +rumbled +paramilitary +reflex +ax +amplitude +migratory +##gall +##ups +midi +barnard +lastly +sherry +##nall +keystone +##kra +carleton +slippery +coloring +foe +socket +otter +##rgos +mats +##tose +consultants +bafta +bison +topping +primal +abandonment +transplant +atoll +hideous +mort +pained +reproduced +tae +howling +##turn +unlawful +billionaire +hotter +poised +lansing +##chang +dinamo +retro +messing +domesday +##mina +blitz +timed +##athing +##kley +ascending +gesturing +##izations +signaled +tis +chinatown +mermaid +savanna +jameson +##aint +catalina +##pet +##hers +cochrane +cy +chatting +##kus +alerted +computation +mused +noelle +majestic +mohawk +campo +octagonal +##sant +##hend +aspiring +##mart +comprehend +iona +paralyzed +shimmering +swindon +rhone +##eley +reputed +configurations +pitchfork +agitation +francais +gillian +lipstick +##ilo +outsiders +pontifical +resisting +bitterness +sewer +rockies +##edd +##ucher +misleading +1756 +exiting +galloway +##nging +risked +##heart +commemoration +schultz +##rka +integrating +##rsa +poses +shrieked +##weiler +guineas +gladys +jerking +owls +goldsmith +nightly +penetrating +##unced +lia +ignited +betsy +##aring +##thorpe +follower +vigorously +##rave +coded +kiran +knit +zoology +tbilisi +##bered +repository +govt +deciduous +dino +growling +##bba +enhancement +unleashed +chanting +pussy +biochemistry +##eric +kettle +repression +toxicity +nrhp +##arth +##kko +##bush +ernesto +commended +outspoken +mca +parchment +kristen +##aton +bisexual +raked +glamour +navajo +conditioned +showcased +##hma +spacious +youthful +##esa +usl +appliances +junta +brest +layne +conglomerate +enchanted +chao +loosened +picasso +circulating +inspect +montevideo +##centric +##kti +piazza +spurred +##aith +bari +freedoms +poultry +stamford +lieu +indigo +sarcastic +bahia +stump +attach +dvds +frankenstein +lille +approx +scriptures +pollen +##script +nmi +overseen +##ivism +tides +proponent +newmarket +inherit +milling +##erland +centralized +##rou +distributors +credentials +drawers +abbreviation +##lco +downing +uncomfortably +ripe +##oes +erase +franchises +populace +##bery +##khar +decomposition +pleas +##tet +daryl +sabah +##wide +fearless +genie +lesions +annette +##ogist +oboe +appendix +nair +dripped +petitioned +maclean +mosquito +parrot +hampered +1648 +operatic +reservoirs +##tham +irrelevant +jolt +summarized +##fp +medallion +##taff +clawed +harlow +narrower +goddard +marcia +bodied +fremont +suarez +altering +tempest +mussolini +porn +##isms +sweetly +oversees +walkers +solitude +grimly +shrines +ich +supervisors +hostess +dietrich +legitimacy +brushes +expressive +##yp +dissipated +##rse +localized +systemic +##nikov +gettysburg +##uaries +dialogues +muttering +housekeeper +sicilian +discouraged +##frey +beamed +kaladin +halftime +kidnap +##amo +##llet +1754 +synonymous +depleted +instituto +insulin +reprised +##opsis +clashed +##ctric +interrupting +radcliffe +insisting +medici +1715 +ejected +playfully +turbulent +starvation +##rini +shipment +rebellious +petersen +verification +merits +##rified +cakes +##charged +1757 +milford +shortages +spying +fidelity +##aker +emitted +storylines +harvested +seismic +##iform +cheung +kilda +theoretically +barbie +lynx +##rgy +##tius +goblin +mata +poisonous +##nburg +reactive +residues +obedience +##евич +conjecture +##rac +hating +sixties +kicker +moaning +motown +##bha +emancipation +neoclassical +##hering +consoles +ebert +professorship +##tures +sustaining +assaults +obeyed +affluent +incurred +tornadoes +##eber +##zow +emphasizing +highlanders +cheated +helmets +##ctus +internship +terence +bony +executions +legislators +berries +peninsular +tinged +##aco +1689 +amplifier +corvette +ribbons +lavish +pennant +##lander +worthless +##chfield +##forms +mariano +pyrenees +expenditures +##icides +chesterfield +mandir +tailor +39th +sergey +nestled +willed +aristocracy +devotees +goodnight +raaf +rumored +weaponry +remy +appropriations +harcourt +burr +riaa +##lence +limitation +unnoticed +guo +soaking +swamps +##tica +collapsing +tatiana +descriptive +brigham +psalm +##chment +maddox +##lization +patti +caliph +##aja +akron +injuring +serra +##ganj +basins +##sari +astonished +launcher +##church +hilary +wilkins +sewing +##sf +stinging +##fia +##ncia +underwood +startup +compilations +vibrations +embankment +jurist +bard +juventus +groundwater +kern +palaces +helium +boca +cramped +marissa +soto +##worm +jae +princely +##ggy +faso +bazaar +warmly +##voking +pairing +##lite +##grate +##nets +wien +freaked +ulysses +rebirth +##alia +mummy +guzman +jimenez +stilled +##nitz +trajectory +tha +woken +archival +professions +##pts +##pta +hilly +shadowy +shrink +##bolt +norwood +glued +migrate +stereotypes +devoid +##pheus +evacuate +horrors +infancy +gotham +knowles +optic +downloaded +sachs +kingsley +parramatta +darryl +mor +##onale +shady +commence +confesses +kan +##meter +##placed +marlborough +roundabout +regents +frigates +##imating +gothenburg +revoked +carvings +clockwise +convertible +intruder +##sche +banged +##ogo +vicky +bourgeois +##mony +dupont +footing +##gum +##real +buckle +yun +penthouse +sane +serviced +stakeholders +neumann +##eers +comb +##gam +catchment +pinning +rallies +typing +##elles +forefront +freiburg +sweetie +giacomo +widowed +goodwill +worshipped +aspirations +midday +##vat +fishery +##trick +bournemouth +turk +hearth +ethanol +guadalajara +murmurs +sl +##uge +afforded +scripted +##hta +wah +##jn +coroner +translucent +memorials +puck +progresses +clumsy +##race +candace +recounted +##slin +##uve +filtering +##mac +howl +strata +heron +leveled +##ays +dubious +##oja +##wheel +citations +exhibiting +##laya +##mics +turkic +##lberg +injunction +##ennial +antibodies +organise +##rigues +cardiovascular +cushion +inverness +##zquez +dia +cocoa +sibling +##tman +##roid +expanse +feasible +tunisian +algiers +##relli +rus +dso +westphalia +bro +tacoma +downloads +##ours +konrad +duran +##hdi +continuum +jett +compares +legislator +secession +##nable +##gues +##zuka +translating +reacher +##gley +##ła +aleppo +##agi +orchards +trapping +linguist +versatile +drumming +postage +calhoun +superiors +##mx +barefoot +leary +##cis +ignacio +alfa +kaplan +##rogen +bratislava +mori +##vot +disturb +haas +cartridges +gilmore +radiated +salford +tunic +hades +##ulsive +archeological +delilah +magistrates +auditioned +brewster +charters +empowerment +blogs +cappella +dynasties +iroquois +whipping +##krishna +raceway +truths +myra +weaken +judah +mcgregor +##horse +mic +refueling +37th +burnley +bosses +markus +premio +query +##gga +dunbar +##economic +darkest +lyndon +sealing +commendation +reappeared +##mun +addicted +ezio +slaughtered +satisfactory +shuffle +##eves +##thic +##uj +fortification +warrington +##otto +resurrected +fargo +mane +##utable +##lei +foreword +ox +##aris +##vern +abrams +hua +##mento +sakura +##alo +sentimental +##skaya +midfield +##eses +sturdy +scrolls +macleod +##kyu +entropy +##lance +mitochondrial +cicero +excelled +thinner +convoys +perceive +##oslav +##urable +systematically +grind +burkina +##tagram +ops +##aman +guantanamo +##cloth +##tite +forcefully +wavy +##jou +pointless +##linger +##tze +layton +portico +superficial +clerical +outlaws +##hism +burials +muir +##inn +creditors +hauling +rattle +##leg +calais +monde +archers +reclaimed +dwell +wexford +hellenic +falsely +remorse +##tek +dough +furnishings +##uttered +gabon +neurological +novice +##igraphy +contemplated +pulpit +nightstand +saratoga +##istan +documenting +pulsing +taluk +##firmed +busted +marital +##rien +disagreements +wasps +##yes +hodge +mcdonnell +mimic +fran +pendant +dhabi +musa +##nington +congratulations +argent +darrell +concussion +losers +regrets +thessaloniki +reversal +donaldson +hardwood +thence +achilles +ritter +##eran +demonic +jurgen +prophets +goethe +eki +classmate +##cking +yank +irrational +##inging +perished +seductive +qur +sourced +##crat +##typic +mustard +ravine +barre +horizontally +characterization +phylogenetic +boise +##dit +##runner +##tower +brutally +intercourse +seduce +##bbing +fay +ferris +ogden +amar +nik +unarmed +##inator +evaluating +kyrgyzstan +sweetness +##lford +##oki +mccormick +meiji +notoriety +stimulate +disrupt +figuring +instructional +mcgrath +##zoo +groundbreaking +##lto +flinch +khorasan +agrarian +bengals +mixer +radiating +##sov +ingram +pitchers +nad +tariff +##cript +tata +##codes +##emi +##ungen +appellate +lehigh +##bled +##giri +brawl +duct +texans +##ciation +##ropolis +skipper +speculative +vomit +doctrines +stresses +davy +graders +whitehead +jozef +timely +cumulative +haryana +paints +appropriately +boon +cactus +##ales +##pid +dow +legions +##pit +perceptions +1730 +picturesque +##yse +periphery +rune +wr +##aha +celtics +sentencing +whoa +##erin +confirms +variance +moines +mathews +spade +rave +fronted +blending +alleging +reared +##paper +grassroots +eroded +##physical +directs +ordeal +##sław +accelerate +hacker +rooftop +##inia +lev +buys +cebu +devote +##lce +specialising +##ulsion +choreographed +repetition +warehouses +##ryl +paisley +tuscany +analogy +sorcerer +hash +huts +shards +descends +exclude +nix +chaplin +ito +vane +##drich +causeway +misconduct +limo +orchestrated +glands +jana +##kot +u2 +##sons +branching +contrasts +scoop +longed +##virus +chattanooga +syrup +cornerstone +##tized +##mind +##iaceae +careless +precedence +frescoes +##uet +chilled +consult +modelled +snatch +peat +##thermal +caucasian +humane +relaxation +spins +temperance +##lbert +occupations +lambda +hybrids +moons +##oese +rolf +societal +yerevan +ness +##ssler +befriended +mechanized +nominate +trough +boasted +cues +seater +##hom +bends +##tangle +conductors +emptiness +eurasian +adriatic +tian +##cie +anxiously +lark +propellers +chichester +jock +##holding +credible +recounts +tori +loyalist +abduction +##hoot +##redo +nepali +##mite +ventral +tempting +##ango +##crats +steered +##wice +javelin +dipping +laborers +prentice +looming +titanium +badges +emir +tensor +##ntation +egyptians +rash +denies +hawthorne +lombard +showers +wehrmacht +dietary +trojan +##reus +welles +executing +horseshoe +lifeboat +##lak +elsa +infirmary +nearing +roberta +boyer +mutter +trillion +joanne +##fine +##oked +sinks +vortex +uruguayan +clasp +sirius +##block +accelerator +prohibit +sunken +byu +chronological +diplomats +ochreous +symmetrical +1644 +maia +##tology +salts +reigns +atrocities +##ия +hess +bared +issn +##vyn +cater +saturated +##cycle +##isse +sable +voyager +dyer +yusuf +##inge +fountains +wolff +##nni +engraving +rollins +atheist +ominous +##ault +herr +chariot +martina +strung +##fell +##farlane +horrific +sahib +gazes +saetan +erased +ptolemy +##olic +flushing +lauderdale +analytic +##ices +navarro +beak +gorilla +herrera +broom +guadalupe +raiding +sykes +bsc +deliveries +1720 +invasions +carmichael +tajikistan +thematic +ecumenical +sentiments +onstage +##rians +##brand +##sume +catastrophic +flanks +molten +##arns +waller +aimee +terminating +##icing +alternately +##oche +nehru +printers +outraged +##eving +empires +template +banners +repetitive +za +##oise +vegetarian +##tell +guiana +opt +cavendish +lucknow +synthesized +##hani +##mada +finalized +##ctable +fictitious +mayoral +unreliable +##enham +embracing +peppers +rbis +##chio +##neo +inhibition +slashed +togo +orderly +embroidered +salty +barron +benito +totaled +##dak +pubs +simulated +caden +devin +tolkien +momma +welding +sesame +##ept +gottingen +hardness +shaman +temeraire +adequately +pediatric +assertion +radicals +composure +cadence +seafood +beaufort +lazarus +mani +warily +cunning +kurdistan +cantata +##kir +ares +##clusive +nape +townland +geared +insulted +flutter +boating +violate +draper +dumping +malmo +##hh +##romatic +firearm +alta +bono +obscured +##clave +exceeds +panorama +unbelievable +##train +preschool +##essed +disconnected +installing +rescuing +secretaries +accessibility +##castle +##ifice +##film +bouts +slug +waterway +mindanao +##buro +##ratic +halves +calming +liter +maternity +adorable +bragg +electrification +mcc +##dote +roxy +schizophrenia +munoz +kaye +whaling +mil +tingling +tolerant +##ago +unconventional +volcanoes +##finder +deportivo +##llie +robson +kaufman +neuroscience +wai +deportation +masovian +scraping +converse +##bh +hacking +bulge +##oun +administratively +yao +mammoth +booster +claremont +hooper +nomenclature +pursuits +mclaughlin +melinda +##sul +catfish +barclay +substrates +taxa +zee +kimberly +packets +padma +##ality +borrowing +ostensibly +solvent +##bri +##genesis +##mist +lukas +shreveport +veracruz +##lou +##wives +cheney +anatolia +hobbs +##zyn +cyclic +radiant +alistair +greenish +siena +dat +independents +##bation +conform +pieter +hyper +applicant +bradshaw +spores +telangana +vinci +inexpensive +nuclei +jang +nme +spd +cradled +receptionist +pow +##rika +fascism +##ifer +experimenting +##ading +##iec +##region +jocelyn +maris +stair +nocturnal +toro +constabulary +elgin +##kker +msc +##giving +##schen +##rase +doherty +doping +sarcastically +batter +maneuvers +##cano +##apple +##gai +##git +intrinsic +##nst +##stor +1753 +showtime +cafes +gasps +lviv +ushered +##thed +fours +restart +astonishment +transmitting +flyer +shrugs +##sau +intriguing +cones +dictated +mushrooms +medial +##kovsky +##elman +escorting +gaped +godfather +##door +##sell +djs +recaptured +timetable +vila +1710 +aerodrome +mortals +scientology +##orne +angelina +mag +convection +unpaid +insertion +intermittent +lego +##nated +endeavor +kota +pereira +##lz +bwv +glamorgan +insults +agatha +fey +##cend +fleetwood +mahogany +protruding +steamship +zeta +##arty +mcguire +suspense +##sphere +advising +urges +##wala +hurriedly +meteor +gilded +inline +arroyo +stalker +##oge +excitedly +revered +##cure +earle +introductory +##break +##ilde +mutants +puff +pulses +reinforcement +##haling +curses +lizards +stalk +correlated +##fixed +fallout +macquarie +##unas +bearded +denton +heaving +##ocation +winery +assign +dortmund +##lkirk +everest +invariant +charismatic +susie +##elling +bled +lesley +telegram +sumner +bk +##ogen +wilcox +needy +colbert +duval +##iferous +##mbled +allotted +attends +imperative +##hita +replacements +hawker +##inda +insurgency +##zee +##eke +casts +##yla +ives +transitioned +##pack +##powering +authoritative +baylor +flex +cringed +plaintiffs +woodrow +##skie +drastic +ape +aroma +unfolded +commotion +preoccupied +theta +routines +lasers +privatization +wand +domino +ek +clenching +nsa +strategically +showered +bile +handkerchief +pere +storing +christophe +insulting +nakamura +romani +asiatic +magdalena +palma +cruises +stripping +konstantin +soaring +##berman +colloquially +forerunner +havilland +incarcerated +parasites +sincerity +##utus +disks +plank +saigon +##ining +corbin +homo +ornaments +powerhouse +##tlement +chong +fastened +feasibility +idf +morphological +usable +##nish +##zuki +aqueduct +jaguars +keepers +##flies +aleksandr +faust +assigns +ewing +bacterium +hurled +tricky +hungarians +integers +wallis +yamaha +##isha +hushed +oblivion +aviator +evangelist +friars +##eller +monograph +ode +##nary +airplanes +labourers +charms +##nee +1661 +hagen +tnt +rudder +fiesta +transcript +dorothea +ska +inhibitor +maccabi +retorted +raining +encompassed +clauses +menacing +1642 +lineman +##gist +vamps +##dick +gloom +##rera +dealings +easing +seekers +##nut +##pment +helens +unmanned +##anu +##isson +basics +##amy +##ckman +adjustments +1688 +brutality +horne +##zell +##mable +aggregator +##thal +rhino +##drick +##vira +counters +##rting +mn +montenegrin +packard +##unciation +##♭ +##kki +reclaim +scholastic +thugs +pulsed +##icia +syriac +quan +saddam +banda +kobe +blaming +buddies +dissent +##lusion +##usia +corbett +jaya +delle +erratic +lexie +##hesis +amiga +hermes +##pressing +##leen +chapels +gospels +jamal +##uating +compute +revolving +warp +##sso +##thes +armory +##eras +##gol +antrim +loki +##kow +##asian +##good +##zano +braid +handwriting +subdistrict +funky +pantheon +##iculate +concurrency +estimation +improper +juliana +##his +newcomers +johnstone +staten +communicated +##oco +##alle +sausage +stormy +##stered +##tters +superfamily +##grade +acidic +collateral +tabloid +##oped +##rza +bladder +austen +##ellant +mcgraw +##hay +hannibal +mein +aquino +lucifer +wo +badger +boar +cher +christensen +greenberg +interruption +##kken +jem +mocked +bottoms +cambridgeshire +##lide +sprawling +##bbly +eastwood +ghent +synth +##buck +advisers +##bah +nominally +hapoel +qu +daggers +estranged +fabricated +towels +vinnie +wcw +misunderstanding +anglia +nothin +unmistakable +##dust +##lova +chilly +marquette +truss +##edge +##erine +reece +##lty +##chemist +##connected +41st +bash +raion +waterfalls +##ump +##main +labyrinth +queue +theorist +##istle +bharatiya +flexed +soundtracks +rooney +leftist +patrolling +wharton +plainly +alleviate +eastman +schuster +topographic +engages +immensely +unbearable +fairchild +1620 +dona +lurking +parisian +oliveira +ia +indictment +hahn +bangladeshi +##aster +##uming +##ential +antonia +expects +indoors +kildare +harlan +##logue +##ogenic +##sities +forgiven +##wat +childish +tavi +##mide +##orra +plausible +grimm +successively +scooted +##bola +##rith +spartans +emery +flatly +epilogue +##wark +flourish +##iny +##tracted +##overs +##oshi +bestseller +distressed +receipt +spitting +hermit +topological +##cot +drilled +subunit +francs +##layer +eel +##fk +##itas +octopus +footprint +petitions +##say +##foil +interfering +leaking +palo +##metry +thistle +valiant +##pic +narayan +mcpherson +##fast +gonzales +##enne +dustin +novgorod +solos +##zman +doin +##patient +##meyer +soluble +ashland +cuffs +carole +pendleton +whistling +vassal +##river +deviation +revisited +constituents +rallied +rotate +loomed +##eil +##nting +amateurs +augsburg +auschwitz +crowns +skeletons +##cona +bonnet +dummy +globalization +simeon +sleeper +mandal +differentiated +##crow +##mare +milne +bundled +exasperated +talmud +owes +segregated +##feng +##uary +dentist +piracy +props +##rang +devlin +##torium +malicious +paws +##laid +dependency +##ergy +##fers +##enna +pistons +rourke +jed +grammatical +tres +maha +wig +ghostly +jayne +##achal +##creen +##ilis +##lins +designate +##with +arrogance +cambodian +clones +showdown +throttle +twain +##ception +lobes +metz +nagoya +braking +##furt +roaming +##minster +amin +crippled +##llary +indifferent +hoffmann +idols +intimidating +1751 +influenza +memo +onions +1748 +bandage +consciously +##landa +##rage +clandestine +observes +swiped +tangle +##ener +##jected +##trum +##bill +##lta +hugs +congresses +josiah +spirited +##dek +humanist +managerial +filmmaking +inmate +rhymes +debuting +grimsby +ur +##laze +duplicate +vigor +republished +bolshevik +refurbishment +antibiotics +martini +methane +newscasts +royale +horizons +levant +iain +visas +##ischen +paler +##around +manifestation +snuck +alf +chop +futile +pedestal +rehab +##kat +bmg +kerman +res +fairbanks +jarrett +abstraction +saharan +##zek +1746 +procedural +clearer +kincaid +sash +luciano +##ffey +crunch +helmut +##vara +revolutionaries +##tute +creamy +leach +##mmon +1747 +permitting +nes +plight +wendell +##lese +contra +clancy +ipa +mach +staples +autopsy +disturbances +nueva +karin +pontiac +##uding +proxy +venerable +haunt +leto +bergman +expands +##helm +wal +##pipe +canning +celine +cords +obesity +##enary +intrusion +planner +##phate +reasoned +sequencing +harrow +##chon +##dora +marred +mcintyre +repay +tarzan +darting +harrisburg +margarita +repulsed +##lding +belinda +hamburger +novo +compliant +runways +bingham +registrar +skyscraper +cuthbert +improvisation +livelihood +##corp +##elial +admiring +##dened +sporadic +believer +casablanca +popcorn +asha +shovel +##bek +##dice +coiled +tangible +##dez +casper +elsie +resin +tenderness +rectory +##ivision +avail +sonar +##mori +boutique +##dier +guerre +bathed +upbringing +vaulted +sandals +blessings +##naut +##utnant +1680 +foxes +pia +corrosion +hesitantly +confederates +crystalline +footprints +shapiro +tirana +valentin +drones +45th +microscope +shipments +texted +inquisition +wry +guernsey +unauthorized +resigning +ripple +schubert +stu +reassure +felony +##ardo +brittle +koreans +##havan +##ives +dun +implicit +tyres +##aldi +##lth +magnolia +##ehan +##puri +##poulos +aggressively +fei +gr +familiarity +##poo +indicative +##trust +fundamentally +jimmie +overrun +anchors +moans +##opus +britannia +armagh +purposely +seizing +##vao +bewildered +mundane +avoidance +cosmopolitan +geometridae +quartermaster +caf +chatter +engulfed +gleam +purge +##icate +juliette +jurisprudence +guerra +revisions +##bn +casimir +brew +##jm +1749 +clapton +cloudy +conde +hermitage +simulations +torches +vincenzo +matteo +##rill +hidalgo +booming +westbound +accomplishment +tentacles +unaffected +##sius +annabelle +flopped +sloping +##litz +dreamer +interceptor +vu +##loh +consecration +copying +messaging +breaker +climates +hospitalized +1752 +torino +afternoons +winfield +witnessing +##teacher +breakers +choirs +sawmill +coldly +##ege +sipping +haste +uninhabited +conical +bibliography +pamphlets +severn +edict +##oca +deux +illnesses +grips +rehearsals +sis +thinkers +tame +##keepers +1690 +acacia +reformer +##osed +##rys +shuffling +##iring +##shima +eastbound +ionic +rhea +flees +littered +##oum +rocker +vomiting +groaning +champ +overwhelmingly +civilizations +paces +sloop +adoptive +##tish +skaters +##vres +aiding +nikola +shriek +##ignon +pharmaceuticals +tuna +calvert +gustavo +stocked +yearbook +##urai +##mana +computed +subsp +riff +hanoi +kelvin +hamid +moors +pastures +summons +jihad +nectar +##ctors +bayou +untitled +pleasing +vastly +republics +intellect +##ulio +##tou +crumbling +stylistic +##ی +consolation +frequented +h₂o +walden +widows +##iens +##ignment +chunks +improves +grit +recited +##dev +snarl +sociological +##arte +##gul +inquired +##held +bruise +clube +consultancy +homogeneous +hornets +multiplication +pasta +prick +savior +##grin +##kou +##phile +yoon +##gara +grimes +vanishing +cheering +reacting +bn +distillery +##quisite +##vity +coe +dockyard +massif +##jord +escorts +voss +##valent +byte +chopped +hawke +illusions +workings +floats +##koto +##vac +kv +annapolis +madden +##onus +alvaro +noctuidae +##cum +##scopic +avenge +steamboat +forte +illustrates +erika +##trip +dew +nationalities +bran +manifested +thirsty +diversified +muscled +reborn +##standing +arson +##lessness +##dran +##logram +##boys +##kushima +##vious +willoughby +##phobia +alsace +dashboard +yuki +##chai +granville +myspace +publicized +tricked +##gang +adjective +##ater +relic +reorganisation +enthusiastically +indications +saxe +##lassified +consolidate +iec +padua +helplessly +ramps +renaming +regulars +pedestrians +accents +convicts +inaccurate +lowers +mana +##pati +barrie +bjp +outta +someplace +berwick +flanking +invoked +marrow +sparsely +excerpts +clothed +rei +##ginal +wept +##straße +##vish +##ptive +membranes +aquitaine +creeks +cutler +sheppard +implementations +##dur +fragrance +budge +concordia +magnesium +marcelo +##antes +gladly +vibrating +##rral +##ggles +montrose +##omba +lew +seamus +1630 +cocky +##ament +##uen +bjorn +##rrick +fielder +fluttering +##lase +methyl +kimberley +mcdowell +reductions +barbed +##jic +##tonic +aeronautical +condensed +distracting +##promising +huffed +##cala +##sle +claudius +invincible +missy +pious +balthazar +##lang +butte +combo +orson +##dication +myriad +1707 +silenced +##fed +##rh +netball +yourselves +##oza +clarify +heller +peg +durban +etudes +offender +roast +blackmail +curvature +##woods +vile +illicit +suriname +##linson +overture +1685 +bubbling +gymnast +tucking +##mming +##ouin +maldives +##bala +gurney +##dda +##eased +##oides +backside +pinto +jars +racehorse +tending +##rdial +baronetcy +wiener +duly +##rke +barbarian +cupping +flawed +##thesis +bertha +pleistocene +puddle +swearing +##nob +##tically +fleeting +prostate +amulet +educating +##mined +##tler +75th +jens +respondents +cavaliers +papacy +raju +##iente +##ulum +##tip +funnel +disneyland +##lley +sociologist +##iam +faulkner +louvre +menon +##dson +##ower +afterlife +mannheim +peptide +referees +comedians +meaningless +##anger +##laise +fabrics +hurley +renal +sleeps +##bour +##icle +breakout +kristin +roadside +animator +clover +disdain +unsafe +redesign +##urity +firth +barnsley +portage +reset +narrows +commandos +expansive +speechless +tubular +essendon +eyelashes +smashwords +##yad +##bang +##claim +craved +sprinted +chet +somme +astor +wrocław +orton +bane +##erving +##uing +mischief +##amps +##sund +scaling +terre +##xious +impairment +offenses +undermine +moi +soy +contiguous +arcadia +inuit +seam +##tops +macbeth +rebelled +##icative +##iot +elaborated +frs +uniformed +##dberg +powerless +priscilla +stimulated +qc +arboretum +frustrating +trieste +bullock +##nified +enriched +glistening +intern +##adia +locus +nouvelle +ollie +ike +lash +starboard +tapestry +headlined +hove +rigged +##vite +pollock +##yme +thrive +clustered +cas +roi +gleamed +olympiad +##lino +pressured +regimes +##hosis +##lick +ripley +##ophone +kickoff +gallon +rockwell +##arable +crusader +glue +revolutions +scrambling +1714 +grover +##jure +englishman +aztec +contemplating +coven +preach +triumphant +tufts +##esian +rotational +##phus +falkland +##brates +strewn +clarissa +rejoin +environmentally +glint +banded +drenched +moat +albanians +johor +rr +maestro +malley +nouveau +shaded +taxonomy +adhere +bunk +airfields +##ritan +1741 +encompass +remington +tran +##erative +amelie +mazda +friar +morals +passions +##zai +breadth +vis +##hae +argus +burnham +caressing +insider +rudd +##imov +##rso +italianate +murderous +textual +wainwright +armada +bam +weave +timer +##taken +##nh +fra +##crest +ardent +salazar +taps +tunis +##ntino +allegro +gland +philanthropic +##chester +implication +##optera +esq +judas +noticeably +wynn +##dara +inched +indexed +crises +villiers +bandit +royalties +patterned +cupboard +interspersed +accessory +isla +kendrick +entourage +stitches +##esthesia +headwaters +##ior +interlude +distraught +draught +1727 +##basket +biased +sy +transient +triad +subgenus +adapting +kidd +shortstop +##umatic +dimly +spiked +mcleod +reprint +nellie +pretoria +windmill +##cek +singled +##mps +reunite +##orous +bankers +outlying +##omp +##ports +##tream +apologies +cosmetics +patsy +##deh +##ocks +##yson +bender +nantes +serene +##nad +lucha +mmm +##cius +##gli +cmll +coinage +nestor +juarez +##rook +smeared +sprayed +twitching +sterile +irina +embodied +juveniles +enveloped +miscellaneous +cancers +dq +gulped +luisa +crested +swat +donegal +ref +##anov +##acker +hearst +mercantile +##lika +doorbell +vicki +##alla +##som +bilbao +psychologists +stryker +sw +horsemen +turkmenistan +wits +##national +anson +mathew +screenings +##umb +rihanna +##agne +##nessy +aisles +##iani +##osphere +hines +kenton +saskatoon +tasha +truncated +##champ +##itan +mildred +advises +fredrik +interpreting +inhibitors +##athi +spectroscopy +##hab +##kong +karim +panda +##oia +##nail +conqueror +kgb +leukemia +##dity +arrivals +cheered +pisa +phosphorus +shielded +##riated +mammal +unitarian +urgently +chopin +sanitary +##mission +spicy +drugged +hinges +##tort +tipping +trier +impoverished +westchester +##caster +epoch +nonstop +##gman +##khov +aromatic +centrally +cerro +##tively +##vio +billions +modulation +sedimentary +facilitating +outrageous +goldstein +##eak +##kt +ld +maitland +penultimate +pollard +##dance +fleets +spaceship +vertebrae +##nig +alcoholism +als +recital +##bham +##omics +##bm +trois +##tropical +commemorates +##meric +marge +##raction +1643 +cosmetic +ravaged +##ige +catastrophe +eng +##shida +albrecht +arterial +bellamy +decor +harmon +##rde +bulbs +synchronized +vito +easiest +shetland +shielding +wnba +##glers +##ssar +##riam +brianna +cumbria +##aceous +##rard +cores +thayer +##nsk +brood +hilltop +luminous +carts +keynote +larkin +logos +##cta +##mund +##quay +lilith +tinted +wrestle +mobilization +##uses +sequential +siam +bloomfield +takahashi +##ieving +presenters +ringo +blazed +witty +##oven +##ignant +devastation +haydn +harmed +newt +therese +##peed +gershwin +molina +rabbis +sudanese +innate +restarted +##sack +##fus +slices +wb +##shah +enroll +hypothetical +hysterical +1743 +fabio +indefinite +warped +exchanging +unsuitable +##sboro +gallo +1603 +bret +cobalt +homemade +##hunter +operatives +##dhar +terraces +durable +latch +pens +whorls +##ctuated +##eaux +billing +ligament +succumbed +##gly +regulators +spawn +##brick +##stead +filmfare +rochelle +##nzo +1725 +circumstance +saber +supplements +##nsky +##tson +crowe +wellesley +carrot +##9th +##movable +primate +drury +sincerely +topical +##mad +##rao +callahan +kyiv +smarter +tits +undo +##yeh +announcements +anthologies +barrio +nebula +##islaus +##shaft +##tyn +bodyguards +assassinate +barns +emmett +scully +##yd +##eland +##tino +##itarian +demoted +gorman +lashed +prized +adventist +writ +##gui +alla +invertebrates +##ausen +1641 +amman +1742 +align +healy +redistribution +##gf +##rize +insulation +##drop +adherents +hezbollah +vitro +ferns +yanking +registering +uppsala +cheerleading +confines +mischievous +tully +##ross +49th +docked +roam +stipulated +pumpkin +##bry +prompt +##ezer +blindly +shuddering +craftsmen +frail +scented +katharine +scramble +shaggy +sponge +helix +zaragoza +43rd +backlash +fontaine +seizures +posse +cowan +nonfiction +telenovela +wwii +hammered +undone +##gpur +encircled +irs +##ivation +artefacts +oneself +searing +smallpox +##belle +##osaurus +shandong +breached +upland +blushing +rankin +infinitely +psyche +tolerated +docking +evicted +##col +unmarked +##lving +gnome +lettering +litres +musique +##oint +benevolent +##jal +blackened +##anna +mccall +racers +tingle +##ocene +##orestation +introductions +radically +##hiff +##باد +1610 +1739 +munchen +plead +##nka +condo +scissors +##sight +##tens +apprehension +##cey +##yin +hallmark +watering +formulas +sequels +##llas +aggravated +bae +commencing +##building +enfield +prohibits +marne +vedic +civilized +euclidean +jagger +beforehand +blasts +dumont +##arney +##nem +conversions +hierarchical +rios +simulator +##dya +##lellan +hedges +oleg +thrusts +shadowed +darby +maximize +1744 +gregorian +##nded +##routed +sham +unspecified +##hog +emory +factual +##smo +fooled +##rger +ortega +wellness +marlon +##oton +##urance +casket +keating +ley +enclave +##ayan +char +influencing +jia +##chenko +ammonia +erebidae +incompatible +violins +cornered +##arat +grooves +astronauts +columbian +rampant +fabrication +kyushu +mahmud +vanish +##dern +mesopotamia +##lete +##rgen +caspian +kenji +pitted +##vered +grimace +roanoke +tchaikovsky +twinned +##analysis +##awan +xinjiang +arias +clemson +kazakh +sizable +1662 +##khand +##vard +plunge +tatum +vittorio +##nden +cholera +##dana +bracing +indifference +projectile +superliga +##chee +realises +upgrading +porte +retribution +##vies +nk +stil +##resses +ama +bureaucracy +blackberry +bosch +testosterone +collapses +greer +##pathic +ioc +fifties +malls +##erved +bao +baskets +adolescents +siegfried +##osity +##tosis +mantra +detecting +existent +fledgling +##cchi +dissatisfied +gan +telecommunication +mingled +sobbed +controversies +outdated +taxis +##raus +fright +slams +##lham +##fect +##tten +detectors +fetal +tanned +##uw +fray +goth +olympian +skipping +mandates +scratches +sheng +unspoken +hyundai +tracey +hotspur +restrictive +##buch +americana +mundo +##bari +burroughs +diva +vulcan +##6th +distinctions +thumping +##ngen +mikey +sheds +fide +rescues +springsteen +vested +valuation +##ece +##ely +pinnacle +rake +sylvie +##edo +almond +quivering +##irus +alteration +faltered +##wad +51st +hydra +ticked +##kato +recommends +##dicated +antigua +arjun +stagecoach +wilfred +trickle +pronouns +##pon +aryan +nighttime +##anian +gall +pea +stitch +##hei +leung +milos +##dini +eritrea +starved +snowfall +kant +parasitic +cot +discus +hana +strikers +appleton +kitchens +##erina +##partisan +##itha +##vius +disclose +metis +##channel +1701 +##vera +fitch +1735 +blooded +##tila +decimal +##tang +##bai +cyclones +eun +bottled +peas +pensacola +basha +bolivian +crabs +boil +lanterns +partridge +roofed +1645 +necks +##phila +opined +patting +##kla +##lland +chuckles +volta +whereupon +##nche +devout +euroleague +suicidal +##dee +inherently +involuntary +knitting +nasser +##hide +puppets +colourful +courageous +southend +stills +miraculous +hodgson +richer +rochdale +ethernet +greta +uniting +prism +umm +##haya +##itical +##utation +deterioration +pointe +prowess +##ropriation +lids +scranton +billings +subcontinent +##koff +##scope +brute +kellogg +psalms +degraded +##vez +stanisław +##ructured +ferreira +pun +astonishing +gunnar +##yat +arya +prc +gottfried +##tight +excursion +##ographer +dina +##quil +##nare +huffington +illustrious +wilbur +verandah +##zard +naacp +##odle +constructive +fjord +kade +##naud +generosity +thrilling +baseline +cayman +frankish +plastics +accommodations +zoological +##fting +cedric +qb +motorized +##dome +##otted +squealed +tackled +canucks +budgets +situ +asthma +dail +gabled +grasslands +whimpered +writhing +judgments +minnie +##carbon +bananas +grille +domes +monique +odin +maguire +markham +tierney +##estra +##chua +libel +poke +speedy +atrium +laval +notwithstanding +##edly +fai +kala +##sur +robb +##sma +listings +luz +supplementary +tianjin +##acing +enzo +jd +ric +scanner +croats +transcribed +arden +##hair +##raphy +##lver +seventies +staggering +alam +horticultural +hs +regression +timbers +blasting +##ounded +montagu +manipulating +##cit +catalytic +1550 +troopers +##meo +condemnation +fitzpatrick +##oire +##roved +inexperienced +1670 +castes +##lative +outing +dubois +flicking +quarrel +ste +learners +1625 +whistled +##class +classify +tariffs +temperament +folly +liszt +##yles +immersed +jordanian +ceasefire +apparel +extras +maru +fished +##bio +harta +stockport +assortment +craftsman +paralysis +transmitters +##cola +blindness +##wk +fatally +proficiency +solemnly +##orno +repairing +amore +groceries +ultraviolet +##chase +schoolhouse +##tua +resurgence +nailed +##otype +ruse +saliva +diagrams +##tructing +albans +rann +thirties +antennas +hilarious +cougars +paddington +stats +##eger +breakaway +reza +authorship +prohibiting +scoffed +##etz +##ttle +conscription +defected +trondheim +##fires +ivanov +keenan +##adan +##ciful +##fb +##slow +locating +##ials +##tford +cadiz +basalt +blankly +interned +rags +rattling +##tick +carpathian +reassured +bum +guildford +iss +staunch +##onga +astronomers +sera +sofie +emergencies +susquehanna +##heard +duc +mastery +vh1 +williamsburg +bayer +buckled +craving +##khan +##rdes +bloomington +##write +alton +barbecue +##bians +justine +##hri +##ndt +delightful +smartphone +newtown +photon +retrieval +peugeot +hissing +##monium +##orough +flavors +lighted +relaunched +tainted +##games +##lysis +anarchy +microscopic +hopping +adept +evade +evie +##beau +inhibit +sinn +adjustable +hurst +intuition +wilton +44th +lawful +lowlands +stockings +thierry +##dalen +##hila +##nai +fates +prank +maison +lobbied +provocative +1724 +utopia +##qual +carbonate +gujarati +purcell +##rford +curtiss +##mei +overgrown +arenas +mediation +swallows +##rnik +respectful +turnbull +##hedron +##hope +alyssa +ozone +##ʻi +ami +gestapo +johansson +snooker +canteen +cuff +declines +empathy +stigma +##ags +##raine +taxpayers +volga +##wright +##copic +lifespan +overcame +tattooed +enactment +giggles +##ador +##camp +barrington +bribe +obligatory +orbiting +peng +##enas +elusive +sucker +##vating +cong +hardship +empowered +anticipating +estrada +cryptic +greasy +detainees +planck +sudbury +plaid +dod +kayla +##ears +##vb +##zd +mortally +##hein +cognition +radha +liechtenstein +meade +richly +argyle +harpsichord +liberalism +trumpets +lauded +tyrant +salsa +tiled +lear +promoters +reused +slicing +trident +##chuk +##gami +##lka +cantor +checkpoint +##points +gaul +leger +mammalian +##tov +##aar +##schaft +doha +frenchman +nirvana +##vino +delgado +headlining +##eron +##iography +jug +tko +1649 +naga +intersections +benfica +nawab +##suka +ashford +gulp +##deck +##vill +##rug +brentford +frazier +pleasures +dunne +potsdam +shenzhen +dentistry +##tec +flanagan +##dorff +##hear +chorale +dinah +prem +quezon +##rogated +relinquished +sutra +terri +##pani +flaps +##rissa +poly +##rnet +homme +aback +##eki +linger +womb +##kson +##lewood +doorstep +orthodoxy +threaded +westfield +##rval +dioceses +fridays +subsided +##gata +loyalists +##biotic +##ettes +letterman +lunatic +prelate +tenderly +invariably +souza +thug +winslow +##otide +furlongs +gogh +jeopardy +##runa +pegasus +##umble +humiliated +standalone +tagged +##roller +freshmen +klan +##bright +attaining +initiating +transatlantic +logged +viz +##uance +1723 +combatants +intervening +stephane +chieftain +despised +grazed +cdc +galveston +godzilla +macro +simulate +##planes +parades +##esses +##ductive +##unes +equator +overdose +##cans +##hosh +##lifting +joshi +epstein +sonora +treacherous +aquatics +manchu +responsive +##sation +supervisory +##christ +##llins +##ibar +##balance +##uso +kimball +karlsruhe +mab +##emy +ignores +phonetic +spaghetti +almighty +danzig +rumbling +tombstone +designations +lured +outset +##felt +supermarkets +grupo +kei +kraft +susanna +##blood +comprehension +genealogy +##aghan +##verted +redding +##ythe +1722 +bowing +##pore +##roi +lest +sharpened +fulbright +valkyrie +sikhs +##unds +swans +bouquet +merritt +##tage +##venting +commuted +redhead +clerks +leasing +cesare +dea +hazy +##vances +fledged +greenfield +servicemen +##gical +armando +blackout +sagged +downloadable +intra +potion +pods +##4th +##mism +attendants +gambia +stale +##ntine +plump +asteroids +rediscovered +buds +flea +hive +##neas +1737 +classifications +debuts +##eles +olympus +scala +##eurs +##gno +##mute +hummed +sigismund +visuals +wiggled +await +pilasters +clench +sulfate +##ances +bellevue +enigma +trainee +snort +##sw +clouded +denim +##rank +churning +hartman +lodges +riches +sima +##missible +accountable +socrates +regulates +mueller +1702 +avoids +solids +himalayas +nutrient +pup +##jevic +squat +fades +nec +##lates +##pina +##rona +##ου +privateer +tequila +##gative +##mpton +hornet +immortals +##dou +asturias +cleansing +dario +##rries +##anta +etymology +servicing +zhejiang +##venor +##nx +horned +erasmus +rayon +relocating +£10 +##bags +escalated +promenade +stubble +2010s +artisans +axial +liquids +mora +sho +yoo +##tsky +bundles +oldies +##nally +notification +bastion +##ths +sparkle +##lved +1728 +leash +pathogen +highs +##hmi +immature +gonzaga +ignatius +mansions +monterrey +sweets +bryson +##loe +polled +regatta +brightest +pei +rosy +squid +hatfield +payroll +addict +meath +cornerback +heaviest +lodging +##mage +capcom +rippled +##sily +barnet +mayhem +ymca +snuggled +rousseau +##cute +blanchard +fragmented +leighton +chromosomes +risking +##strel +##utter +corinne +coyotes +cynical +hiroshi +yeomanry +##ractive +ebook +grading +mandela +plume +agustin +magdalene +##rkin +bea +femme +trafford +##coll +##lun +##tance +52nd +fourier +upton +##mental +camilla +gust +iihf +islamabad +longevity +##kala +feldman +netting +##rization +endeavour +foraging +mfa +orr +##open +greyish +contradiction +graz +##ruff +handicapped +marlene +tweed +oaxaca +spp +campos +miocene +pri +configured +cooks +pluto +cozy +pornographic +##entes +70th +fairness +glided +jonny +lynne +rounding +sired +##emon +##nist +remade +uncover +##mack +complied +lei +newsweek +##jured +##parts +##enting +##pg +finer +guerrillas +athenian +deng +disused +stepmother +accuse +gingerly +seduction +confronting +##going +gora +nostalgia +sabres +virginity +wrenched +##minated +syndication +wielding +eyre +##gnon +##igny +behaved +taxpayer +sweeps +##growth +childless +gallant +##ywood +amplified +geraldine +scrape +##ffi +babylonian +fresco +##rdan +##kney +##position +1718 +restricting +tack +fukuoka +osborn +selector +partnering +##dlow +kia +tak +whitley +gables +##mania +mri +softness +immersion +##bots +##evsky +1713 +chilling +insignificant +pcs +##uis +elites +lina +purported +supplemental +teaming +##americana +##dding +##inton +proficient +rouen +##nage +##rret +niccolo +selects +##bread +fluffy +1621 +gruff +knotted +mukherjee +polgara +thrash +nicholls +secluded +smoothing +thru +corsica +loaf +whitaker +inquiries +##rrier +##kam +indochina +marlins +myles +peking +##tea +extracts +pastry +superhuman +connacht +vogel +##ditional +##het +##udged +##lash +gloss +quarries +refit +teaser +##alic +##gaon +20s +materialized +sling +camped +pickering +tung +tracker +pursuant +##cide +cranes +##cini +##typical +##viere +anhalt +overboard +workout +chores +fares +orphaned +stains +##logie +fenton +surpassing +joyah +triggers +##itte +grandmaster +##lass +##lists +clapping +fraudulent +ledger +nagasaki +##cor +##nosis +##tsa +eucalyptus +tun +##icio +##rney +##tara +dax +heroism +ina +wrexham +onboard +unsigned +##dates +moshe +galley +winnie +droplets +exiles +praises +watered +noodles +##aia +fein +leland +multicultural +stink +bingo +comets +erskine +modernized +canned +constraint +domestically +chemotherapy +featherweight +stifled +##mum +darkly +irresistible +refreshing +hasty +isolate +##oys +kitchener +planners +##wehr +cages +yarn +implant +toulon +elects +childbirth +yue +##lind +rightful +sportsman +junctions +remodeled +specifies +##rgh +##oons +complimented +##urgent +lister +ot +##logic +bequeathed +cheekbones +fontana +gabby +##dial +amadeus +corrugated +maverick +resented +triangles +##hered +##usly +nazareth +tyrol +1675 +assent +poorer +sectional +aegean +##cous +nylon +ghanaian +##egorical +##weig +cushions +forbid +fusiliers +obstruction +somerville +##scia +dime +earrings +elliptical +leyte +oder +polymers +timmy +midtown +piloted +settles +continual +externally +mayfield +##uh +enrichment +henson +keane +persians +1733 +benji +braden +pep +##efe +contenders +pepsi +valet +##isches +##asse +##earing +goofy +stroll +##amen +authoritarian +occurrences +adversary +ahmedabad +tangent +toppled +dorchester +1672 +modernism +marxism +islamist +charlemagne +exponential +racks +brunette +pic +skirmish +##bund +##lad +##powered +##yst +hoisted +messina +shatter +##ctum +jedi +vantage +##music +##neil +clemens +mahmoud +corrupted +authentication +lowry +nils +##washed +omnibus +wounding +jillian +##itors +##opped +serialized +narcotics +handheld +##arm +##plicity +intersecting +stimulating +##onis +crate +fellowships +hemingway +casinos +climatic +fordham +copeland +drip +beatty +leaflets +robber +brothel +madeira +##hedral +sphinx +ultrasound +##vana +valor +forbade +leonid +villas +##aldo +duane +marquez +##cytes +disadvantaged +forearms +kawasaki +reacts +consular +lax +uncles +uphold +##hopper +concepcion +dorsey +lass +##izan +arching +passageway +1708 +researches +tia +internationals +##graphs +##opers +distinguishes +javanese +divert +##uven +plotted +##listic +##rwin +##erik +##tify +affirmative +signifies +validation +##bson +kari +felicity +georgina +zulu +##eros +##rained +##rath +overcoming +argyll +##rbin +1734 +chiba +ratification +windy +earls +parapet +##marks +hunan +pristine +astrid +punta +##gart +brodie +##kota +##oder +malaga +minerva +rouse +##phonic +bellowed +pagoda +portals +reclamation +##gur +##odies +##⁄₄ +parentheses +quoting +allergic +palette +showcases +benefactor +heartland +nonlinear +##tness +bladed +cheerfully +scans +##ety +1666 +girlfriends +pedersen +hiram +sous +##liche +##nator +1683 +##nery +##orio +##umen +bobo +primaries +smiley +##cb +unearthed +uniformly +fis +metadata +1635 +ind +##oted +recoil +##titles +##tura +##ια +hilbert +jamestown +mcmillan +tulane +seychelles +##frid +antics +coli +fated +stucco +##grants +1654 +bulky +accolades +arrays +caledonian +carnage +optimism +puebla +##tative +##cave +enforcing +rotherham +dunlop +aeronautics +chimed +incline +zoning +archduke +hellenistic +##oses +##sions +candi +thong +##ople +magnate +rustic +##rsk +projective +slant +##offs +danes +hollis +vocalists +##ammed +congenital +contend +gesellschaft +##ocating +##pressive +douglass +quieter +##kshi +howled +salim +spontaneously +townsville +buena +southport +##bold +kato +1638 +faerie +stiffly +##vus +##rled +flawless +realising +taboo +##7th +straightening +jena +##hid +cartwright +berber +bertram +soloists +noses +coping +fission +hardin +inca +##cen +1717 +mobilized +vhf +##raf +biscuits +curate +##anial +gaunt +neighbourhoods +1540 +##abas +blanca +bypassed +sockets +behold +coincidentally +##bane +nara +shave +splinter +terrific +##arion +##erian +commonplace +juris +redwood +waistband +boxed +caitlin +fingerprints +jennie +naturalized +##ired +balfour +craters +jody +bungalow +hugely +quilt +glitter +pigeons +undertaker +bulging +constrained +##sil +##akh +assimilation +reworked +##person +persuasion +##pants +felicia +##cliff +##ulent +1732 +explodes +##dun +##inium +##zic +lyman +vulture +hog +overlook +begs +northwards +ow +spoil +##urer +fatima +favorably +accumulate +sargent +sorority +corresponded +dispersal +kochi +toned +##imi +##lita +internacional +newfound +##agger +##lynn +##rigue +booths +peanuts +##eborg +medicare +muriel +nur +##uram +crates +millennia +pajamas +worsened +##breakers +jimi +vanuatu +yawned +##udeau +carousel +##hony +hurdle +##ccus +##mounted +##pod +rv +##eche +airship +ambiguity +compulsion +recapture +##claiming +arthritis +##osomal +1667 +asserting +ngc +sniffing +dade +discontent +glendale +ported +##amina +defamation +rammed +##scent +fling +livingstone +##fleet +875 +apocalyptic +comrade +##lowe +cessna +eine +persecuted +subsistence +demi +hoop +reliefs +coptic +progressing +stemmed +perpetrators +1665 +priestess +##nio +dobson +ebony +rooster +itf +tortricidae +##bbon +##jian +cleanup +##jean +##øy +1721 +eighties +taxonomic +holiness +##hearted +##spar +antilles +showcasing +stabilized +##nb +gia +mascara +michelangelo +dawned +##uria +##vinsky +extinguished +fitz +grotesque +£100 +##fera +##loid +##mous +barges +neue +throbbed +cipher +johnnie +##mpt +outburst +##swick +spearheaded +administrations +heartbreak +pixels +pleasantly +##enay +lombardy +plush +##nsed +bobbie +##hly +reapers +tremor +xiang +minogue +substantive +hitch +barak +##wyl +kwan +##encia +910 +obscene +elegance +indus +surfer +bribery +conserve +##hyllum +##masters +horatio +##fat +apes +rebound +psychotic +##pour +iteration +##mium +##vani +botanic +horribly +antiques +dispose +paxton +##hli +##wg +timeless +1704 +disregard +engraver +hounds +##bau +##version +looted +uno +facilitates +groans +masjid +rutland +antibody +disqualification +decatur +footballers +quake +slacks +48th +rein +scribe +stabilize +commits +exemplary +tho +##hort +##chison +pantry +traversed +##hiti +disrepair +identifiable +vibrated +baccalaureate +csa +interviewing +##iensis +##raße +greaves +wealthiest +classed +jogged +£5 +##atal +illuminating +knicks +respecting +##uno +scrubbed +##iji +##dles +kruger +moods +growls +raider +silvia +chefs +kam +cree +percival +##terol +gunter +counterattack +defiant +henan +ze +##rasia +##riety +equivalence +submissions +##fra +##thor +bautista +mechanically +##heater +cornice +herbal +templar +##mering +outputs +ruining +ligand +renumbered +extravagant +mika +blockbuster +eta +insurrection +##ilia +darkening +ferocious +pianos +strife +kinship +##aer +melee +##anor +##iste +##oue +decidedly +weep +##jad +##missive +##ppel +puget +unease +##gnant +1629 +hammering +kassel +wessex +##lga +bromwich +egan +paranoia +utilization +##atable +##idad +contradictory +provoke +##ols +##ouring +##tangled +knesset +##very +##lette +plumbing +##sden +greensboro +occult +sniff +zev +beaming +gamer +haggard +mahal +##olt +##pins +mendes +utmost +briefing +gunnery +##gut +##pher +##zh +##rok +1679 +khalifa +sonya +##boot +principals +urbana +wiring +##liffe +##minating +##rrado +dahl +nyu +skepticism +townspeople +ithaca +lobster +somethin +##fur +##arina +##−1 +freighter +zimmerman +biceps +contractual +##herton +amend +hurrying +subconscious +##anal +meng +clermont +spawning +##eia +##lub +dignitaries +impetus +snacks +spotting +twigs +##bilis +##cz +##ouk +libertadores +nic +skylar +##aina +gustave +asean +##anum +dieter +legislatures +flirt +bromley +trolls +umar +##bbies +##tyle +blah +parc +bridgeport +crank +negligence +##nction +46th +constantin +molded +bandages +seriousness +00pm +siegel +carpets +compartments +upbeat +statehood +##dner +##edging +marko +platt +##hane +paving +##iy +1738 +abbess +impatience +limousine +nbl +lucille +mojo +nightfall +robbers +##nais +karel +brisk +calves +replicate +ascribed +telescopes +##olf +intimidated +ballast +specialization +aerodynamic +caliphate +visionary +##arded +epsilon +##aday +##onte +aggregation +auditory +boosted +reunification +kathmandu +loco +robyn +acknowledges +appointing +humanoid +newell +redeveloped +restraints +##tained +barbarians +chopper +1609 +italiana +##lez +##lho +investigates +wrestlemania +##anies +##bib +##falls +creaked +dragoons +gravely +minions +stupidity +volley +##harat +##week +musik +##eries +##uously +fungal +massimo +semantics +malvern +##ahl +##pee +discourage +embryo +imperialism +1910s +profoundly +##ddled +jiangsu +sparkled +stat +##holz +sweatshirt +tobin +##iction +sneered +##cheon +##oit +brit +causal +smyth +##neuve +diffuse +perrin +silvio +##ipes +##recht +detonated +iqbal +selma +##nism +##zumi +roasted +##riders +tay +##ados +##mament +##mut +##rud +completes +nipples +flavour +hirsch +##laus +calderon +sneakers +moravian +##ksha +1622 +##imeters +bodo +##isance +##pre +##ronia +anatomical +excerpt +##lke +dh +kunst +##tablished +##scoe +biomass +panted +unharmed +gael +housemates +montpellier +coa +rodents +tonic +hickory +singleton +##taro +1719 +aldo +breaststroke +dempsey +och +rocco +##cuit +merton +dissemination +midsummer +serials +##idi +haji +polynomials +enoch +prematurely +shutter +taunton +£3 +##grating +##inates +archangel +harassed +##asco +archway +dazzling +##ecin +1736 +sumo +wat +##kovich +1086 +honneur +##ently +##nostic +##ttal +##idon +1605 +1716 +rents +##gnan +hires +##ikh +##dant +howie +##rons +handler +retracted +shocks +1632 +arun +duluth +kepler +trumpeter +##lary +peeking +seasoned +trooper +##mara +laszlo +##iciencies +##rti +heterosexual +##inatory +indira +jogging +##inga +##lism +beit +dissatisfaction +malice +##ately +nedra +peeling +##rgeon +47th +stadiums +vertigo +##ains +iced +restroom +##plify +##tub +illustrating +pear +##chner +##sibility +inorganic +rappers +receipts +watery +##kura +lucinda +##oulos +reintroduced +##8th +##tched +gracefully +saxons +nutritional +wastewater +rained +favourites +bedrock +fisted +hallways +likeness +upscale +##lateral +1580 +blinds +prequel +##pps +##tama +deter +humiliating +restraining +tn +vents +1659 +laundering +recess +rosary +tractors +coulter +federer +##ifiers +##plin +persistence +##quitable +geschichte +pendulum +quakers +##beam +bassett +pictorial +koln +##sitor +drills +reciprocal +shooters +##cton +##tees +converge +pip +dmitri +donnelly +yamamoto +aqua +azores +demographics +hypnotic +spitfire +suspend +wryly +roderick +##rran +sebastien +##asurable +mavericks +##fles +himalayan +prodigy +##iance +transvaal +demonstrators +handcuffs +dodged +mcnamara +sublime +1726 +crazed +##efined +##till +ivo +pondered +reconciled +shrill +sava +##duk +bal +heresy +jaipur +goran +##nished +lux +shelly +whitehall +##hre +israelis +peacekeeping +##wled +1703 +demetrius +ousted +##arians +##zos +beale +anwar +backstroke +raged +shrinking +cremated +##yck +benign +towing +wadi +darmstadt +landfill +parana +soothe +colleen +sidewalks +mayfair +tumble +hepatitis +ferrer +superstructure +##gingly +##urse +##wee +anthropological +translators +##mies +closeness +hooves +##pw +mondays +##roll +##vita +landscaping +##urized +purification +sock +thorns +thwarted +jalan +tiberius +##taka +saline +##rito +confidently +khyber +sculptors +##ij +brahms +hammersmith +inspectors +battista +fivb +fragmentation +hackney +##uls +arresting +exercising +antoinette +bedfordshire +##zily +dyed +##hema +1656 +racetrack +variability +##tique +1655 +austrians +deteriorating +madman +theorists +aix +lehman +weathered +1731 +decreed +eruptions +1729 +flaw +quinlan +sorbonne +flutes +nunez +1711 +adored +downwards +fable +rasped +1712 +moritz +mouthful +renegade +shivers +stunts +dysfunction +restrain +translit +pancakes +##avio +##cision +##tray +vial +##lden +bain +##maid +##oxide +chihuahua +malacca +vimes +##rba +##rnier +1664 +donnie +plaques +##ually +bangs +floppy +huntsville +loretta +nikolay +##otte +eater +handgun +ubiquitous +##hett +eras +zodiac +1634 +##omorphic +1820s +##zog +cochran +##bula +##lithic +warring +##rada +dalai +excused +blazers +mcconnell +reeling +este +##abi +geese +hoax +taxon +##bla +guitarists +condemning +hunts +inversion +moffat +taekwondo +##lvis +1624 +stammered +##rest +##rzy +sousa +fundraiser +marylebone +navigable +uptown +cabbage +daniela +salman +shitty +whimper +##kian +##utive +programmers +protections +##rmi +##rued +forceful +##enes +fuss +##tao +##wash +brat +oppressive +reykjavik +spartak +ticking +##inkles +##kiewicz +adolph +horst +maui +protege +straighten +cpc +landau +concourse +clements +resultant +##ando +imaginative +joo +reactivated +##rem +##ffled +##uising +consultative +##guide +flop +kaitlyn +mergers +parenting +somber +##vron +supervise +vidhan +##imum +courtship +exemplified +harmonies +medallist +refining +##rrow +##ка +amara +##hum +goalscorer +sited +overshadowed +rohan +displeasure +secretive +multiplied +osman +##orth +engravings +padre +##kali +##veda +miniatures +mis +##yala +clap +pali +rook +##cana +1692 +57th +antennae +astro +oskar +1628 +bulldog +crotch +hackett +yucatan +##sure +amplifiers +brno +ferrara +migrating +##gree +thanking +turing +##eza +mccann +ting +andersson +onslaught +gaines +ganga +incense +standardization +##mation +sentai +scuba +stuffing +turquoise +waivers +alloys +##vitt +regaining +vaults +##clops +##gizing +digger +furry +memorabilia +probing +##iad +payton +rec +deutschland +filippo +opaque +seamen +zenith +afrikaans +##filtration +disciplined +inspirational +##merie +banco +confuse +grafton +tod +##dgets +championed +simi +anomaly +biplane +##ceptive +electrode +##para +1697 +cleavage +crossbow +swirl +informant +##lars +##osta +afi +bonfire +spec +##oux +lakeside +slump +##culus +##lais +##qvist +##rrigan +1016 +facades +borg +inwardly +cervical +pointedly +stabilization +##odon +chests +1699 +hacked +ctv +orthogonal +suzy +##lastic +gaulle +jacobite +rearview +##erted +ashby +##drik +##igate +##mise +##zbek +affectionately +canine +disperse +latham +##istles +##ivar +spielberg +##orin +##idium +ezekiel +cid +##sg +durga +middletown +##cina +customized +frontiers +harden +##etano +##zzy +1604 +bolsheviks +coloration +yoko +##bedo +briefs +slabs +debra +liquidation +plumage +##oin +blossoms +dementia +subsidy +1611 +proctor +relational +jerseys +parochial +ter +##ici +esa +peshawar +cavalier +loren +idiots +shamrock +1646 +dutton +malabar +mustache +##endez +##ocytes +referencing +terminates +marche +yarmouth +##sop +acton +mated +seton +subtly +baptised +beige +extremes +jolted +kristina +telecast +##actic +safeguard +waldo +##baldi +##bular +endeavors +sloppy +subterranean +##ensburg +##itung +delicately +pigment +tq +##scu +1626 +collisions +coveted +herds +##personal +##meister +##nberger +chopra +##ricting +abnormalities +defective +galician +lucie +##dilly +alligator +likened +##genase +burundi +clears +complexion +derelict +deafening +diablo +fingered +champaign +dogg +enlist +isotope +labeling +mrna +##erre +brilliance +marvelous +##ayo +1652 +crawley +ether +footed +dwellers +deserts +hamish +rubs +warlock +skimmed +##lizer +buick +embark +heraldic +irregularities +##ajan +kiara +##kulam +##ieg +antigen +kowalski +##lge +oakley +visitation +##mbit +vt +##suit +1570 +murderers +##miento +##rites +chimneys +##sling +condemn +custer +exchequer +havre +##ghi +fluctuations +##rations +dfb +hendricks +vaccines +##tarian +nietzsche +biking +juicy +##duced +brooding +scrolling +selangor +##ragan +annum +boomed +seminole +sugarcane +##dna +departmental +dismissing +innsbruck +arteries +ashok +batavia +daze +kun +overtook +##rga +##tlan +beheaded +gaddafi +holm +electronically +faulty +galilee +fractures +kobayashi +##lized +gunmen +magma +aramaic +mala +eastenders +inference +messengers +bf +##qu +bathrooms +##vere +1658 +flashbacks +ideally +misunderstood +##jali +##weather +mendez +##grounds +uncanny +##iii +1709 +friendships +##nbc +sacrament +accommodated +reiterated +logistical +pebbles +thumped +##escence +administering +decrees +drafts +##flight +##cased +##tula +futuristic +picket +intimidation +winthrop +##fahan +interfered +afar +francoise +morally +uta +cochin +croft +dwarfs +##bruck +##dents +##nami +biker +##hner +##meral +##isen +##ometric +##pres +##ан +brightened +meek +parcels +securely +gunners +##jhl +##zko +agile +hysteria +##lten +##rcus +bukit +champs +chevy +cuckoo +leith +sadler +theologians +welded +##section +1663 +plurality +xander +##rooms +##formed +shredded +temps +intimately +pau +tormented +##lok +##stellar +1618 +charred +essen +##mmel +alarms +spraying +ascot +blooms +twinkle +##abia +##apes +internment +obsidian +##chaft +snoop +##dav +##ooping +malibu +##tension +quiver +##itia +hays +mcintosh +travers +walsall +##ffie +1623 +beverley +schwarz +plunging +structurally +rosenthal +vikram +##tsk +ghz +##onda +##tiv +chalmers +groningen +pew +reckon +unicef +##rvis +55th +##gni +1651 +sulawesi +avila +cai +metaphysical +screwing +turbulence +##mberg +augusto +samba +56th +baffled +momentary +toxin +##urian +##wani +aachen +condoms +dali +steppe +##oed +##year +adolescence +dauphin +electrically +inaccessible +microscopy +nikita +##ega +atv +##enter +##oles +##oteric +accountants +punishments +wrongly +bribes +adventurous +clinch +flinders +southland +##hem +##kata +gough +##ciency +lads +soared +##ה +undergoes +deformation +outlawed +rubbish +##arus +##mussen +##nidae +##rzburg +arcs +##ingdon +##tituted +1695 +wheelbase +wheeling +bombardier +campground +zebra +##lices +##oj +##bain +lullaby +##ecure +donetsk +wylie +grenada +##arding +##ης +squinting +eireann +opposes +##andra +maximal +runes +##broken +##cuting +##iface +##ror +##rosis +additive +britney +adultery +triggering +##drome +detrimental +aarhus +containment +jc +swapped +vichy +##ioms +madly +##oric +##rag +brant +##ckey +1560 +1612 +broughton +rustling +##stems +##uder +asbestos +mentoring +##nivorous +finley +leaps +##isan +apical +pry +slits +substitutes +##dict +intuitive +fantasia +insistent +unreasonable +##igen +##vna +domed +hannover +margot +ponder +##zziness +impromptu +jian +rampage +stemming +##eft +andrey +gerais +whichever +amnesia +appropriated +anzac +clicks +modifying +ultimatum +cambrian +maids +verve +yellowstone +##mbs +conservatoire +##scribe +adherence +dinners +spectra +imperfect +mysteriously +sidekick +tatar +tuba +##aks +##ifolia +distrust +##athan +##zle +ronin +zac +##pse +celaena +instrumentalist +scents +skopje +##mbling +comical +compensated +vidal +condor +intersect +jingle +wavelengths +##urrent +mcqueen +##izzly +carp +weasel +militias +postdoctoral +eugen +gunslinger +##ɛ +faux +hospice +##for +appalled +derivation +dwarves +##elis +dilapidated +##folk +astoria +philology +##lwyn +##otho +##saka +inducing +philanthropy +##bf +##itative +geek +markedly +##yce +bessie +indices +##flict +frowns +resolving +weightlifting +tugs +cleric +contentious +1653 +mania +rms +##miya +##reate +##ruck +##tucket +bien +eels +marek +##ayton +##cence +discreet +unofficially +##ife +leaks +##bber +1705 +dung +compressor +hillsborough +pandit +shillings +distal +##skin +##tat +nosed +##nir +mangrove +undeveloped +##idia +textures +##inho +##rise +irritating +nay +amazingly +bancroft +apologetic +compassionate +kata +symphonies +##lovic +airspace +##lch +gifford +precautions +fulfillment +sevilla +vulgar +martinique +##urities +looting +piccolo +tidy +##dermott +quadrant +armchair +incomes +mathematicians +stampede +nilsson +##inking +##scan +foo +quarterfinal +##ostal +shang +shouldered +squirrels +##owe +vinegar +##bner +##rchy +##systems +delaying +##trics +ars +dwyer +rhapsody +sponsoring +##gration +bipolar +cinder +starters +##olio +##urst +signage +##nty +aground +figurative +mons +acquaintances +duets +erroneously +soyuz +elliptic +recreated +##cultural +##quette +##ssed +##tma +##zcz +moderator +scares +##itaire +##stones +##udence +juniper +sighting +##just +##nsen +britten +calabria +ry +bop +cramer +forsyth +stillness +airmen +gathers +unfit +##umber +##upt +taunting +seeker +streamlined +##bution +holster +schumann +tread +vox +##gano +##onzo +strive +dil +reforming +covent +newbury +predicting +##orro +decorate +tre +##puted +andover +asahi +dept +dunkirk +gills +##tori +buren +huskies +##stis +##stov +abstracts +bets +loosen +##opa +1682 +yearning +##glio +##sir +berman +effortlessly +enamel +napoli +persist +##peration +##uez +attache +elisa +invitations +##kic +accelerating +reindeer +boardwalk +clutches +nelly +polka +##kei +adamant +huey +lough +unbroken +adventurer +embroidery +inspecting +stanza +##ducted +naia +taluka +##pone +##roids +chases +deprivation +florian +##ppet +earthly +##lib +##ssee +colossal +foreigner +vet +freaks +patrice +rosewood +triassic +upstate +##pkins +dominates +ata +chants +ks +vo +##bley +##raya +##rmed +agra +infiltrate +##ailing +##ilation +##tzer +##uppe +##werk +binoculars +enthusiast +fujian +squeak +##avs +abolitionist +almeida +boredom +hampstead +marsden +rations +##ands +inflated +bonuses +rosalie +patna +##rco +detachments +penitentiary +54th +flourishing +woolf +##dion +##etched +papyrus +##lster +##nsor +##toy +bobbed +dismounted +endelle +inhuman +motorola +wince +wreath +##ticus +hideout +inspections +sanjay +disgrace +infused +pudding +stalks +##urbed +arsenic +leases +##hyl +##rrard +collarbone +##waite +##wil +dowry +##bant +##edance +genealogical +nitrate +salamanca +scandals +thyroid +necessitated +##` +##¡ +##¢ +##¦ +##¨ +##ª +##¬ +##´ +##¶ +##¾ +##¿ +##ð +##þ +##ħ +##œ +##ƒ +##ɐ +##ɑ +##ɒ +##ɕ +##ɣ +##ɨ +##ɪ +##ɫ +##ɬ +##ɯ +##ɲ +##ɴ +##ɹ +##ɾ +##ʀ +##ʁ +##ʂ +##ʃ +##ʉ +##ʊ +##ʋ +##ʌ +##ʎ +##ʐ +##ʑ +##ʒ +##ʔ +##ʲ +##ʳ +##ʷ +##ʸ +##ʻ +##ʼ +##ʾ +##ʿ +##ˡ +##ˣ +##ˤ +##ζ +##ξ +##щ +##ъ +##э +##ю +##ђ +##є +##ј +##љ +##њ +##ћ +##ӏ +##ա +##բ +##գ +##դ +##ե +##թ +##ի +##լ +##կ +##հ +##մ +##յ +##ն +##ո +##պ +##ս +##վ +##տ +##ր +##ւ +##ք +##־ +##א +##ב +##ג +##ד +##ו +##ז +##ח +##ט +##י +##ך +##כ +##ל +##ם +##מ +##ן +##נ +##ס +##ע +##ף +##פ +##ץ +##צ +##ק +##ר +##ש +##ת +##، +##ء +##ث +##ج +##ح +##خ +##ذ +##ز +##ش +##ص +##ض +##ط +##ظ +##غ +##ـ +##ف +##ق +##ك +##ى +##ٹ +##پ +##چ +##ک +##گ +##ں +##ھ +##ہ +##ے +##अ +##आ +##उ +##ए +##क +##ख +##ग +##च +##ज +##ट +##ड +##ण +##त +##थ +##द +##ध +##न +##प +##ब +##भ +##म +##य +##र +##ल +##व +##श +##ष +##स +##ह +##ा +##ि +##ी +##ो +##। +##॥ +##ং +##অ +##আ +##ই +##উ +##এ +##ও +##ক +##খ +##গ +##চ +##ছ +##জ +##ট +##ড +##ণ +##ত +##থ +##দ +##ধ +##ন +##প +##ব +##ভ +##ম +##য +##র +##ল +##শ +##ষ +##স +##হ +##া +##ি +##ী +##ে +##க +##ச +##ட +##த +##ந +##ன +##ப +##ம +##ய +##ர +##ல +##ள +##வ +##ா +##ி +##ு +##ே +##ை +##ನ +##ರ +##ಾ +##ක +##ය +##ර +##ල +##ව +##ා +##ต +##ท +##พ +##ล +##ว +##ส +##། +##ག +##ང +##ད +##ན +##པ +##བ +##མ +##འ +##ར +##ལ +##ས +##မ +##ა +##ბ +##გ +##დ +##ე +##ვ +##თ +##ი +##კ +##ლ +##მ +##ნ +##ო +##რ +##ს +##ტ +##უ +##ᄊ +##ᴬ +##ᴮ +##ᴰ +##ᴵ +##ᴺ +##ᵀ +##ᵇ +##ᵈ +##ᵖ +##ᵗ +##ᵣ +##ᵤ +##ᵥ +##ᶜ +##ᶠ +##‐ +##‑ +##‒ +##– +##— +##― +##‘ +##’ +##‚ +##“ +##” +##‡ +##… +##⁰ +##⁴ +##⁵ +##⁶ +##⁷ +##⁸ +##⁹ +##⁻ +##₅ +##₆ +##₇ +##₈ +##₉ +##₊ +##₍ +##₎ +##ₐ +##ₑ +##ₒ +##ₓ +##ₕ +##ₖ +##ₗ +##ₘ +##ₚ +##ₛ +##ₜ +##₤ +##₩ +##₱ +##₹ +##ℓ +##ℝ +##⅓ +##⅔ +##↦ +##⇄ +##⇌ +##∂ +##∅ +##∆ +##∇ +##∈ +##∗ +##∘ +##∧ +##∨ +##∪ +##⊂ +##⊆ +##⊕ +##⊗ +##☉ +##♯ +##⟨ +##⟩ +##ⱼ +##⺩ +##⺼ +##⽥ +##亻 +##宀 +##彳 +##忄 +##扌 +##氵 +##疒 +##糹 +##訁 +##辶 +##阝 +##龸 +##fi +##fl diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab_org.txt b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab_org.txt new file mode 100644 index 0000000000000000000000000000000000000000..ca4f9781030019ab9b253c6dcb8c7878b6dc87a5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_clip/vocab_org.txt @@ -0,0 +1,21128 @@ +[PAD] +[unused1] +[unused2] +[unused3] +[unused4] +[unused5] +[unused6] +[unused7] +[unused8] +[unused9] +[unused10] +[unused11] +[unused12] +[unused13] +[unused14] +[unused15] +[unused16] +[unused17] +[unused18] +[unused19] +[unused20] +[unused21] +[unused22] +[unused23] +[unused24] +[unused25] +[unused26] +[unused27] +[unused28] +[unused29] +[unused30] +[unused31] +[unused32] +[unused33] +[unused34] +[unused35] +[unused36] +[unused37] +[unused38] +[unused39] +[unused40] +[unused41] +[unused42] +[unused43] +[unused44] +[unused45] +[unused46] +[unused47] +[unused48] +[unused49] +[unused50] +[unused51] +[unused52] +[unused53] +[unused54] +[unused55] +[unused56] +[unused57] +[unused58] +[unused59] +[unused60] +[unused61] +[unused62] +[unused63] +[unused64] +[unused65] +[unused66] +[unused67] +[unused68] +[unused69] +[unused70] +[unused71] +[unused72] +[unused73] +[unused74] +[unused75] +[unused76] +[unused77] +[unused78] +[unused79] +[unused80] +[unused81] +[unused82] +[unused83] +[unused84] +[unused85] +[unused86] +[unused87] +[unused88] +[unused89] +[unused90] +[unused91] +[unused92] +[unused93] +[unused94] +[unused95] +[unused96] +[unused97] +[unused98] +[unused99] +[UNK] +[CLS] +[SEP] +[MASK] + + +! +" +# +$ +% +& +' +( +) +* ++ +, +- +. +/ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +: +; +< += +> +? +@ +[ +\ +] +^ +_ +a +b +c +d +e +f +g +h +i +j +k +l +m +n +o +p +q +r +s +t +u +v +w +x +y +z +{ +| +} +~ +£ +¤ +¥ +§ +© +« +® +° +± +² +³ +µ +· +¹ +º +» +¼ +× +ß +æ +÷ +ø +đ +ŋ +ɔ +ə +ɡ +ʰ +ˇ +ˈ +ˊ +ˋ +ˍ +ː +˙ +˚ +ˢ +α +β +γ +δ +ε +η +θ +ι +κ +λ +μ +ν +ο +π +ρ +ς +σ +τ +υ +φ +χ +ψ +ω +а +б +в +г +д +е +ж +з +и +к +л +м +н +о +п +р +с +т +у +ф +х +ц +ч +ш +ы +ь +я +і +ا +ب +ة +ت +د +ر +س +ع +ل +م +ن +ه +و +ي +۩ +ก +ง +น +ม +ย +ร +อ +า +เ +๑ +་ +ღ +ᄀ +ᄁ +ᄂ +ᄃ +ᄅ +ᄆ +ᄇ +ᄈ +ᄉ +ᄋ +ᄌ +ᄎ +ᄏ +ᄐ +ᄑ +ᄒ +ᅡ +ᅢ +ᅣ +ᅥ +ᅦ +ᅧ +ᅨ +ᅩ +ᅪ +ᅬ +ᅭ +ᅮ +ᅯ +ᅲ +ᅳ +ᅴ +ᅵ +ᆨ +ᆫ +ᆯ +ᆷ +ᆸ +ᆺ +ᆻ +ᆼ +ᗜ +ᵃ +ᵉ +ᵍ +ᵏ +ᵐ +ᵒ +ᵘ +‖ +„ +† +• +‥ +‧ +
 +‰ +′ +″ +‹ +› +※ +‿ +⁄ +ⁱ +⁺ +ⁿ +₁ +₂ +₃ +₄ +€ +℃ +№ +™ +ⅰ +ⅱ +ⅲ +ⅳ +ⅴ +← +↑ +→ +↓ +↔ +↗ +↘ +⇒ +∀ +− +∕ +∙ +√ +∞ +∟ +∠ +∣ +∥ +∩ +∮ +∶ +∼ +∽ +≈ +≒ +≡ +≤ +≥ +≦ +≧ +≪ +≫ +⊙ +⋅ +⋈ +⋯ +⌒ +① +② +③ +④ +⑤ +⑥ +⑦ +⑧ +⑨ +⑩ +⑴ +⑵ +⑶ +⑷ +⑸ +⒈ +⒉ +⒊ +⒋ +ⓒ +ⓔ +ⓘ +─ +━ +│ +┃ +┅ +┆ +┊ +┌ +└ +├ +┣ +═ +║ +╚ +╞ +╠ +╭ +╮ +╯ +╰ +╱ +╳ +▂ +▃ +▅ +▇ +█ +▉ +▋ +▌ +▍ +▎ +■ +□ +▪ +▫ +▬ +▲ +△ +▶ +► +▼ +▽ +◆ +◇ +○ +◎ +● +◕ +◠ +◢ +◤ +☀ +★ +☆ +☕ +☞ +☺ +☼ +♀ +♂ +♠ +♡ +♣ +♥ +♦ +♪ +♫ +♬ +✈ +✔ +✕ +✖ +✦ +✨ +✪ +✰ +✿ +❀ +❤ +➜ +➤ +⦿ +、 +。 +〃 +々 +〇 +〈 +〉 +《 +》 +「 +」 +『 +』 +【 +】 +〓 +〔 +〕 +〖 +〗 +〜 +〝 +〞 +ぁ +あ +ぃ +い +う +ぇ +え +お +か +き +く +け +こ +さ +し +す +せ +そ +た +ち +っ +つ +て +と +な +に +ぬ +ね +の +は +ひ +ふ +へ +ほ +ま +み +む +め +も +ゃ +や +ゅ +ゆ +ょ +よ +ら +り +る +れ +ろ +わ +を +ん +゜ +ゝ +ァ +ア +ィ +イ +ゥ +ウ +ェ +エ +ォ +オ +カ +キ +ク +ケ +コ +サ +シ +ス +セ +ソ +タ +チ +ッ +ツ +テ +ト +ナ +ニ +ヌ +ネ +ノ +ハ +ヒ +フ +ヘ +ホ +マ +ミ +ム +メ +モ +ャ +ヤ +ュ +ユ +ョ +ヨ +ラ +リ +ル +レ +ロ +ワ +ヲ +ン +ヶ +・ +ー +ヽ +ㄅ +ㄆ +ㄇ +ㄉ +ㄋ +ㄌ +ㄍ +ㄎ +ㄏ +ㄒ +ㄚ +ㄛ +ㄞ +ㄟ +ㄢ +ㄤ +ㄥ +ㄧ +ㄨ +ㆍ +㈦ +㊣ +㎡ +㗎 +一 +丁 +七 +万 +丈 +三 +上 +下 +不 +与 +丐 +丑 +专 +且 +丕 +世 +丘 +丙 +业 +丛 +东 +丝 +丞 +丟 +両 +丢 +两 +严 +並 +丧 +丨 +个 +丫 +中 +丰 +串 +临 +丶 +丸 +丹 +为 +主 +丼 +丽 +举 +丿 +乂 +乃 +久 +么 +义 +之 +乌 +乍 +乎 +乏 +乐 +乒 +乓 +乔 +乖 +乗 +乘 +乙 +乜 +九 +乞 +也 +习 +乡 +书 +乩 +买 +乱 +乳 +乾 +亀 +亂 +了 +予 +争 +事 +二 +于 +亏 +云 +互 +五 +井 +亘 +亙 +亚 +些 +亜 +亞 +亟 +亡 +亢 +交 +亥 +亦 +产 +亨 +亩 +享 +京 +亭 +亮 +亲 +亳 +亵 +人 +亿 +什 +仁 +仃 +仄 +仅 +仆 +仇 +今 +介 +仍 +从 +仏 +仑 +仓 +仔 +仕 +他 +仗 +付 +仙 +仝 +仞 +仟 +代 +令 +以 +仨 +仪 +们 +仮 +仰 +仲 +件 +价 +任 +份 +仿 +企 +伉 +伊 +伍 +伎 +伏 +伐 +休 +伕 +众 +优 +伙 +会 +伝 +伞 +伟 +传 +伢 +伤 +伦 +伪 +伫 +伯 +估 +伴 +伶 +伸 +伺 +似 +伽 +佃 +但 +佇 +佈 +位 +低 +住 +佐 +佑 +体 +佔 +何 +佗 +佘 +余 +佚 +佛 +作 +佝 +佞 +佟 +你 +佢 +佣 +佤 +佥 +佩 +佬 +佯 +佰 +佳 +併 +佶 +佻 +佼 +使 +侃 +侄 +來 +侈 +例 +侍 +侏 +侑 +侖 +侗 +供 +依 +侠 +価 +侣 +侥 +侦 +侧 +侨 +侬 +侮 +侯 +侵 +侶 +侷 +便 +係 +促 +俄 +俊 +俎 +俏 +俐 +俑 +俗 +俘 +俚 +保 +俞 +俟 +俠 +信 +俨 +俩 +俪 +俬 +俭 +修 +俯 +俱 +俳 +俸 +俺 +俾 +倆 +倉 +個 +倌 +倍 +倏 +們 +倒 +倔 +倖 +倘 +候 +倚 +倜 +借 +倡 +値 +倦 +倩 +倪 +倫 +倬 +倭 +倶 +债 +值 +倾 +偃 +假 +偈 +偉 +偌 +偎 +偏 +偕 +做 +停 +健 +側 +偵 +偶 +偷 +偻 +偽 +偿 +傀 +傅 +傍 +傑 +傘 +備 +傚 +傢 +傣 +傥 +储 +傩 +催 +傭 +傲 +傳 +債 +傷 +傻 +傾 +僅 +働 +像 +僑 +僕 +僖 +僚 +僥 +僧 +僭 +僮 +僱 +僵 +價 +僻 +儀 +儂 +億 +儆 +儉 +儋 +儒 +儕 +儘 +償 +儡 +優 +儲 +儷 +儼 +儿 +兀 +允 +元 +兄 +充 +兆 +兇 +先 +光 +克 +兌 +免 +児 +兑 +兒 +兔 +兖 +党 +兜 +兢 +入 +內 +全 +兩 +八 +公 +六 +兮 +兰 +共 +兲 +关 +兴 +兵 +其 +具 +典 +兹 +养 +兼 +兽 +冀 +内 +円 +冇 +冈 +冉 +冊 +册 +再 +冏 +冒 +冕 +冗 +写 +军 +农 +冠 +冢 +冤 +冥 +冨 +冪 +冬 +冯 +冰 +冲 +决 +况 +冶 +冷 +冻 +冼 +冽 +冾 +净 +凄 +准 +凇 +凈 +凉 +凋 +凌 +凍 +减 +凑 +凛 +凜 +凝 +几 +凡 +凤 +処 +凪 +凭 +凯 +凰 +凱 +凳 +凶 +凸 +凹 +出 +击 +函 +凿 +刀 +刁 +刃 +分 +切 +刈 +刊 +刍 +刎 +刑 +划 +列 +刘 +则 +刚 +创 +初 +删 +判 +別 +刨 +利 +刪 +别 +刮 +到 +制 +刷 +券 +刹 +刺 +刻 +刽 +剁 +剂 +剃 +則 +剉 +削 +剋 +剌 +前 +剎 +剐 +剑 +剔 +剖 +剛 +剜 +剝 +剣 +剤 +剥 +剧 +剩 +剪 +副 +割 +創 +剷 +剽 +剿 +劃 +劇 +劈 +劉 +劊 +劍 +劏 +劑 +力 +劝 +办 +功 +加 +务 +劣 +动 +助 +努 +劫 +劭 +励 +劲 +劳 +労 +劵 +効 +劾 +势 +勁 +勃 +勇 +勉 +勋 +勐 +勒 +動 +勖 +勘 +務 +勛 +勝 +勞 +募 +勢 +勤 +勧 +勳 +勵 +勸 +勺 +勻 +勾 +勿 +匀 +包 +匆 +匈 +匍 +匐 +匕 +化 +北 +匙 +匝 +匠 +匡 +匣 +匪 +匮 +匯 +匱 +匹 +区 +医 +匾 +匿 +區 +十 +千 +卅 +升 +午 +卉 +半 +卍 +华 +协 +卑 +卒 +卓 +協 +单 +卖 +南 +単 +博 +卜 +卞 +卟 +占 +卡 +卢 +卤 +卦 +卧 +卫 +卮 +卯 +印 +危 +即 +却 +卵 +卷 +卸 +卻 +卿 +厂 +厄 +厅 +历 +厉 +压 +厌 +厕 +厘 +厚 +厝 +原 +厢 +厥 +厦 +厨 +厩 +厭 +厮 +厲 +厳 +去 +县 +叁 +参 +參 +又 +叉 +及 +友 +双 +反 +収 +发 +叔 +取 +受 +变 +叙 +叛 +叟 +叠 +叡 +叢 +口 +古 +句 +另 +叨 +叩 +只 +叫 +召 +叭 +叮 +可 +台 +叱 +史 +右 +叵 +叶 +号 +司 +叹 +叻 +叼 +叽 +吁 +吃 +各 +吆 +合 +吉 +吊 +吋 +同 +名 +后 +吏 +吐 +向 +吒 +吓 +吕 +吖 +吗 +君 +吝 +吞 +吟 +吠 +吡 +否 +吧 +吨 +吩 +含 +听 +吭 +吮 +启 +吱 +吳 +吴 +吵 +吶 +吸 +吹 +吻 +吼 +吽 +吾 +呀 +呂 +呃 +呆 +呈 +告 +呋 +呎 +呐 +呓 +呕 +呗 +员 +呛 +呜 +呢 +呤 +呦 +周 +呱 +呲 +味 +呵 +呷 +呸 +呻 +呼 +命 +咀 +咁 +咂 +咄 +咆 +咋 +和 +咎 +咏 +咐 +咒 +咔 +咕 +咖 +咗 +咘 +咙 +咚 +咛 +咣 +咤 +咦 +咧 +咨 +咩 +咪 +咫 +咬 +咭 +咯 +咱 +咲 +咳 +咸 +咻 +咽 +咿 +哀 +品 +哂 +哄 +哆 +哇 +哈 +哉 +哋 +哌 +响 +哎 +哏 +哐 +哑 +哒 +哔 +哗 +哟 +員 +哥 +哦 +哧 +哨 +哩 +哪 +哭 +哮 +哲 +哺 +哼 +哽 +唁 +唄 +唆 +唇 +唉 +唏 +唐 +唑 +唔 +唠 +唤 +唧 +唬 +售 +唯 +唰 +唱 +唳 +唷 +唸 +唾 +啃 +啄 +商 +啉 +啊 +問 +啓 +啕 +啖 +啜 +啞 +啟 +啡 +啤 +啥 +啦 +啧 +啪 +啫 +啬 +啮 +啰 +啱 +啲 +啵 +啶 +啷 +啸 +啻 +啼 +啾 +喀 +喂 +喃 +善 +喆 +喇 +喉 +喊 +喋 +喎 +喏 +喔 +喘 +喙 +喚 +喜 +喝 +喟 +喧 +喪 +喫 +喬 +單 +喰 +喱 +喲 +喳 +喵 +営 +喷 +喹 +喺 +喻 +喽 +嗅 +嗆 +嗇 +嗎 +嗑 +嗒 +嗓 +嗔 +嗖 +嗚 +嗜 +嗝 +嗟 +嗡 +嗣 +嗤 +嗦 +嗨 +嗪 +嗬 +嗯 +嗰 +嗲 +嗳 +嗶 +嗷 +嗽 +嘀 +嘅 +嘆 +嘈 +嘉 +嘌 +嘍 +嘎 +嘔 +嘖 +嘗 +嘘 +嘚 +嘛 +嘜 +嘞 +嘟 +嘢 +嘣 +嘤 +嘧 +嘩 +嘭 +嘮 +嘯 +嘰 +嘱 +嘲 +嘴 +嘶 +嘸 +嘹 +嘻 +嘿 +噁 +噌 +噎 +噓 +噔 +噗 +噙 +噜 +噠 +噢 +噤 +器 +噩 +噪 +噬 +噱 +噴 +噶 +噸 +噹 +噻 +噼 +嚀 +嚇 +嚎 +嚏 +嚐 +嚓 +嚕 +嚟 +嚣 +嚥 +嚨 +嚮 +嚴 +嚷 +嚼 +囂 +囉 +囊 +囍 +囑 +囔 +囗 +囚 +四 +囝 +回 +囟 +因 +囡 +团 +団 +囤 +囧 +囪 +囫 +园 +困 +囱 +囲 +図 +围 +囹 +固 +国 +图 +囿 +圃 +圄 +圆 +圈 +國 +圍 +圏 +園 +圓 +圖 +團 +圜 +土 +圣 +圧 +在 +圩 +圭 +地 +圳 +场 +圻 +圾 +址 +坂 +均 +坊 +坍 +坎 +坏 +坐 +坑 +块 +坚 +坛 +坝 +坞 +坟 +坠 +坡 +坤 +坦 +坨 +坪 +坯 +坳 +坵 +坷 +垂 +垃 +垄 +型 +垒 +垚 +垛 +垠 +垢 +垣 +垦 +垩 +垫 +垭 +垮 +垵 +埂 +埃 +埋 +城 +埔 +埕 +埗 +域 +埠 +埤 +埵 +執 +埸 +培 +基 +埼 +堀 +堂 +堃 +堅 +堆 +堇 +堑 +堕 +堙 +堡 +堤 +堪 +堯 +堰 +報 +場 +堵 +堺 +堿 +塊 +塌 +塑 +塔 +塗 +塘 +塚 +塞 +塢 +塩 +填 +塬 +塭 +塵 +塾 +墀 +境 +墅 +墉 +墊 +墒 +墓 +増 +墘 +墙 +墜 +增 +墟 +墨 +墩 +墮 +墳 +墻 +墾 +壁 +壅 +壆 +壇 +壊 +壑 +壓 +壕 +壘 +壞 +壟 +壢 +壤 +壩 +士 +壬 +壮 +壯 +声 +売 +壳 +壶 +壹 +壺 +壽 +处 +备 +変 +复 +夏 +夔 +夕 +外 +夙 +多 +夜 +够 +夠 +夢 +夥 +大 +天 +太 +夫 +夭 +央 +夯 +失 +头 +夷 +夸 +夹 +夺 +夾 +奂 +奄 +奇 +奈 +奉 +奋 +奎 +奏 +奐 +契 +奔 +奕 +奖 +套 +奘 +奚 +奠 +奢 +奥 +奧 +奪 +奬 +奮 +女 +奴 +奶 +奸 +她 +好 +如 +妃 +妄 +妆 +妇 +妈 +妊 +妍 +妒 +妓 +妖 +妘 +妙 +妝 +妞 +妣 +妤 +妥 +妨 +妩 +妪 +妮 +妲 +妳 +妹 +妻 +妾 +姆 +姉 +姊 +始 +姍 +姐 +姑 +姒 +姓 +委 +姗 +姚 +姜 +姝 +姣 +姥 +姦 +姨 +姪 +姫 +姬 +姹 +姻 +姿 +威 +娃 +娄 +娅 +娆 +娇 +娉 +娑 +娓 +娘 +娛 +娜 +娟 +娠 +娣 +娥 +娩 +娱 +娲 +娴 +娶 +娼 +婀 +婁 +婆 +婉 +婊 +婕 +婚 +婢 +婦 +婧 +婪 +婭 +婴 +婵 +婶 +婷 +婺 +婿 +媒 +媚 +媛 +媞 +媧 +媲 +媳 +媽 +媾 +嫁 +嫂 +嫉 +嫌 +嫑 +嫔 +嫖 +嫘 +嫚 +嫡 +嫣 +嫦 +嫩 +嫲 +嫵 +嫻 +嬅 +嬉 +嬌 +嬗 +嬛 +嬢 +嬤 +嬪 +嬰 +嬴 +嬷 +嬸 +嬿 +孀 +孃 +子 +孑 +孔 +孕 +孖 +字 +存 +孙 +孚 +孛 +孜 +孝 +孟 +孢 +季 +孤 +学 +孩 +孪 +孫 +孬 +孰 +孱 +孳 +孵 +學 +孺 +孽 +孿 +宁 +它 +宅 +宇 +守 +安 +宋 +完 +宏 +宓 +宕 +宗 +官 +宙 +定 +宛 +宜 +宝 +实 +実 +宠 +审 +客 +宣 +室 +宥 +宦 +宪 +宫 +宮 +宰 +害 +宴 +宵 +家 +宸 +容 +宽 +宾 +宿 +寂 +寄 +寅 +密 +寇 +富 +寐 +寒 +寓 +寛 +寝 +寞 +察 +寡 +寢 +寥 +實 +寧 +寨 +審 +寫 +寬 +寮 +寰 +寵 +寶 +寸 +对 +寺 +寻 +导 +対 +寿 +封 +専 +射 +将 +將 +專 +尉 +尊 +尋 +對 +導 +小 +少 +尔 +尕 +尖 +尘 +尚 +尝 +尤 +尧 +尬 +就 +尴 +尷 +尸 +尹 +尺 +尻 +尼 +尽 +尾 +尿 +局 +屁 +层 +屄 +居 +屆 +屈 +屉 +届 +屋 +屌 +屍 +屎 +屏 +屐 +屑 +展 +屜 +属 +屠 +屡 +屢 +層 +履 +屬 +屯 +山 +屹 +屿 +岀 +岁 +岂 +岌 +岐 +岑 +岔 +岖 +岗 +岘 +岙 +岚 +岛 +岡 +岩 +岫 +岬 +岭 +岱 +岳 +岷 +岸 +峇 +峋 +峒 +峙 +峡 +峤 +峥 +峦 +峨 +峪 +峭 +峯 +峰 +峴 +島 +峻 +峽 +崁 +崂 +崆 +崇 +崎 +崑 +崔 +崖 +崗 +崙 +崛 +崧 +崩 +崭 +崴 +崽 +嵇 +嵊 +嵋 +嵌 +嵐 +嵘 +嵩 +嵬 +嵯 +嶂 +嶄 +嶇 +嶋 +嶙 +嶺 +嶼 +嶽 +巅 +巍 +巒 +巔 +巖 +川 +州 +巡 +巢 +工 +左 +巧 +巨 +巩 +巫 +差 +己 +已 +巳 +巴 +巷 +巻 +巽 +巾 +巿 +币 +市 +布 +帅 +帆 +师 +希 +帐 +帑 +帕 +帖 +帘 +帚 +帛 +帜 +帝 +帥 +带 +帧 +師 +席 +帮 +帯 +帰 +帳 +帶 +帷 +常 +帼 +帽 +幀 +幂 +幄 +幅 +幌 +幔 +幕 +幟 +幡 +幢 +幣 +幫 +干 +平 +年 +并 +幸 +幹 +幺 +幻 +幼 +幽 +幾 +广 +庁 +広 +庄 +庆 +庇 +床 +序 +庐 +库 +应 +底 +庖 +店 +庙 +庚 +府 +庞 +废 +庠 +度 +座 +庫 +庭 +庵 +庶 +康 +庸 +庹 +庾 +廁 +廂 +廃 +廈 +廉 +廊 +廓 +廖 +廚 +廝 +廟 +廠 +廢 +廣 +廬 +廳 +延 +廷 +建 +廿 +开 +弁 +异 +弃 +弄 +弈 +弊 +弋 +式 +弑 +弒 +弓 +弔 +引 +弗 +弘 +弛 +弟 +张 +弥 +弦 +弧 +弩 +弭 +弯 +弱 +張 +強 +弹 +强 +弼 +弾 +彅 +彆 +彈 +彌 +彎 +归 +当 +录 +彗 +彙 +彝 +形 +彤 +彥 +彦 +彧 +彩 +彪 +彫 +彬 +彭 +彰 +影 +彷 +役 +彻 +彼 +彿 +往 +征 +径 +待 +徇 +很 +徉 +徊 +律 +後 +徐 +徑 +徒 +従 +徕 +得 +徘 +徙 +徜 +從 +徠 +御 +徨 +復 +循 +徬 +微 +徳 +徴 +徵 +德 +徹 +徼 +徽 +心 +必 +忆 +忌 +忍 +忏 +忐 +忑 +忒 +忖 +志 +忘 +忙 +応 +忠 +忡 +忤 +忧 +忪 +快 +忱 +念 +忻 +忽 +忿 +怀 +态 +怂 +怅 +怆 +怎 +怏 +怒 +怔 +怕 +怖 +怙 +怜 +思 +怠 +怡 +急 +怦 +性 +怨 +怪 +怯 +怵 +总 +怼 +恁 +恃 +恆 +恋 +恍 +恐 +恒 +恕 +恙 +恚 +恢 +恣 +恤 +恥 +恨 +恩 +恪 +恫 +恬 +恭 +息 +恰 +恳 +恵 +恶 +恸 +恺 +恻 +恼 +恿 +悄 +悅 +悉 +悌 +悍 +悔 +悖 +悚 +悟 +悠 +患 +悦 +您 +悩 +悪 +悬 +悯 +悱 +悲 +悴 +悵 +悶 +悸 +悻 +悼 +悽 +情 +惆 +惇 +惊 +惋 +惑 +惕 +惘 +惚 +惜 +惟 +惠 +惡 +惦 +惧 +惨 +惩 +惫 +惬 +惭 +惮 +惯 +惰 +惱 +想 +惴 +惶 +惹 +惺 +愁 +愆 +愈 +愉 +愍 +意 +愕 +愚 +愛 +愜 +感 +愣 +愤 +愧 +愫 +愷 +愿 +慄 +慈 +態 +慌 +慎 +慑 +慕 +慘 +慚 +慟 +慢 +慣 +慧 +慨 +慫 +慮 +慰 +慳 +慵 +慶 +慷 +慾 +憂 +憊 +憋 +憎 +憐 +憑 +憔 +憚 +憤 +憧 +憨 +憩 +憫 +憬 +憲 +憶 +憾 +懂 +懇 +懈 +應 +懊 +懋 +懑 +懒 +懦 +懲 +懵 +懶 +懷 +懸 +懺 +懼 +懾 +懿 +戀 +戈 +戊 +戌 +戍 +戎 +戏 +成 +我 +戒 +戕 +或 +战 +戚 +戛 +戟 +戡 +戦 +截 +戬 +戮 +戰 +戲 +戳 +戴 +戶 +户 +戸 +戻 +戾 +房 +所 +扁 +扇 +扈 +扉 +手 +才 +扎 +扑 +扒 +打 +扔 +払 +托 +扛 +扣 +扦 +执 +扩 +扪 +扫 +扬 +扭 +扮 +扯 +扰 +扱 +扳 +扶 +批 +扼 +找 +承 +技 +抄 +抉 +把 +抑 +抒 +抓 +投 +抖 +抗 +折 +抚 +抛 +抜 +択 +抟 +抠 +抡 +抢 +护 +报 +抨 +披 +抬 +抱 +抵 +抹 +押 +抽 +抿 +拂 +拄 +担 +拆 +拇 +拈 +拉 +拋 +拌 +拍 +拎 +拐 +拒 +拓 +拔 +拖 +拗 +拘 +拙 +拚 +招 +拜 +拟 +拡 +拢 +拣 +拥 +拦 +拧 +拨 +择 +括 +拭 +拮 +拯 +拱 +拳 +拴 +拷 +拼 +拽 +拾 +拿 +持 +挂 +指 +挈 +按 +挎 +挑 +挖 +挙 +挚 +挛 +挝 +挞 +挟 +挠 +挡 +挣 +挤 +挥 +挨 +挪 +挫 +振 +挲 +挹 +挺 +挽 +挾 +捂 +捅 +捆 +捉 +捋 +捌 +捍 +捎 +捏 +捐 +捕 +捞 +损 +捡 +换 +捣 +捧 +捨 +捩 +据 +捱 +捲 +捶 +捷 +捺 +捻 +掀 +掂 +掃 +掇 +授 +掉 +掌 +掏 +掐 +排 +掖 +掘 +掙 +掛 +掠 +採 +探 +掣 +接 +控 +推 +掩 +措 +掬 +掰 +掲 +掳 +掴 +掷 +掸 +掺 +揀 +揃 +揄 +揆 +揉 +揍 +描 +提 +插 +揖 +揚 +換 +握 +揣 +揩 +揪 +揭 +揮 +援 +揶 +揸 +揹 +揽 +搀 +搁 +搂 +搅 +損 +搏 +搐 +搓 +搔 +搖 +搗 +搜 +搞 +搡 +搪 +搬 +搭 +搵 +搶 +携 +搽 +摀 +摁 +摄 +摆 +摇 +摈 +摊 +摒 +摔 +摘 +摞 +摟 +摧 +摩 +摯 +摳 +摸 +摹 +摺 +摻 +撂 +撃 +撅 +撇 +撈 +撐 +撑 +撒 +撓 +撕 +撚 +撞 +撤 +撥 +撩 +撫 +撬 +播 +撮 +撰 +撲 +撵 +撷 +撸 +撻 +撼 +撿 +擀 +擁 +擂 +擄 +擅 +擇 +擊 +擋 +操 +擎 +擒 +擔 +擘 +據 +擞 +擠 +擡 +擢 +擦 +擬 +擰 +擱 +擲 +擴 +擷 +擺 +擼 +擾 +攀 +攏 +攒 +攔 +攘 +攙 +攜 +攝 +攞 +攢 +攣 +攤 +攥 +攪 +攫 +攬 +支 +收 +攸 +改 +攻 +放 +政 +故 +效 +敌 +敍 +敎 +敏 +救 +敕 +敖 +敗 +敘 +教 +敛 +敝 +敞 +敢 +散 +敦 +敬 +数 +敲 +整 +敵 +敷 +數 +斂 +斃 +文 +斋 +斌 +斎 +斐 +斑 +斓 +斗 +料 +斛 +斜 +斟 +斡 +斤 +斥 +斧 +斩 +斫 +斬 +断 +斯 +新 +斷 +方 +於 +施 +旁 +旃 +旅 +旋 +旌 +旎 +族 +旖 +旗 +无 +既 +日 +旦 +旧 +旨 +早 +旬 +旭 +旮 +旱 +时 +旷 +旺 +旻 +昀 +昂 +昆 +昇 +昉 +昊 +昌 +明 +昏 +易 +昔 +昕 +昙 +星 +映 +春 +昧 +昨 +昭 +是 +昱 +昴 +昵 +昶 +昼 +显 +晁 +時 +晃 +晉 +晋 +晌 +晏 +晒 +晓 +晔 +晕 +晖 +晗 +晚 +晝 +晞 +晟 +晤 +晦 +晨 +晩 +普 +景 +晰 +晴 +晶 +晷 +智 +晾 +暂 +暄 +暇 +暈 +暉 +暌 +暐 +暑 +暖 +暗 +暝 +暢 +暧 +暨 +暫 +暮 +暱 +暴 +暸 +暹 +曄 +曆 +曇 +曉 +曖 +曙 +曜 +曝 +曠 +曦 +曬 +曰 +曲 +曳 +更 +書 +曹 +曼 +曾 +替 +最 +會 +月 +有 +朋 +服 +朐 +朔 +朕 +朗 +望 +朝 +期 +朦 +朧 +木 +未 +末 +本 +札 +朮 +术 +朱 +朴 +朵 +机 +朽 +杀 +杂 +权 +杆 +杈 +杉 +李 +杏 +材 +村 +杓 +杖 +杜 +杞 +束 +杠 +条 +来 +杨 +杭 +杯 +杰 +東 +杳 +杵 +杷 +杼 +松 +板 +极 +构 +枇 +枉 +枋 +析 +枕 +林 +枚 +果 +枝 +枢 +枣 +枪 +枫 +枭 +枯 +枰 +枱 +枳 +架 +枷 +枸 +柄 +柏 +某 +柑 +柒 +染 +柔 +柘 +柚 +柜 +柞 +柠 +柢 +查 +柩 +柬 +柯 +柱 +柳 +柴 +柵 +査 +柿 +栀 +栃 +栄 +栅 +标 +栈 +栉 +栋 +栎 +栏 +树 +栓 +栖 +栗 +校 +栩 +株 +样 +核 +根 +格 +栽 +栾 +桀 +桁 +桂 +桃 +桅 +框 +案 +桉 +桌 +桎 +桐 +桑 +桓 +桔 +桜 +桠 +桡 +桢 +档 +桥 +桦 +桧 +桨 +桩 +桶 +桿 +梁 +梅 +梆 +梏 +梓 +梗 +條 +梟 +梢 +梦 +梧 +梨 +梭 +梯 +械 +梳 +梵 +梶 +检 +棂 +棄 +棉 +棋 +棍 +棒 +棕 +棗 +棘 +棚 +棟 +棠 +棣 +棧 +森 +棱 +棲 +棵 +棹 +棺 +椁 +椅 +椋 +植 +椎 +椒 +検 +椪 +椭 +椰 +椹 +椽 +椿 +楂 +楊 +楓 +楔 +楚 +楝 +楞 +楠 +楣 +楨 +楫 +業 +楮 +極 +楷 +楸 +楹 +楼 +楽 +概 +榄 +榆 +榈 +榉 +榔 +榕 +榖 +榛 +榜 +榨 +榫 +榭 +榮 +榱 +榴 +榷 +榻 +槁 +槃 +構 +槌 +槍 +槎 +槐 +槓 +様 +槛 +槟 +槤 +槭 +槲 +槳 +槻 +槽 +槿 +樁 +樂 +樊 +樑 +樓 +標 +樞 +樟 +模 +樣 +権 +横 +樫 +樯 +樱 +樵 +樸 +樹 +樺 +樽 +樾 +橄 +橇 +橋 +橐 +橘 +橙 +機 +橡 +橢 +橫 +橱 +橹 +橼 +檀 +檄 +檎 +檐 +檔 +檗 +檜 +檢 +檬 +檯 +檳 +檸 +檻 +櫃 +櫚 +櫛 +櫥 +櫸 +櫻 +欄 +權 +欒 +欖 +欠 +次 +欢 +欣 +欧 +欲 +欸 +欺 +欽 +款 +歆 +歇 +歉 +歌 +歎 +歐 +歓 +歙 +歛 +歡 +止 +正 +此 +步 +武 +歧 +歩 +歪 +歯 +歲 +歳 +歴 +歷 +歸 +歹 +死 +歼 +殁 +殃 +殆 +殇 +殉 +殊 +残 +殒 +殓 +殖 +殘 +殞 +殡 +殤 +殭 +殯 +殲 +殴 +段 +殷 +殺 +殼 +殿 +毀 +毁 +毂 +毅 +毆 +毋 +母 +毎 +每 +毒 +毓 +比 +毕 +毗 +毘 +毙 +毛 +毡 +毫 +毯 +毽 +氈 +氏 +氐 +民 +氓 +气 +氖 +気 +氙 +氛 +氟 +氡 +氢 +氣 +氤 +氦 +氧 +氨 +氪 +氫 +氮 +氯 +氰 +氲 +水 +氷 +永 +氹 +氾 +汀 +汁 +求 +汆 +汇 +汉 +汎 +汐 +汕 +汗 +汙 +汛 +汝 +汞 +江 +池 +污 +汤 +汨 +汩 +汪 +汰 +汲 +汴 +汶 +汹 +決 +汽 +汾 +沁 +沂 +沃 +沅 +沈 +沉 +沌 +沏 +沐 +沒 +沓 +沖 +沙 +沛 +沟 +没 +沢 +沣 +沥 +沦 +沧 +沪 +沫 +沭 +沮 +沱 +河 +沸 +油 +治 +沼 +沽 +沾 +沿 +況 +泄 +泉 +泊 +泌 +泓 +法 +泗 +泛 +泞 +泠 +泡 +波 +泣 +泥 +注 +泪 +泫 +泮 +泯 +泰 +泱 +泳 +泵 +泷 +泸 +泻 +泼 +泽 +泾 +洁 +洄 +洋 +洒 +洗 +洙 +洛 +洞 +津 +洩 +洪 +洮 +洱 +洲 +洵 +洶 +洸 +洹 +活 +洼 +洽 +派 +流 +浃 +浄 +浅 +浆 +浇 +浊 +测 +济 +浏 +浑 +浒 +浓 +浔 +浙 +浚 +浜 +浣 +浦 +浩 +浪 +浬 +浮 +浯 +浴 +海 +浸 +涂 +涅 +涇 +消 +涉 +涌 +涎 +涓 +涔 +涕 +涙 +涛 +涝 +涞 +涟 +涠 +涡 +涣 +涤 +润 +涧 +涨 +涩 +涪 +涮 +涯 +液 +涵 +涸 +涼 +涿 +淀 +淄 +淅 +淆 +淇 +淋 +淌 +淑 +淒 +淖 +淘 +淙 +淚 +淞 +淡 +淤 +淦 +淨 +淩 +淪 +淫 +淬 +淮 +深 +淳 +淵 +混 +淹 +淺 +添 +淼 +清 +済 +渉 +渊 +渋 +渍 +渎 +渐 +渔 +渗 +渙 +渚 +減 +渝 +渠 +渡 +渣 +渤 +渥 +渦 +温 +測 +渭 +港 +渲 +渴 +游 +渺 +渾 +湃 +湄 +湊 +湍 +湖 +湘 +湛 +湟 +湧 +湫 +湮 +湯 +湳 +湾 +湿 +満 +溃 +溅 +溉 +溏 +源 +準 +溜 +溝 +溟 +溢 +溥 +溧 +溪 +溫 +溯 +溱 +溴 +溶 +溺 +溼 +滁 +滂 +滄 +滅 +滇 +滋 +滌 +滑 +滓 +滔 +滕 +滙 +滚 +滝 +滞 +滟 +满 +滢 +滤 +滥 +滦 +滨 +滩 +滬 +滯 +滲 +滴 +滷 +滸 +滾 +滿 +漁 +漂 +漆 +漉 +漏 +漓 +演 +漕 +漠 +漢 +漣 +漩 +漪 +漫 +漬 +漯 +漱 +漲 +漳 +漸 +漾 +漿 +潆 +潇 +潋 +潍 +潑 +潔 +潘 +潛 +潜 +潞 +潟 +潢 +潤 +潦 +潧 +潭 +潮 +潰 +潴 +潸 +潺 +潼 +澀 +澄 +澆 +澈 +澍 +澎 +澗 +澜 +澡 +澤 +澧 +澱 +澳 +澹 +激 +濁 +濂 +濃 +濑 +濒 +濕 +濘 +濛 +濟 +濠 +濡 +濤 +濫 +濬 +濮 +濯 +濱 +濺 +濾 +瀅 +瀆 +瀉 +瀋 +瀏 +瀑 +瀕 +瀘 +瀚 +瀛 +瀝 +瀞 +瀟 +瀧 +瀨 +瀬 +瀰 +瀾 +灌 +灏 +灑 +灘 +灝 +灞 +灣 +火 +灬 +灭 +灯 +灰 +灵 +灶 +灸 +灼 +災 +灾 +灿 +炀 +炁 +炅 +炉 +炊 +炎 +炒 +炔 +炕 +炖 +炙 +炜 +炫 +炬 +炭 +炮 +炯 +炳 +炷 +炸 +点 +為 +炼 +炽 +烁 +烂 +烃 +烈 +烊 +烏 +烘 +烙 +烛 +烟 +烤 +烦 +烧 +烨 +烩 +烫 +烬 +热 +烯 +烷 +烹 +烽 +焉 +焊 +焕 +焖 +焗 +焘 +焙 +焚 +焜 +無 +焦 +焯 +焰 +焱 +然 +焼 +煅 +煉 +煊 +煌 +煎 +煒 +煖 +煙 +煜 +煞 +煤 +煥 +煦 +照 +煨 +煩 +煮 +煲 +煸 +煽 +熄 +熊 +熏 +熒 +熔 +熙 +熟 +熠 +熨 +熬 +熱 +熵 +熹 +熾 +燁 +燃 +燄 +燈 +燉 +燊 +燎 +燒 +燔 +燕 +燙 +燜 +營 +燥 +燦 +燧 +燭 +燮 +燴 +燻 +燼 +燿 +爆 +爍 +爐 +爛 +爪 +爬 +爭 +爰 +爱 +爲 +爵 +父 +爷 +爸 +爹 +爺 +爻 +爽 +爾 +牆 +片 +版 +牌 +牍 +牒 +牙 +牛 +牝 +牟 +牠 +牡 +牢 +牦 +牧 +物 +牯 +牲 +牴 +牵 +特 +牺 +牽 +犀 +犁 +犄 +犊 +犍 +犒 +犢 +犧 +犬 +犯 +状 +犷 +犸 +犹 +狀 +狂 +狄 +狈 +狎 +狐 +狒 +狗 +狙 +狞 +狠 +狡 +狩 +独 +狭 +狮 +狰 +狱 +狸 +狹 +狼 +狽 +猎 +猕 +猖 +猗 +猙 +猛 +猜 +猝 +猥 +猩 +猪 +猫 +猬 +献 +猴 +猶 +猷 +猾 +猿 +獄 +獅 +獎 +獐 +獒 +獗 +獠 +獣 +獨 +獭 +獰 +獲 +獵 +獷 +獸 +獺 +獻 +獼 +獾 +玄 +率 +玉 +王 +玑 +玖 +玛 +玟 +玠 +玥 +玩 +玫 +玮 +环 +现 +玲 +玳 +玷 +玺 +玻 +珀 +珂 +珅 +珈 +珉 +珊 +珍 +珏 +珐 +珑 +珙 +珞 +珠 +珣 +珥 +珩 +珪 +班 +珮 +珲 +珺 +現 +球 +琅 +理 +琇 +琉 +琊 +琍 +琏 +琐 +琛 +琢 +琥 +琦 +琨 +琪 +琬 +琮 +琰 +琲 +琳 +琴 +琵 +琶 +琺 +琼 +瑀 +瑁 +瑄 +瑋 +瑕 +瑗 +瑙 +瑚 +瑛 +瑜 +瑞 +瑟 +瑠 +瑣 +瑤 +瑩 +瑪 +瑯 +瑰 +瑶 +瑾 +璀 +璁 +璃 +璇 +璉 +璋 +璎 +璐 +璜 +璞 +璟 +璧 +璨 +環 +璽 +璿 +瓊 +瓏 +瓒 +瓜 +瓢 +瓣 +瓤 +瓦 +瓮 +瓯 +瓴 +瓶 +瓷 +甄 +甌 +甕 +甘 +甙 +甚 +甜 +生 +產 +産 +甥 +甦 +用 +甩 +甫 +甬 +甭 +甯 +田 +由 +甲 +申 +电 +男 +甸 +町 +画 +甾 +畀 +畅 +界 +畏 +畑 +畔 +留 +畜 +畝 +畢 +略 +畦 +番 +畫 +異 +畲 +畳 +畴 +當 +畸 +畹 +畿 +疆 +疇 +疊 +疏 +疑 +疔 +疖 +疗 +疙 +疚 +疝 +疟 +疡 +疣 +疤 +疥 +疫 +疮 +疯 +疱 +疲 +疳 +疵 +疸 +疹 +疼 +疽 +疾 +痂 +病 +症 +痈 +痉 +痊 +痍 +痒 +痔 +痕 +痘 +痙 +痛 +痞 +痠 +痢 +痣 +痤 +痧 +痨 +痪 +痫 +痰 +痱 +痴 +痹 +痺 +痼 +痿 +瘀 +瘁 +瘋 +瘍 +瘓 +瘘 +瘙 +瘟 +瘠 +瘡 +瘢 +瘤 +瘦 +瘧 +瘩 +瘪 +瘫 +瘴 +瘸 +瘾 +療 +癇 +癌 +癒 +癖 +癜 +癞 +癡 +癢 +癣 +癥 +癫 +癬 +癮 +癱 +癲 +癸 +発 +登 +發 +白 +百 +皂 +的 +皆 +皇 +皈 +皋 +皎 +皑 +皓 +皖 +皙 +皚 +皮 +皰 +皱 +皴 +皺 +皿 +盂 +盃 +盅 +盆 +盈 +益 +盎 +盏 +盐 +监 +盒 +盔 +盖 +盗 +盘 +盛 +盜 +盞 +盟 +盡 +監 +盤 +盥 +盧 +盪 +目 +盯 +盱 +盲 +直 +相 +盹 +盼 +盾 +省 +眈 +眉 +看 +県 +眙 +眞 +真 +眠 +眦 +眨 +眩 +眯 +眶 +眷 +眸 +眺 +眼 +眾 +着 +睁 +睇 +睏 +睐 +睑 +睛 +睜 +睞 +睡 +睢 +督 +睥 +睦 +睨 +睪 +睫 +睬 +睹 +睽 +睾 +睿 +瞄 +瞅 +瞇 +瞋 +瞌 +瞎 +瞑 +瞒 +瞓 +瞞 +瞟 +瞠 +瞥 +瞧 +瞩 +瞪 +瞬 +瞭 +瞰 +瞳 +瞻 +瞼 +瞿 +矇 +矍 +矗 +矚 +矛 +矜 +矢 +矣 +知 +矩 +矫 +短 +矮 +矯 +石 +矶 +矽 +矾 +矿 +码 +砂 +砌 +砍 +砒 +研 +砖 +砗 +砚 +砝 +砣 +砥 +砧 +砭 +砰 +砲 +破 +砷 +砸 +砺 +砼 +砾 +础 +硅 +硐 +硒 +硕 +硝 +硫 +硬 +确 +硯 +硼 +碁 +碇 +碉 +碌 +碍 +碎 +碑 +碓 +碗 +碘 +碚 +碛 +碟 +碣 +碧 +碩 +碰 +碱 +碳 +碴 +確 +碼 +碾 +磁 +磅 +磊 +磋 +磐 +磕 +磚 +磡 +磨 +磬 +磯 +磲 +磷 +磺 +礁 +礎 +礙 +礡 +礦 +礪 +礫 +礴 +示 +礼 +社 +祀 +祁 +祂 +祇 +祈 +祉 +祎 +祐 +祕 +祖 +祗 +祚 +祛 +祜 +祝 +神 +祟 +祠 +祢 +祥 +票 +祭 +祯 +祷 +祸 +祺 +祿 +禀 +禁 +禄 +禅 +禍 +禎 +福 +禛 +禦 +禧 +禪 +禮 +禱 +禹 +禺 +离 +禽 +禾 +禿 +秀 +私 +秃 +秆 +秉 +秋 +种 +科 +秒 +秘 +租 +秣 +秤 +秦 +秧 +秩 +秭 +积 +称 +秸 +移 +秽 +稀 +稅 +程 +稍 +税 +稔 +稗 +稚 +稜 +稞 +稟 +稠 +稣 +種 +稱 +稲 +稳 +稷 +稹 +稻 +稼 +稽 +稿 +穀 +穂 +穆 +穌 +積 +穎 +穗 +穢 +穩 +穫 +穴 +究 +穷 +穹 +空 +穿 +突 +窃 +窄 +窈 +窍 +窑 +窒 +窓 +窕 +窖 +窗 +窘 +窜 +窝 +窟 +窠 +窥 +窦 +窨 +窩 +窪 +窮 +窯 +窺 +窿 +竄 +竅 +竇 +竊 +立 +竖 +站 +竜 +竞 +竟 +章 +竣 +童 +竭 +端 +競 +竹 +竺 +竽 +竿 +笃 +笆 +笈 +笋 +笏 +笑 +笔 +笙 +笛 +笞 +笠 +符 +笨 +第 +笹 +笺 +笼 +筆 +等 +筊 +筋 +筍 +筏 +筐 +筑 +筒 +答 +策 +筛 +筝 +筠 +筱 +筲 +筵 +筷 +筹 +签 +简 +箇 +箋 +箍 +箏 +箐 +箔 +箕 +算 +箝 +管 +箩 +箫 +箭 +箱 +箴 +箸 +節 +篁 +範 +篆 +篇 +築 +篑 +篓 +篙 +篝 +篠 +篡 +篤 +篩 +篪 +篮 +篱 +篷 +簇 +簌 +簍 +簡 +簦 +簧 +簪 +簫 +簷 +簸 +簽 +簾 +簿 +籁 +籃 +籌 +籍 +籐 +籟 +籠 +籤 +籬 +籮 +籲 +米 +类 +籼 +籽 +粄 +粉 +粑 +粒 +粕 +粗 +粘 +粟 +粤 +粥 +粧 +粪 +粮 +粱 +粲 +粳 +粵 +粹 +粼 +粽 +精 +粿 +糅 +糊 +糍 +糕 +糖 +糗 +糙 +糜 +糞 +糟 +糠 +糧 +糬 +糯 +糰 +糸 +系 +糾 +紀 +紂 +約 +紅 +紉 +紊 +紋 +納 +紐 +紓 +純 +紗 +紘 +紙 +級 +紛 +紜 +素 +紡 +索 +紧 +紫 +紮 +累 +細 +紳 +紹 +紺 +終 +絃 +組 +絆 +経 +結 +絕 +絞 +絡 +絢 +給 +絨 +絮 +統 +絲 +絳 +絵 +絶 +絹 +綁 +綏 +綑 +經 +継 +続 +綜 +綠 +綢 +綦 +綫 +綬 +維 +綱 +網 +綴 +綵 +綸 +綺 +綻 +綽 +綾 +綿 +緊 +緋 +総 +緑 +緒 +緘 +線 +緝 +緞 +締 +緣 +編 +緩 +緬 +緯 +練 +緹 +緻 +縁 +縄 +縈 +縛 +縝 +縣 +縫 +縮 +縱 +縴 +縷 +總 +績 +繁 +繃 +繆 +繇 +繋 +織 +繕 +繚 +繞 +繡 +繩 +繪 +繫 +繭 +繳 +繹 +繼 +繽 +纂 +續 +纍 +纏 +纓 +纔 +纖 +纜 +纠 +红 +纣 +纤 +约 +级 +纨 +纪 +纫 +纬 +纭 +纯 +纰 +纱 +纲 +纳 +纵 +纶 +纷 +纸 +纹 +纺 +纽 +纾 +线 +绀 +练 +组 +绅 +细 +织 +终 +绊 +绍 +绎 +经 +绑 +绒 +结 +绔 +绕 +绘 +给 +绚 +绛 +络 +绝 +绞 +统 +绡 +绢 +绣 +绥 +绦 +继 +绩 +绪 +绫 +续 +绮 +绯 +绰 +绳 +维 +绵 +绶 +绷 +绸 +绻 +综 +绽 +绾 +绿 +缀 +缄 +缅 +缆 +缇 +缈 +缉 +缎 +缓 +缔 +缕 +编 +缘 +缙 +缚 +缜 +缝 +缠 +缢 +缤 +缥 +缨 +缩 +缪 +缭 +缮 +缰 +缱 +缴 +缸 +缺 +缽 +罂 +罄 +罌 +罐 +网 +罔 +罕 +罗 +罚 +罡 +罢 +罩 +罪 +置 +罰 +署 +罵 +罷 +罹 +羁 +羅 +羈 +羊 +羌 +美 +羔 +羚 +羞 +羟 +羡 +羣 +群 +羥 +羧 +羨 +義 +羯 +羲 +羸 +羹 +羽 +羿 +翁 +翅 +翊 +翌 +翎 +習 +翔 +翘 +翟 +翠 +翡 +翦 +翩 +翰 +翱 +翳 +翹 +翻 +翼 +耀 +老 +考 +耄 +者 +耆 +耋 +而 +耍 +耐 +耒 +耕 +耗 +耘 +耙 +耦 +耨 +耳 +耶 +耷 +耸 +耻 +耽 +耿 +聂 +聆 +聊 +聋 +职 +聒 +联 +聖 +聘 +聚 +聞 +聪 +聯 +聰 +聲 +聳 +聴 +聶 +職 +聽 +聾 +聿 +肃 +肄 +肅 +肆 +肇 +肉 +肋 +肌 +肏 +肓 +肖 +肘 +肚 +肛 +肝 +肠 +股 +肢 +肤 +肥 +肩 +肪 +肮 +肯 +肱 +育 +肴 +肺 +肽 +肾 +肿 +胀 +胁 +胃 +胄 +胆 +背 +胍 +胎 +胖 +胚 +胛 +胜 +胝 +胞 +胡 +胤 +胥 +胧 +胫 +胭 +胯 +胰 +胱 +胳 +胴 +胶 +胸 +胺 +能 +脂 +脅 +脆 +脇 +脈 +脉 +脊 +脍 +脏 +脐 +脑 +脓 +脖 +脘 +脚 +脛 +脣 +脩 +脫 +脯 +脱 +脲 +脳 +脸 +脹 +脾 +腆 +腈 +腊 +腋 +腌 +腎 +腐 +腑 +腓 +腔 +腕 +腥 +腦 +腩 +腫 +腭 +腮 +腰 +腱 +腳 +腴 +腸 +腹 +腺 +腻 +腼 +腾 +腿 +膀 +膈 +膊 +膏 +膑 +膘 +膚 +膛 +膜 +膝 +膠 +膦 +膨 +膩 +膳 +膺 +膻 +膽 +膾 +膿 +臀 +臂 +臃 +臆 +臉 +臊 +臍 +臓 +臘 +臟 +臣 +臥 +臧 +臨 +自 +臬 +臭 +至 +致 +臺 +臻 +臼 +臾 +舀 +舂 +舅 +舆 +與 +興 +舉 +舊 +舌 +舍 +舎 +舐 +舒 +舔 +舖 +舗 +舛 +舜 +舞 +舟 +航 +舫 +般 +舰 +舱 +舵 +舶 +舷 +舸 +船 +舺 +舾 +艇 +艋 +艘 +艙 +艦 +艮 +良 +艰 +艱 +色 +艳 +艷 +艹 +艺 +艾 +节 +芃 +芈 +芊 +芋 +芍 +芎 +芒 +芙 +芜 +芝 +芡 +芥 +芦 +芩 +芪 +芫 +芬 +芭 +芮 +芯 +花 +芳 +芷 +芸 +芹 +芻 +芽 +芾 +苁 +苄 +苇 +苋 +苍 +苏 +苑 +苒 +苓 +苔 +苕 +苗 +苛 +苜 +苞 +苟 +苡 +苣 +若 +苦 +苫 +苯 +英 +苷 +苹 +苻 +茁 +茂 +范 +茄 +茅 +茉 +茎 +茏 +茗 +茜 +茧 +茨 +茫 +茬 +茭 +茯 +茱 +茲 +茴 +茵 +茶 +茸 +茹 +茼 +荀 +荃 +荆 +草 +荊 +荏 +荐 +荒 +荔 +荖 +荘 +荚 +荞 +荟 +荠 +荡 +荣 +荤 +荥 +荧 +荨 +荪 +荫 +药 +荳 +荷 +荸 +荻 +荼 +荽 +莅 +莆 +莉 +莊 +莎 +莒 +莓 +莖 +莘 +莞 +莠 +莢 +莧 +莪 +莫 +莱 +莲 +莴 +获 +莹 +莺 +莽 +莿 +菀 +菁 +菅 +菇 +菈 +菊 +菌 +菏 +菓 +菖 +菘 +菜 +菟 +菠 +菡 +菩 +華 +菱 +菲 +菸 +菽 +萁 +萃 +萄 +萊 +萋 +萌 +萍 +萎 +萘 +萝 +萤 +营 +萦 +萧 +萨 +萩 +萬 +萱 +萵 +萸 +萼 +落 +葆 +葉 +著 +葚 +葛 +葡 +董 +葦 +葩 +葫 +葬 +葭 +葯 +葱 +葳 +葵 +葷 +葺 +蒂 +蒋 +蒐 +蒔 +蒙 +蒜 +蒞 +蒟 +蒡 +蒨 +蒲 +蒸 +蒹 +蒻 +蒼 +蒿 +蓁 +蓄 +蓆 +蓉 +蓋 +蓑 +蓓 +蓖 +蓝 +蓟 +蓦 +蓬 +蓮 +蓼 +蓿 +蔑 +蔓 +蔔 +蔗 +蔘 +蔚 +蔡 +蔣 +蔥 +蔫 +蔬 +蔭 +蔵 +蔷 +蔺 +蔻 +蔼 +蔽 +蕁 +蕃 +蕈 +蕉 +蕊 +蕎 +蕙 +蕤 +蕨 +蕩 +蕪 +蕭 +蕲 +蕴 +蕻 +蕾 +薄 +薅 +薇 +薈 +薊 +薏 +薑 +薔 +薙 +薛 +薦 +薨 +薩 +薪 +薬 +薯 +薰 +薹 +藉 +藍 +藏 +藐 +藓 +藕 +藜 +藝 +藤 +藥 +藩 +藹 +藻 +藿 +蘆 +蘇 +蘊 +蘋 +蘑 +蘚 +蘭 +蘸 +蘼 +蘿 +虎 +虏 +虐 +虑 +虔 +處 +虚 +虛 +虜 +虞 +號 +虢 +虧 +虫 +虬 +虱 +虹 +虻 +虽 +虾 +蚀 +蚁 +蚂 +蚊 +蚌 +蚓 +蚕 +蚜 +蚝 +蚣 +蚤 +蚩 +蚪 +蚯 +蚱 +蚵 +蛀 +蛆 +蛇 +蛊 +蛋 +蛎 +蛐 +蛔 +蛙 +蛛 +蛟 +蛤 +蛭 +蛮 +蛰 +蛳 +蛹 +蛻 +蛾 +蜀 +蜂 +蜃 +蜆 +蜇 +蜈 +蜊 +蜍 +蜒 +蜓 +蜕 +蜗 +蜘 +蜚 +蜜 +蜡 +蜢 +蜥 +蜱 +蜴 +蜷 +蜻 +蜿 +蝇 +蝈 +蝉 +蝌 +蝎 +蝕 +蝗 +蝙 +蝟 +蝠 +蝦 +蝨 +蝴 +蝶 +蝸 +蝼 +螂 +螃 +融 +螞 +螢 +螨 +螯 +螳 +螺 +蟀 +蟄 +蟆 +蟋 +蟎 +蟑 +蟒 +蟠 +蟬 +蟲 +蟹 +蟻 +蟾 +蠅 +蠍 +蠔 +蠕 +蠛 +蠟 +蠡 +蠢 +蠣 +蠱 +蠶 +蠹 +蠻 +血 +衄 +衅 +衆 +行 +衍 +術 +衔 +街 +衙 +衛 +衝 +衞 +衡 +衢 +衣 +补 +表 +衩 +衫 +衬 +衮 +衰 +衲 +衷 +衹 +衾 +衿 +袁 +袂 +袄 +袅 +袈 +袋 +袍 +袒 +袖 +袜 +袞 +袤 +袪 +被 +袭 +袱 +裁 +裂 +装 +裆 +裊 +裏 +裔 +裕 +裘 +裙 +補 +裝 +裟 +裡 +裤 +裨 +裱 +裳 +裴 +裸 +裹 +製 +裾 +褂 +複 +褐 +褒 +褓 +褔 +褚 +褥 +褪 +褫 +褲 +褶 +褻 +襁 +襄 +襟 +襠 +襪 +襬 +襯 +襲 +西 +要 +覃 +覆 +覇 +見 +規 +覓 +視 +覚 +覦 +覧 +親 +覬 +観 +覷 +覺 +覽 +觀 +见 +观 +规 +觅 +视 +览 +觉 +觊 +觎 +觐 +觑 +角 +觞 +解 +觥 +触 +觸 +言 +訂 +計 +訊 +討 +訓 +訕 +訖 +託 +記 +訛 +訝 +訟 +訣 +訥 +訪 +設 +許 +訳 +訴 +訶 +診 +註 +証 +詆 +詐 +詔 +評 +詛 +詞 +詠 +詡 +詢 +詣 +試 +詩 +詫 +詬 +詭 +詮 +詰 +話 +該 +詳 +詹 +詼 +誅 +誇 +誉 +誌 +認 +誓 +誕 +誘 +語 +誠 +誡 +誣 +誤 +誥 +誦 +誨 +說 +説 +読 +誰 +課 +誹 +誼 +調 +諄 +談 +請 +諏 +諒 +論 +諗 +諜 +諡 +諦 +諧 +諫 +諭 +諮 +諱 +諳 +諷 +諸 +諺 +諾 +謀 +謁 +謂 +謄 +謊 +謎 +謐 +謔 +謗 +謙 +講 +謝 +謠 +謨 +謬 +謹 +謾 +譁 +證 +譎 +譏 +識 +譙 +譚 +譜 +警 +譬 +譯 +議 +譲 +譴 +護 +譽 +讀 +變 +讓 +讚 +讞 +计 +订 +认 +讥 +讧 +讨 +让 +讪 +讫 +训 +议 +讯 +记 +讲 +讳 +讴 +讶 +讷 +许 +讹 +论 +讼 +讽 +设 +访 +诀 +证 +诃 +评 +诅 +识 +诈 +诉 +诊 +诋 +词 +诏 +译 +试 +诗 +诘 +诙 +诚 +诛 +话 +诞 +诟 +诠 +诡 +询 +诣 +诤 +该 +详 +诧 +诩 +诫 +诬 +语 +误 +诰 +诱 +诲 +说 +诵 +诶 +请 +诸 +诺 +读 +诽 +课 +诿 +谀 +谁 +调 +谄 +谅 +谆 +谈 +谊 +谋 +谌 +谍 +谎 +谏 +谐 +谑 +谒 +谓 +谔 +谕 +谗 +谘 +谙 +谚 +谛 +谜 +谟 +谢 +谣 +谤 +谥 +谦 +谧 +谨 +谩 +谪 +谬 +谭 +谯 +谱 +谲 +谴 +谶 +谷 +豁 +豆 +豇 +豈 +豉 +豊 +豌 +豎 +豐 +豔 +豚 +象 +豢 +豪 +豫 +豬 +豹 +豺 +貂 +貅 +貌 +貓 +貔 +貘 +貝 +貞 +負 +財 +貢 +貧 +貨 +販 +貪 +貫 +責 +貯 +貰 +貳 +貴 +貶 +買 +貸 +費 +貼 +貽 +貿 +賀 +賁 +賂 +賃 +賄 +資 +賈 +賊 +賑 +賓 +賜 +賞 +賠 +賡 +賢 +賣 +賤 +賦 +質 +賬 +賭 +賴 +賺 +購 +賽 +贅 +贈 +贊 +贍 +贏 +贓 +贖 +贛 +贝 +贞 +负 +贡 +财 +责 +贤 +败 +账 +货 +质 +贩 +贪 +贫 +贬 +购 +贮 +贯 +贰 +贱 +贲 +贴 +贵 +贷 +贸 +费 +贺 +贻 +贼 +贾 +贿 +赁 +赂 +赃 +资 +赅 +赈 +赊 +赋 +赌 +赎 +赏 +赐 +赓 +赔 +赖 +赘 +赚 +赛 +赝 +赞 +赠 +赡 +赢 +赣 +赤 +赦 +赧 +赫 +赭 +走 +赳 +赴 +赵 +赶 +起 +趁 +超 +越 +趋 +趕 +趙 +趟 +趣 +趨 +足 +趴 +趵 +趸 +趺 +趾 +跃 +跄 +跆 +跋 +跌 +跎 +跑 +跖 +跚 +跛 +距 +跟 +跡 +跤 +跨 +跩 +跪 +路 +跳 +践 +跷 +跹 +跺 +跻 +踉 +踊 +踌 +踏 +踐 +踝 +踞 +踟 +踢 +踩 +踪 +踮 +踱 +踴 +踵 +踹 +蹂 +蹄 +蹇 +蹈 +蹉 +蹊 +蹋 +蹑 +蹒 +蹙 +蹟 +蹣 +蹤 +蹦 +蹩 +蹬 +蹭 +蹲 +蹴 +蹶 +蹺 +蹼 +蹿 +躁 +躇 +躉 +躊 +躋 +躍 +躏 +躪 +身 +躬 +躯 +躲 +躺 +軀 +車 +軋 +軌 +軍 +軒 +軟 +転 +軸 +軼 +軽 +軾 +較 +載 +輒 +輓 +輔 +輕 +輛 +輝 +輟 +輩 +輪 +輯 +輸 +輻 +輾 +輿 +轄 +轅 +轆 +轉 +轍 +轎 +轟 +车 +轧 +轨 +轩 +转 +轭 +轮 +软 +轰 +轲 +轴 +轶 +轻 +轼 +载 +轿 +较 +辄 +辅 +辆 +辇 +辈 +辉 +辊 +辍 +辐 +辑 +输 +辕 +辖 +辗 +辘 +辙 +辛 +辜 +辞 +辟 +辣 +辦 +辨 +辩 +辫 +辭 +辮 +辯 +辰 +辱 +農 +边 +辺 +辻 +込 +辽 +达 +迁 +迂 +迄 +迅 +过 +迈 +迎 +运 +近 +返 +还 +这 +进 +远 +违 +连 +迟 +迢 +迤 +迥 +迦 +迩 +迪 +迫 +迭 +述 +迴 +迷 +迸 +迹 +迺 +追 +退 +送 +适 +逃 +逅 +逆 +选 +逊 +逍 +透 +逐 +递 +途 +逕 +逗 +這 +通 +逛 +逝 +逞 +速 +造 +逢 +連 +逮 +週 +進 +逵 +逶 +逸 +逻 +逼 +逾 +遁 +遂 +遅 +遇 +遊 +運 +遍 +過 +遏 +遐 +遑 +遒 +道 +達 +違 +遗 +遙 +遛 +遜 +遞 +遠 +遢 +遣 +遥 +遨 +適 +遭 +遮 +遲 +遴 +遵 +遶 +遷 +選 +遺 +遼 +遽 +避 +邀 +邁 +邂 +邃 +還 +邇 +邈 +邊 +邋 +邏 +邑 +邓 +邕 +邛 +邝 +邢 +那 +邦 +邨 +邪 +邬 +邮 +邯 +邰 +邱 +邳 +邵 +邸 +邹 +邺 +邻 +郁 +郅 +郊 +郎 +郑 +郜 +郝 +郡 +郢 +郤 +郦 +郧 +部 +郫 +郭 +郴 +郵 +郷 +郸 +都 +鄂 +鄉 +鄒 +鄔 +鄙 +鄞 +鄢 +鄧 +鄭 +鄰 +鄱 +鄲 +鄺 +酉 +酊 +酋 +酌 +配 +酐 +酒 +酗 +酚 +酝 +酢 +酣 +酥 +酩 +酪 +酬 +酮 +酯 +酰 +酱 +酵 +酶 +酷 +酸 +酿 +醃 +醇 +醉 +醋 +醍 +醐 +醒 +醚 +醛 +醜 +醞 +醣 +醪 +醫 +醬 +醮 +醯 +醴 +醺 +釀 +釁 +采 +釉 +释 +釋 +里 +重 +野 +量 +釐 +金 +釗 +釘 +釜 +針 +釣 +釦 +釧 +釵 +鈀 +鈉 +鈍 +鈎 +鈔 +鈕 +鈞 +鈣 +鈦 +鈪 +鈴 +鈺 +鈾 +鉀 +鉄 +鉅 +鉉 +鉑 +鉗 +鉚 +鉛 +鉤 +鉴 +鉻 +銀 +銃 +銅 +銑 +銓 +銖 +銘 +銜 +銬 +銭 +銮 +銳 +銷 +銹 +鋁 +鋅 +鋒 +鋤 +鋪 +鋰 +鋸 +鋼 +錄 +錐 +錘 +錚 +錠 +錢 +錦 +錨 +錫 +錮 +錯 +録 +錳 +錶 +鍊 +鍋 +鍍 +鍛 +鍥 +鍰 +鍵 +鍺 +鍾 +鎂 +鎊 +鎌 +鎏 +鎔 +鎖 +鎗 +鎚 +鎧 +鎬 +鎮 +鎳 +鏈 +鏖 +鏗 +鏘 +鏞 +鏟 +鏡 +鏢 +鏤 +鏽 +鐘 +鐮 +鐲 +鐳 +鐵 +鐸 +鐺 +鑄 +鑊 +鑑 +鑒 +鑣 +鑫 +鑰 +鑲 +鑼 +鑽 +鑾 +鑿 +针 +钉 +钊 +钎 +钏 +钒 +钓 +钗 +钙 +钛 +钜 +钝 +钞 +钟 +钠 +钡 +钢 +钣 +钤 +钥 +钦 +钧 +钨 +钩 +钮 +钯 +钰 +钱 +钳 +钴 +钵 +钺 +钻 +钼 +钾 +钿 +铀 +铁 +铂 +铃 +铄 +铅 +铆 +铉 +铎 +铐 +铛 +铜 +铝 +铠 +铡 +铢 +铣 +铤 +铨 +铩 +铬 +铭 +铮 +铰 +铲 +铵 +银 +铸 +铺 +链 +铿 +销 +锁 +锂 +锄 +锅 +锆 +锈 +锉 +锋 +锌 +锏 +锐 +锑 +错 +锚 +锟 +锡 +锢 +锣 +锤 +锥 +锦 +锭 +键 +锯 +锰 +锲 +锵 +锹 +锺 +锻 +镀 +镁 +镂 +镇 +镉 +镌 +镍 +镐 +镑 +镕 +镖 +镗 +镛 +镜 +镣 +镭 +镯 +镰 +镳 +镶 +長 +长 +門 +閃 +閉 +開 +閎 +閏 +閑 +閒 +間 +閔 +閘 +閡 +関 +閣 +閥 +閨 +閩 +閱 +閲 +閹 +閻 +閾 +闆 +闇 +闊 +闌 +闍 +闔 +闕 +闖 +闘 +關 +闡 +闢 +门 +闪 +闫 +闭 +问 +闯 +闰 +闲 +间 +闵 +闷 +闸 +闹 +闺 +闻 +闽 +闾 +阀 +阁 +阂 +阅 +阆 +阇 +阈 +阉 +阎 +阐 +阑 +阔 +阕 +阖 +阙 +阚 +阜 +队 +阡 +阪 +阮 +阱 +防 +阳 +阴 +阵 +阶 +阻 +阿 +陀 +陂 +附 +际 +陆 +陇 +陈 +陋 +陌 +降 +限 +陕 +陛 +陝 +陞 +陟 +陡 +院 +陣 +除 +陨 +险 +陪 +陰 +陲 +陳 +陵 +陶 +陷 +陸 +険 +陽 +隅 +隆 +隈 +隊 +隋 +隍 +階 +随 +隐 +隔 +隕 +隘 +隙 +際 +障 +隠 +隣 +隧 +隨 +險 +隱 +隴 +隶 +隸 +隻 +隼 +隽 +难 +雀 +雁 +雄 +雅 +集 +雇 +雉 +雋 +雌 +雍 +雎 +雏 +雑 +雒 +雕 +雖 +雙 +雛 +雜 +雞 +離 +難 +雨 +雪 +雯 +雰 +雲 +雳 +零 +雷 +雹 +電 +雾 +需 +霁 +霄 +霆 +震 +霈 +霉 +霊 +霍 +霎 +霏 +霑 +霓 +霖 +霜 +霞 +霧 +霭 +霰 +露 +霸 +霹 +霽 +霾 +靂 +靄 +靈 +青 +靓 +靖 +静 +靚 +靛 +靜 +非 +靠 +靡 +面 +靥 +靦 +革 +靳 +靴 +靶 +靼 +鞅 +鞋 +鞍 +鞏 +鞑 +鞘 +鞠 +鞣 +鞦 +鞭 +韆 +韋 +韌 +韓 +韜 +韦 +韧 +韩 +韬 +韭 +音 +韵 +韶 +韻 +響 +頁 +頂 +頃 +項 +順 +須 +頌 +預 +頑 +頒 +頓 +頗 +領 +頜 +頡 +頤 +頫 +頭 +頰 +頷 +頸 +頹 +頻 +頼 +顆 +題 +額 +顎 +顏 +顔 +願 +顛 +類 +顧 +顫 +顯 +顱 +顴 +页 +顶 +顷 +项 +顺 +须 +顼 +顽 +顾 +顿 +颁 +颂 +预 +颅 +领 +颇 +颈 +颉 +颊 +颌 +颍 +颐 +频 +颓 +颔 +颖 +颗 +题 +颚 +颛 +颜 +额 +颞 +颠 +颡 +颢 +颤 +颦 +颧 +風 +颯 +颱 +颳 +颶 +颼 +飄 +飆 +风 +飒 +飓 +飕 +飘 +飙 +飚 +飛 +飞 +食 +飢 +飨 +飩 +飪 +飯 +飲 +飼 +飽 +飾 +餃 +餅 +餉 +養 +餌 +餐 +餒 +餓 +餘 +餚 +餛 +餞 +餡 +館 +餮 +餵 +餾 +饅 +饈 +饋 +饌 +饍 +饑 +饒 +饕 +饗 +饞 +饥 +饨 +饪 +饬 +饭 +饮 +饯 +饰 +饱 +饲 +饴 +饵 +饶 +饷 +饺 +饼 +饽 +饿 +馀 +馁 +馄 +馅 +馆 +馈 +馋 +馍 +馏 +馒 +馔 +首 +馗 +香 +馥 +馨 +馬 +馭 +馮 +馳 +馴 +駁 +駄 +駅 +駆 +駐 +駒 +駕 +駛 +駝 +駭 +駱 +駿 +騁 +騎 +騏 +験 +騙 +騨 +騰 +騷 +驀 +驅 +驊 +驍 +驒 +驕 +驗 +驚 +驛 +驟 +驢 +驥 +马 +驭 +驮 +驯 +驰 +驱 +驳 +驴 +驶 +驷 +驸 +驹 +驻 +驼 +驾 +驿 +骁 +骂 +骄 +骅 +骆 +骇 +骈 +骊 +骋 +验 +骏 +骐 +骑 +骗 +骚 +骛 +骜 +骞 +骠 +骡 +骤 +骥 +骧 +骨 +骯 +骰 +骶 +骷 +骸 +骼 +髂 +髅 +髋 +髏 +髒 +髓 +體 +髖 +高 +髦 +髪 +髮 +髯 +髻 +鬃 +鬆 +鬍 +鬓 +鬚 +鬟 +鬢 +鬣 +鬥 +鬧 +鬱 +鬼 +魁 +魂 +魄 +魅 +魇 +魍 +魏 +魔 +魘 +魚 +魯 +魷 +鮑 +鮨 +鮪 +鮭 +鮮 +鯉 +鯊 +鯖 +鯛 +鯨 +鯰 +鯽 +鰍 +鰓 +鰭 +鰲 +鰻 +鰾 +鱈 +鱉 +鱔 +鱗 +鱷 +鱸 +鱼 +鱿 +鲁 +鲈 +鲍 +鲑 +鲛 +鲜 +鲟 +鲢 +鲤 +鲨 +鲫 +鲱 +鲲 +鲶 +鲷 +鲸 +鳃 +鳄 +鳅 +鳌 +鳍 +鳕 +鳖 +鳗 +鳝 +鳞 +鳥 +鳩 +鳳 +鳴 +鳶 +鴉 +鴕 +鴛 +鴦 +鴨 +鴻 +鴿 +鵑 +鵜 +鵝 +鵡 +鵬 +鵰 +鵲 +鶘 +鶩 +鶯 +鶴 +鷗 +鷲 +鷹 +鷺 +鸚 +鸞 +鸟 +鸠 +鸡 +鸢 +鸣 +鸥 +鸦 +鸨 +鸪 +鸭 +鸯 +鸳 +鸵 +鸽 +鸾 +鸿 +鹂 +鹃 +鹄 +鹅 +鹈 +鹉 +鹊 +鹌 +鹏 +鹑 +鹕 +鹘 +鹜 +鹞 +鹤 +鹦 +鹧 +鹫 +鹭 +鹰 +鹳 +鹵 +鹹 +鹼 +鹽 +鹿 +麂 +麋 +麒 +麓 +麗 +麝 +麟 +麥 +麦 +麩 +麴 +麵 +麸 +麺 +麻 +麼 +麽 +麾 +黃 +黄 +黍 +黎 +黏 +黑 +黒 +黔 +默 +黛 +黜 +黝 +點 +黠 +黨 +黯 +黴 +鼋 +鼎 +鼐 +鼓 +鼠 +鼬 +鼹 +鼻 +鼾 +齁 +齊 +齋 +齐 +齒 +齡 +齢 +齣 +齦 +齿 +龄 +龅 +龈 +龊 +龋 +龌 +龍 +龐 +龔 +龕 +龙 +龚 +龛 +龜 +龟 +︰ +︱ +︶ +︿ +﹁ +﹂ +﹍ +﹏ +﹐ +﹑ +﹒ +﹔ +﹕ +﹖ +﹗ +﹙ +﹚ +﹝ +﹞ +﹡ +﹣ +！ +＂ +＃ +＄ +％ +＆ +＇ +（ +） +＊ +＋ +， +－ +． +／ +０ +１ +２ +３ +４ +５ +６ +７ +８ +９ +： +； +＜ +＝ +＞ +？ +＠ +［ +＼ +］ +＾ +＿ +｀ +ａ +ｂ +ｃ +ｄ +ｅ +ｆ +ｇ +ｈ +ｉ +ｊ +ｋ +ｌ +ｍ +ｎ +ｏ +ｐ +ｑ +ｒ +ｓ +ｔ +ｕ +ｖ +ｗ +ｘ +ｙ +ｚ +｛ +｜ +｝ +～ +｡ +｢ +｣ +､ +･ +ｯ +ｰ +ｲ +ｸ +ｼ +ｽ +ﾄ +ﾉ +ﾌ +ﾗ +ﾙ +ﾝ +ﾞ +ﾟ +￣ +￥ +👍 +🔥 +😂 +😎 +... +yam +10 +2017 +12 +11 +2016 +20 +30 +15 +06 +lofter +##s +2015 +by +16 +14 +18 +13 +24 +17 +2014 +21 +##0 +22 +19 +25 +23 +com +100 +00 +05 +2013 +##a +03 +09 +08 +28 +##2 +50 +01 +04 +##1 +27 +02 +2012 +##3 +26 +##e +07 +##8 +##5 +##6 +##4 +##9 +##7 +29 +2011 +40 +##t +2010 +##o +##d +##i +2009 +##n +app +www +the +##m +31 +##c +##l +##y +##r +##g +2008 +60 +http +200 +qq +##p +80 +##f +google +pixnet +90 +cookies +tripadvisor +500 +##er +##k +35 +##h +facebook +2007 +2000 +70 +##b +of +##x +##u +45 +300 +iphone +32 +1000 +2006 +48 +ip +36 +in +38 +3d +##w +##ing +55 +ctrip +##on +##v +33 +##の +to +34 +400 +id +2005 +it +37 +windows +llc +top +99 +42 +39 +000 +led +at +##an +41 +51 +52 +46 +49 +43 +53 +44 +##z +android +58 +and +59 +2004 +56 +vr +##か +5000 +2003 +47 +blogthis +twitter +54 +##le +150 +ok +2018 +57 +75 +cn +no +ios +##in +##mm +##00 +800 +on +te +3000 +65 +2001 +360 +95 +ig +lv +120 +##ng +##を +##us +##に +pc +てす +── +600 +##te +85 +2002 +88 +##ed +html +ncc +wifi +email +64 +blog +is +##10 +##て +mail +online +##al +dvd +##ic +studio +##は +##℃ +##ia +##と +line +vip +72 +##q +98 +##ce +##en +for +##is +##ra +##es +##j +usb +net +cp +1999 +asia +4g +##cm +diy +new +3c +##お +ta +66 +language +vs +apple +tw +86 +web +##ne +ipad +62 +you +##re +101 +68 +##tion +ps +de +bt +pony +atm +##2017 +1998 +67 +##ch +ceo +##or +go +##na +av +pro +cafe +96 +pinterest +97 +63 +pixstyleme3c +##ta +more +said +##2016 +1997 +mp3 +700 +##ll +nba +jun +##20 +92 +tv +1995 +pm +61 +76 +nbsp +250 +##ie +linux +##ma +cd +110 +hd +##17 +78 +##ion +77 +6000 +am +##th +##st +94 +##se +##et +69 +180 +gdp +my +105 +81 +abc +89 +flash +79 +one +93 +1990 +1996 +##ck +gps +##も +##ly +web885 +106 +2020 +91 +##ge +4000 +1500 +xd +boss +isbn +1994 +org +##ry +me +love +##11 +0fork +73 +##12 +3g +##ter +##ar +71 +82 +##la +hotel +130 +1970 +pk +83 +87 +140 +ie +##os +##30 +##el +74 +##50 +seo +cpu +##ml +p2p +84 +may +##る +sun +tue +internet +cc +posted +youtube +##at +##ン +##man +ii +##ル +##15 +abs +nt +pdf +yahoo +ago +1980 +##it +news +mac +104 +##てす +##me +##り +java +1992 +spa +##de +##nt +hk +all +plus +la +1993 +##mb +##16 +##ve +west +##da +160 +air +##い +##ps +から +##to +1989 +logo +htc +php +https +fi +momo +##son +sat +##ke +##80 +ebd +suv +wi +day +apk +##88 +##um +mv +galaxy +wiki +or +brake +##ス +1200 +する +this +1991 +mon +##こ +❤2017 +po +##ない +javascript +life +home +june +##ss +system +900 +##ー +##０ +pp +1988 +world +fb +4k +br +##as +ic +ai +leonardo +safari +##60 +live +free +xx +wed +win7 +kiehl +##co +lg +o2o +##go +us +235 +1949 +mm +しい +vfm +kanye +##90 +##2015 +##id +jr +##ey +123 +rss +##sa +##ro +##am +##no +thu +fri +350 +##sh +##ki +103 +comments +name +##のて +##pe +##ine +max +1987 +8000 +uber +##mi +##ton +wordpress +office +1986 +1985 +##ment +107 +bd +win10 +##ld +##li +gmail +bb +dior +##rs +##ri +##rd +##ます +up +cad +##® +dr +して +read +##21 +をお +##io +##99 +url +1984 +pvc +paypal +show +policy +##40 +##ty +##18 +with +##★ +##01 +txt +102 +##ba +dna +from +post +mini +ar +taiwan +john +##ga +privacy +agoda +##13 +##ny +word +##24 +##22 +##by +##ur +##hz +1982 +##ang +265 +cookie +netscape +108 +##ka +##～ +##ad +house +share +note +ibm +code +hello +nike +sim +survey +##016 +1979 +1950 +wikia +##32 +##017 +5g +cbc +##tor +##kg +1983 +##rt +##14 +campaign +store +2500 +os +##ct +##ts +##° +170 +api +##ns +365 +excel +##な +##ao +##ら +##し +～～ +##nd +university +163 +には +518 +##70 +##ya +##il +##25 +pierre +ipo +0020 +897 +##23 +hotels +##ian +のお +125 +years +6606 +##ers +##26 +high +##day +time +##ay +bug +##line +##く +##す +##be +xp +talk2yam +yamservice +10000 +coco +##dy +sony +##ies +1978 +microsoft +david +people +##ha +1960 +instagram +intel +その +##ot +iso +1981 +##va +115 +##mo +##land +xxx +man +co +ltxsw +##ation +baby +220 +##pa +##ol +1945 +7000 +tag +450 +##ue +msn +##31 +oppo +##ト +##ca +control +##om +st +chrome +##ure +##ん +be +##き +lol +##19 +した +##bo +240 +lady +##100 +##way +##から +4600 +##ko +##do +##un +4s +corporation +168 +##ni +herme +##28 +ｃｐ +978 +##up +##06 +ui +##ds +ppt +admin +three +します +bbc +re +128 +##48 +ca +##015 +##35 +hp +##ee +tpp +##た +##ive +×× +root +##cc +##ました +##ble +##ity +adobe +park +114 +et +oled +city +##ex +##ler +##ap +china +##book +20000 +view +##ice +global +##km +your +hong +##mg +out +##ms +ng +ebay +##29 +menu +ubuntu +##cy +rom +##view +open +ktv +do +server +##lo +if +english +##ね +##５ +##oo +1600 +##02 +step1 +kong +club +135 +july +inc +1976 +mr +hi +##net +touch +##ls +##ii +michael +lcd +##05 +##33 +phone +james +step2 +1300 +ios9 +##box +dc +##２ +##ley +samsung +111 +280 +pokemon +css +##ent +##les +いいえ +##１ +s8 +atom +play +bmw +##said +sa +etf +ctrl +♥yoyo♥ +##55 +2025 +##2014 +##66 +adidas +amazon +1958 +##ber +##ner +visa +##77 +##der +1800 +connectivity +##hi +firefox +109 +118 +hr +so +style +mark +pop +ol +skip +1975 +as +##27 +##ir +##61 +190 +mba +##う +##ai +le +##ver +1900 +cafe2017 +lte +super +113 +129 +##ron +amd +like +##☆ +are +##ster +we +##sk +paul +data +international +##ft +longchamp +ssd +good +##ート +##ti +reply +##my +↓↓↓ +apr +star +##ker +source +136 +js +112 +get +force +photo +##one +126 +##2013 +##ow +link +bbs +1972 +goods +##lin +python +119 +##ip +game +##ics +##ません +blue +##● +520 +##45 +page +itunes +##03 +1955 +260 +1968 +gt +gif +618 +##ff +##47 +group +くたさい +about +bar +ganji +##nce +music +lee +not +1977 +1971 +1973 +##per +an +faq +comment +##って +days +##ock +116 +##bs +1974 +1969 +v1 +player +1956 +xbox +sql +fm +f1 +139 +##ah +210 +##lv +##mp +##000 +melody +1957 +##３ +550 +17life +199 +1966 +xml +market +##au +##71 +999 +##04 +what +gl +##95 +##age +tips +##68 +book +##ting +mysql +can +1959 +230 +##ung +wonderland +watch +10℃ +##ction +9000 +mar +mobile +1946 +1962 +article +##db +part +▲top +party +って +1967 +1964 +1948 +##07 +##ore +##op +この +dj +##78 +##38 +010 +main +225 +1965 +##ong +art +320 +ad +134 +020 +##73 +117 +pm2 +japan +228 +##08 +ts +1963 +##ica +der +sm +##36 +2019 +##wa +ct +##７ +##や +##64 +1937 +homemesh +search +##85 +##れは +##tv +##di +macbook +##９ +##くたさい +service +##♥ +type +った +750 +##ier +##si +##75 +##います +##ok +best +##ット +goris +lock +##った +cf +3m +big +##ut +ftp +carol +##vi +１０ +1961 +happy +sd +##ac +122 +anti +pe +cnn +iii +1920 +138 +##ラ +1940 +esp +jan +tags +##98 +##51 +august +vol +##86 +154 +##™ +##fs +##れ +##sion +design +ac +##ム +press +jordan +ppp +that +key +check +##６ +##tt +##㎡ +1080p +##lt +power +##42 +1952 +##bc +vivi +##ック +he +133 +121 +jpg +##rry +201 +175 +3500 +1947 +nb +##ted +##rn +しています +1954 +usd +##t00 +master +##ンク +001 +model +##58 +al +##09 +1953 +##34 +ram +goo +ても +##ui +127 +1930 +red +##ary +rpg +item +##pm +##41 +270 +##za +project +##2012 +hot +td +blogabstract +##ger +##62 +650 +##44 +gr2 +##します +##ｍ +black +electronic +nfc +year +asus +また +html5 +cindy +##hd +m3 +132 +esc +##od +booking +##53 +fed +tvb +##81 +##ina +mit +165 +##いる +chan +192 +distribution +next +になる +peter +bios +steam +cm +1941 +にも +pk10 +##ix +##65 +##91 +dec +nasa +##ana +icecat +00z +b1 +will +##46 +li +se +##ji +##み +##ard +oct +##ain +jp +##ze +##bi +cio +##56 +smart +h5 +##39 +##port +curve +vpn +##nm +##dia +utc +##あり +12345678910 +##52 +rmvb +chanel +a4 +miss +##and +##im +media +who +##63 +she +girl +5s +124 +vera +##して +class +vivo +king +##フ +##ei +national +ab +1951 +5cm +888 +145 +ipod +ap +1100 +5mm +211 +ms +2756 +##69 +mp4 +msci +##po +##89 +131 +mg +index +380 +##bit +##out +##zz +##97 +##67 +158 +apec +##８ +photoshop +opec +￥799 +ては +##96 +##tes +##ast +2g +○○ +##ール +￥2899 +##ling +##よ +##ory +1938 +##ical +kitty +content +##43 +step3 +##cn +win8 +155 +vc +1400 +iphone7 +robert +##した +tcl +137 +beauty +##87 +en +dollars +##ys +##oc +step +pay +yy +a1 +##2011 +##lly +##ks +##♪ +1939 +188 +download +1944 +sep +exe +ph +います +school +gb +center +pr +street +##board +uv +##37 +##lan +winrar +##que +##ua +##com +1942 +1936 +480 +gpu +##４ +ettoday +fu +tom +##54 +##ren +##via +149 +##72 +b2b +144 +##79 +##tch +rose +arm +mb +##49 +##ial +##nn +nvidia +step4 +mvp +00㎡ +york +156 +##イ +how +cpi +591 +2765 +gov +kg +joe +##xx +mandy +pa +##ser +copyright +fashion +1935 +don +##け +ecu +##ist +##art +erp +wap +have +##lm +talk +##ek +##ning +##if +ch +##ite +video +1943 +cs +san +iot +look +##84 +##2010 +##ku +october +##ux +trump +##hs +##ide +box +141 +first +##ins +april +##ight +##83 +185 +angel +protected +aa +151 +162 +x1 +m2 +##fe +##× +##ho +size +143 +min +ofo +fun +gomaji +ex +hdmi +food +dns +march +chris +kevin +##のか +##lla +##pp +##ec +ag +ems +6s +720p +##rm +##ham +off +##92 +asp +team +fandom +ed +299 +▌♥ +##ell +info +されています +##82 +sina +4066 +161 +##able +##ctor +330 +399 +315 +dll +rights +ltd +idc +jul +3kg +1927 +142 +ma +surface +##76 +##ク +～～～ +304 +mall +eps +146 +green +##59 +map +space +donald +v2 +sodu +##light +1931 +148 +1700 +まて +310 +reserved +htm +##han +##57 +2d +178 +mod +##ise +##tions +152 +ti +##shi +doc +1933 +icp +055 +wang +##ram +shopping +aug +##pi +##well +now +wam +b2 +からお +##hu +236 +1928 +##gb +266 +f2 +##93 +153 +mix +##ef +##uan +bwl +##plus +##res +core +##ess +tea +5℃ +hktvmall +nhk +##ate +list +##ese +301 +feb +4m +inn +ての +nov +159 +12345 +daniel +##ci +pass +##bet +##nk +coffee +202 +ssl +airbnb +##ute +fbi +woshipm +skype +ea +cg +sp +##fc +##www +yes +edge +alt +007 +##94 +fpga +##ght +##gs +iso9001 +さい +##ile +##wood +##uo +image +lin +icon +american +##em +1932 +set +says +##king +##tive +blogger +##74 +なと +256 +147 +##ox +##zy +##red +##ium +##lf +nokia +claire +##リ +##ding +november +lohas +##500 +##tic +##マ +##cs +##ある +##che +##ire +##gy +##ult +db +january +win +##カ +166 +road +ptt +##ま +##つ +198 +##fa +##mer +anna +pchome +はい +udn +ef +420 +##time +##tte +2030 +##ア +g20 +white +かかります +1929 +308 +garden +eleven +di +##おります +chen +309b +777 +172 +young +cosplay +ちてない +4500 +bat +##123 +##tra +##ては +kindle +npc +steve +etc +##ern +##｜ +call +xperia +ces +travel +sk +s7 +##ous +1934 +##int +みいたたけます +183 +edu +file +cho +qr +##car +##our +186 +##ant +##ｄ +eric +1914 +rends +##jo +##する +mastercard +##2000 +kb +##min +290 +##ino +vista +##ris +##ud +jack +2400 +##set +169 +pos +1912 +##her +##ou +taipei +しく +205 +beta +##ませんか +232 +##fi +express +255 +body +##ill +aphojoy +user +december +meiki +##ick +tweet +richard +##av +##ᆫ +iphone6 +##dd +ちてすか +views +##mark +321 +pd +##００ +times +##▲ +level +##ash +10g +point +5l +##ome +208 +koreanmall +##ak +george +q2 +206 +wma +tcp +##200 +スタッフ +full +mlb +##lle +##watch +tm +run +179 +911 +smith +business +##und +1919 +color +##tal +222 +171 +##less +moon +4399 +##rl +update +pcb +shop +499 +157 +little +なし +end +##mhz +van +dsp +easy +660 +##house +##key +history +##ｏ +oh +##001 +##hy +##web +oem +let +was +##2009 +##gg +review +##wan +182 +##°c +203 +uc +title +##val +united +233 +2021 +##ons +doi +trivago +overdope +sbs +##ance +##ち +grand +special +573032185 +imf +216 +wx17house +##so +##ーム +audi +##he +london +william +##rp +##ake +science +beach +cfa +amp +ps4 +880 +##800 +##link +##hp +crm +ferragamo +bell +make +##eng +195 +under +zh +photos +2300 +##style +##ント +via +176 +da +##gi +company +i7 +##ray +thomas +370 +ufo +i5 +##max +plc +ben +back +research +8g +173 +mike +##pc +##ッフ +september +189 +##ace +vps +february +167 +pantos +wp +lisa +1921 +★★ +jquery +night +long +offer +##berg +##news +1911 +##いて +ray +fks +wto +せます +over +164 +340 +##all +##rus +1924 +##888 +##works +blogtitle +loftpermalink +##→ +187 +martin +test +ling +km +##め +15000 +fda +v3 +##ja +##ロ +ｗedding +かある +outlet +family +##ea +をこ +##top +story +##ness +salvatore +##lu +204 +swift +215 +room +している +oracle +##ul +1925 +sam +b2c +week +pi +rock +##のは +##ａ +##けと +##ean +##300 +##gle +cctv +after +chinese +##back +powered +x2 +##tan +1918 +##nes +##イン +canon +only +181 +##zi +##las +say +##oe +184 +##sd +221 +##bot +##world +##zo +sky +made +top100 +just +1926 +pmi +802 +234 +gap +##vr +177 +les +174 +▲topoct +ball +vogue +vi +ing +ofweek +cos +##list +##ort +▲topmay +##なら +##lon +として +last +##tc +##of +##bus +##gen +real +eva +##コ +a3 +nas +##lie +##ria +##coin +##bt +▲topapr +his +212 +cat +nata +vive +health +⋯⋯ +drive +sir +▲topmar +du +cup +##カー +##ook +##よう +##sy +alex +msg +tour +しました +3ce +##word +193 +ebooks +r8 +block +318 +##より +2200 +nice +pvp +207 +months +1905 +rewards +##ther +1917 +0800 +##xi +##チ +##sc +micro +850 +gg +blogfp +op +1922 +daily +m1 +264 +true +##bb +ml +##tar +##のお +##ky +anthony +196 +253 +##yo +state +218 +##ara +##aa +##rc +##tz +##ston +より +gear +##eo +##ade +ge +see +1923 +##win +##ura +ss +heart +##den +##ita +down +##sm +el +png +2100 +610 +rakuten +whatsapp +bay +dream +add +##use +680 +311 +pad +gucci +mpv +##ode +##fo +island +▲topjun +##▼ +223 +jason +214 +chicago +##❤ +しの +##hone +io +##れる +##ことか +sogo +be2 +##ology +990 +cloud +vcd +##con +2～3 +##ford +##joy +##kb +##こさいます +##rade +but +##ach +docker +##ful +rfid +ul +##ase +hit +ford +##star +580 +##○ +１１ +a2 +sdk +reading +edited +##are +cmos +##mc +238 +siri +light +##ella +##ため +bloomberg +##read +pizza +##ison +jimmy +##vm +college +node +journal +ba +18k +##play +245 +##cer +２０ +magic +##yu +191 +jump +288 +tt +##ings +asr +##lia +3200 +step5 +network +##cd +mc +いします +1234 +pixstyleme +273 +##600 +2800 +money +★★★★★ +1280 +１２ +430 +bl +みの +act +##tus +tokyo +##rial +##life +emba +##ae +saas +tcs +##rk +##wang +summer +##sp +ko +##ving +390 +premium +##その +netflix +##ヒ +uk +mt +##lton +right +frank +two +209 +える +##ple +##cal +021 +##んな +##sen +##ville +hold +nexus +dd +##ius +てお +##mah +##なく +tila +zero +820 +ce +##tin +resort +##ws +charles +old +p10 +5d +report +##360 +##ru +##には +bus +vans +lt +##est +pv +##レ +links +rebecca +##ツ +##dm +azure +##365 +きな +limited +bit +4gb +##mon +1910 +moto +##eam +213 +1913 +var +eos +なとの +226 +blogspot +された +699 +e3 +dos +dm +fc +##ments +##ik +##kw +boy +##bin +##ata +960 +er +##せ +219 +##vin +##tu +##ula +194 +##∥ +station +##ろ +##ature +835 +files +zara +hdr +top10 +nature +950 +magazine +s6 +marriott +##シ +avira +case +##っと +tab +##ran +tony +##home +oculus +im +##ral +jean +saint +cry +307 +rosie +##force +##ini +ice +##bert +のある +##nder +##mber +pet +2600 +##◆ +plurk +▲topdec +##sis +00kg +▲topnov +720 +##ence +tim +##ω +##nc +##ても +##name +log +ips +great +ikea +malaysia +unix +##イト +3600 +##ncy +##nie +12000 +akb48 +##ye +##oid +404 +##chi +##いた +oa +xuehai +##1000 +##orm +##rf +275 +さん +##ware +##リー +980 +ho +##pro +text +##era +560 +bob +227 +##ub +##2008 +8891 +scp +avi +##zen +2022 +mi +wu +museum +qvod +apache +lake +jcb +▲topaug +★★★ +ni +##hr +hill +302 +ne +weibo +490 +ruby +##ーシ +##ヶ +##row +4d +▲topjul +iv +##ish +github +306 +mate +312 +##スト +##lot +##ane +andrew +のハイト +##tina +t1 +rf +ed2k +##vel +##900 +way +final +りの +ns +5a +705 +197 +##メ +sweet +bytes +##ene +▲topjan +231 +##cker +##2007 +##px +100g +topapp +229 +helpapp +rs +low +14k +g4g +care +630 +ldquo +あり +##fork +leave +rm +edition +##gan +##zon +##qq +▲topsep +##google +##ism +gold +224 +explorer +##zer +toyota +category +select +visual +##labels +restaurant +##md +posts +s1 +##ico +もっと +angelababy +123456 +217 +sports +s3 +mbc +1915 +してくたさい +shell +x86 +candy +##new +kbs +face +xl +470 +##here +4a +swissinfo +v8 +▲topfeb +dram +##ual +##vice +3a +##wer +sport +q1 +ios10 +public +int +card +##ｃ +ep +au +rt +##れた +1080 +bill +##mll +kim +３０ +460 +wan +##uk +##ミ +x3 +298 +0t +scott +##ming +239 +e5 +##3d +h7n9 +worldcat +brown +##あります +##vo +##led +##580 +##ax +249 +410 +##ert +paris +##～6 +polo +925 +##lr +599 +##ナ +capital +##hing +bank +cv +1g +##chat +##ｓ +##たい +adc +##ule +2m +##ｅ +digital +hotmail +268 +##pad +870 +bbq +quot +##ring +before +wali +##まて +mcu +2k +2b +という +costco +316 +north +333 +switch +##city +##ｐ +philips +##mann +management +panasonic +##cl +##vd +##ping +##rge +alice +##lk +##ましょう +css3 +##ney +vision +alpha +##ular +##400 +##tter +lz +にお +##ありません +mode +gre +1916 +pci +##tm +237 +1～2 +##yan +##そ +について +##let +##キ +work +war +coach +ah +mary +##ᅵ +huang +##pt +a8 +pt +follow +##berry +1895 +##ew +a5 +ghost +##ション +##wn +##og +south +##code +girls +##rid +action +villa +git +r11 +table +games +##cket +error +##anonymoussaid +##ag +here +##ame +##gc +qa +##■ +##lis +gmp +##gin +vmalife +##cher +yu +wedding +##tis +demo +dragon +530 +soho +social +bye +##rant +river +orz +acer +325 +##↑ +##ース +##ats +261 +del +##ven +440 +ups +##ように +##ター +305 +value +macd +yougou +##dn +661 +##ano +ll +##urt +##rent +continue +script +##wen +##ect +paper +263 +319 +shift +##chel +##フト +##cat +258 +x5 +fox +243 +##さん +car +aaa +##blog +loading +##yn +##tp +kuso +799 +si +sns +イカせるテンマ +ヒンクテンマ3 +rmb +vdc +forest +central +prime +help +ultra +##rmb +##ような +241 +square +688 +##しい +のないフロクに +##field +##reen +##ors +##ju +c1 +start +510 +##air +##map +cdn +##wo +cba +stephen +m8 +100km +##get +opera +##base +##ood +vsa +com™ +##aw +##ail +251 +なのて +count +t2 +##ᅡ +##een +2700 +hop +##gp +vsc +tree +##eg +##ose +816 +285 +##ories +##shop +alphago +v4 +1909 +simon +##ᆼ +fluke62max +zip +スホンサー +##sta +louis +cr +bas +##～10 +bc +##yer +hadoop +##ube +##wi +1906 +0755 +hola +##low +place +centre +5v +d3 +##fer +252 +##750 +##media +281 +540 +0l +exchange +262 +series +##ハー +##san +eb +##bank +##ｋ +q3 +##nge +##mail +take +##lp +259 +1888 +client +east +cache +event +vincent +##ールを +きを +##nse +sui +855 +adchoice +##и +##stry +##なたの +246 +##zone +ga +apps +sea +##ab +248 +cisco +##タ +##rner +kymco +##care +dha +##pu +##yi +minkoff +royal +p1 +への +annie +269 +collection +kpi +playstation +257 +になります +866 +bh +##bar +queen +505 +radio +1904 +andy +armani +##xy +manager +iherb +##ery +##share +spring +raid +johnson +1908 +##ob +volvo +hall +##ball +v6 +our +taylor +##hk +bi +242 +##cp +kate +bo +water +technology +##rie +サイトは +277 +##ona +##sl +hpv +303 +gtx +hip +rdquo +jayz +stone +##lex +##rum +namespace +##やり +620 +##ale +##atic +des +##erson +##ql +##ves +##type +enter +##この +##てきます +d2 +##168 +##mix +##bian +との +a9 +jj +ky +##lc +access +movie +##hc +リストに +tower +##ration +##mit +ます +##nch +ua +tel +prefix +##o2 +1907 +##point +1901 +ott +～10 +##http +##ury +baidu +##ink +member +##logy +bigbang +nownews +##js +##shot +##tb +##こと +247 +eba +##tics +##lus +ける +v5 +spark +##ama +there +##ions +god +##lls +##down +hiv +##ress +burberry +day2 +##kv +◆◆ +jeff +related +film +edit +joseph +283 +##ark +cx +32gb +order +g9 +30000 +##ans +##tty +s5 +##bee +かあります +thread +xr +buy +sh +005 +land +spotify +mx +##ari +276 +##verse +×email +sf +why +##ことて +244 +7headlines +nego +sunny +dom +exo +401 +666 +positioning +fit +rgb +##tton +278 +kiss +alexa +adam +lp +みリストを +##ｇ +mp +##ties +##llow +amy +##du +np +002 +institute +271 +##rth +##lar +2345 +590 +##des +sidebar +１５ +imax +site +##cky +##kit +##ime +##009 +season +323 +##fun +##ンター +##ひ +gogoro +a7 +pu +lily +fire +twd600 +##ッセーシを +いて +##vis +30ml +##cture +##をお +information +##オ +close +friday +##くれる +yi +nick +てすか +##tta +##tel +6500 +##lock +cbd +economy +254 +かお +267 +tinker +double +375 +8gb +voice +##app +oops +channel +today +985 +##right +raw +xyz +##＋ +jim +edm +##cent +7500 +supreme +814 +ds +##its +##asia +dropbox +##てすか +##tti +books +272 +100ml +##tle +##ller +##ken +##more +##boy +sex +309 +##dom +t3 +##ider +##なります +##unch +1903 +810 +feel +5500 +##かった +##put +により +s2 +mo +##gh +men +ka +amoled +div +##tr +##n1 +port +howard +##tags +ken +dnf +##nus +adsense +##а +ide +##へ +buff +thunder +##town +##ique +has +##body +auto +pin +##erry +tee +てした +295 +number +##the +##013 +object +psp +cool +udnbkk +16gb +##mic +miui +##tro +most +r2 +##alk +##nity +1880 +±0 +##いました +428 +s4 +law +version +##oa +n1 +sgs +docomo +##tf +##ack +henry +fc2 +##ded +##sco +##014 +##rite +286 +0mm +linkedin +##ada +##now +wii +##ndy +ucbug +##◎ +sputniknews +legalminer +##ika +##xp +2gb +##bu +q10 +oo +b6 +come +##rman +cheese +ming +maker +##gm +nikon +##fig +ppi +kelly +##ります +jchere +てきます +ted +md +003 +fgo +tech +##tto +dan +soc +##gl +##len +hair +earth +640 +521 +img +##pper +##a1 +##てきる +##ロク +acca +##ition +##ference +suite +##ig +outlook +##mond +##cation +398 +##pr +279 +101vip +358 +##999 +282 +64gb +3800 +345 +airport +##over +284 +##おり +jones +##ith +lab +##su +##いるのて +co2 +town +piece +##llo +no1 +vmware +24h +##qi +focus +reader +##admin +##ora +tb +false +##log +1898 +know +lan +838 +##ces +f4 +##ume +motel +stop +##oper +na +flickr +netcomponents +##af +##─ +pose +williams +local +##ound +##cg +##site +##iko +いお +274 +5m +gsm +con +##ath +1902 +friends +##hip +cell +317 +##rey +780 +cream +##cks +012 +##dp +facebooktwitterpinterestgoogle +sso +324 +shtml +song +swiss +##mw +##キンク +lumia +xdd +string +tiffany +522 +marc +られた +insee +russell +sc +dell +##ations +ｏｋ +camera +289 +##vs +##flow +##late +classic +287 +##nter +stay +g1 +mtv +512 +##ever +##lab +##nger +qe +sata +ryan +d1 +50ml +cms +##cing +su +292 +3300 +editor +296 +##nap +security +sunday +association +##ens +##700 +##bra +acg +##かり +sofascore +とは +mkv +##ign +jonathan +gary +build +labels +##oto +tesla +moba +qi +gohappy +general +ajax +1024 +##かる +サイト +society +##test +##urs +wps +fedora +##ich +mozilla +328 +##480 +##dr +usa +urn +##lina +##ｒ +grace +##die +##try +##ader +1250 +##なり +elle +570 +##chen +##ᆯ +price +##ten +uhz +##ough +eq +##hen +states +push +session +balance +wow +506 +##cus +##py +when +##ward +##ep +34e +wong +library +prada +##サイト +##cle +running +##ree +313 +ck +date +q4 +##ctive +##ool +##＞ +mk +##ira +##163 +388 +die +secret +rq +dota +buffet +は１ヶ +e6 +##ez +pan +368 +ha +##card +##cha +2a +##さ +alan +day3 +eye +f3 +##end +france +keep +adi +rna +tvbs +##ala +solo +nova +##え +##tail +##ょう +support +##ries +##なる +##ved +base +copy +iis +fps +##ways +hero +hgih +profile +fish +mu +ssh +entertainment +chang +##wd +click +cake +##ond +pre +##tom +kic +pixel +##ov +##fl +product +6a +##pd +dear +##gate +es +yumi +audio +##² +##sky +echo +bin +where +##ture +329 +##ape +find +sap +isis +##なと +nand +##101 +##load +##ream +band +a6 +525 +never +##post +festival +50cm +##we +555 +guide +314 +zenfone +##ike +335 +gd +forum +jessica +strong +alexander +##ould +software +allen +##ious +program +360° +else +lohasthree +##gar +することかてきます +please +##れます +rc +##ggle +##ric +bim +50000 +##own +eclipse +355 +brian +3ds +##side +061 +361 +##other +##ける +##tech +##ator +485 +engine +##ged +##ｔ +plaza +##fit +cia +ngo +westbrook +shi +tbs +50mm +##みませんか +sci +291 +reuters +##ily +contextlink +##hn +af +##cil +bridge +very +##cel +1890 +cambridge +##ize +15g +##aid +##data +790 +frm +##head +award +butler +##sun +meta +##mar +america +ps3 +puma +pmid +##すか +lc +670 +kitchen +##lic +オーフン5 +きなしソフトサーヒス +そして +day1 +future +★★★★ +##text +##page +##rris +pm1 +##ket +fans +##っています +1001 +christian +bot +kids +trackback +##hai +c3 +display +##hl +n2 +1896 +idea +さんも +##sent +airmail +##ug +##men +pwm +けます +028 +##lution +369 +852 +awards +schemas +354 +asics +wikipedia +font +##tional +##vy +c2 +293 +##れている +##dget +##ein +っている +contact +pepper +スキル +339 +##～5 +294 +##uel +##ument +730 +##hang +みてす +q5 +##sue +rain +##ndi +wei +swatch +##cept +わせ +331 +popular +##ste +##tag +p2 +501 +trc +1899 +##west +##live +justin +honda +ping +messenger +##rap +v9 +543 +##とは +unity +appqq +はすへて +025 +leo +##tone +##テ +##ass +uniqlo +##010 +502 +her +jane +memory +moneydj +##tical +human +12306 +していると +##m2 +coc +miacare +##mn +tmt +##core +vim +kk +##may +fan +target +use +too +338 +435 +2050 +867 +737 +fast +##2c +services +##ope +omega +energy +##わ +pinkoi +1a +##なから +##rain +jackson +##ement +##シャンルの +374 +366 +そんな +p9 +rd +##ᆨ +1111 +##tier +##vic +zone +##│ +385 +690 +dl +isofix +cpa +m4 +322 +kimi +めて +davis +##lay +lulu +##uck +050 +weeks +qs +##hop +920 +##ｎ +ae +##ear +～5 +eia +405 +##fly +korea +jpeg +boost +##ship +small +##リア +1860 +eur +297 +425 +valley +##iel +simple +##ude +rn +k2 +##ena +されます +non +patrick +しているから +##ナー +feed +5757 +30g +process +well +qqmei +##thing +they +aws +lu +pink +##ters +##kin +または +board +##vertisement +wine +##ien +unicode +##dge +r1 +359 +##tant +いを +##twitter +##3c +cool1 +される +##れて +##ｌ +isp +##012 +standard +45㎡2 +402 +##150 +matt +##fu +326 +##iner +googlemsn +pixnetfacebookyahoo +##ラン +x7 +886 +##uce +メーカー +sao +##ev +##きました +##file +9678 +403 +xddd +shirt +6l +##rio +##hat +3mm +givenchy +ya +bang +##lio +monday +crystal +ロクイン +##abc +336 +head +890 +ubuntuforumwikilinuxpastechat +##vc +##～20 +##rity +cnc +7866 +ipv6 +null +1897 +##ost +yang +imsean +tiger +##fet +##ンス +352 +##＝ +dji +327 +ji +maria +##come +##んて +foundation +3100 +##beth +##なった +1m +601 +active +##aft +##don +3p +sr +349 +emma +##khz +living +415 +353 +1889 +341 +709 +457 +sas +x6 +##face +pptv +x4 +##mate +han +sophie +##jing +337 +fifa +##mand +other +sale +inwedding +##gn +てきちゃいます +##mmy +##pmlast +bad +nana +nbc +してみてくたさいね +なとはお +##wu +##かあります +##あ +note7 +single +##340 +せからこ +してくたさい♪この +しにはとんとんワークケートを +するとあなたにもっとマッチした +ならワークケートへ +もみつかっちゃうかも +ワークケートの +##bel +window +##dio +##ht +union +age +382 +１４ +##ivity +##ｙ +コメント +domain +neo +##isa +##lter +5k +f5 +steven +##cts +powerpoint +tft +self +g2 +ft +##テル +zol +##act +mwc +381 +343 +もう +nbapop +408 +てある +eds +ace +##room +previous +author +tomtom +il +##ets +hu +financial +☆☆☆ +っています +bp +5t +chi +1gb +##hg +fairmont +cross +008 +gay +h2 +function +##けて +356 +also +1b +625 +##ータ +##raph +1894 +3～5 +##ils +i3 +334 +avenue +##host +による +##bon +##tsu +message +navigation +50g +fintech +h6 +##ことを +8cm +##ject +##vas +##firm +credit +##wf +xxxx +form +##nor +##space +huawei +plan +json +sbl +##dc +machine +921 +392 +wish +##120 +##sol +windows7 +edward +##ために +development +washington +##nsis +lo +818 +##sio +##ym +##bor +planet +##～8 +##wt +ieee +gpa +##めて +camp +ann +gm +##tw +##oka +connect +##rss +##work +##atus +wall +chicken +soul +2mm +##times +fa +##ather +##cord +009 +##eep +hitachi +gui +harry +##pan +e1 +disney +##press +##ーション +wind +386 +frigidaire +##tl +liu +hsu +332 +basic +von +ev +いた +てきる +スホンサーサイト +learning +##ull +expedia +archives +change +##wei +santa +cut +ins +6gb +turbo +brand +cf1 +508 +004 +return +747 +##rip +h1 +##nis +##をこ +128gb +##にお +3t +application +しており +emc +rx +##oon +384 +quick +412 +15058 +wilson +wing +chapter +##bug +beyond +##cms +##dar +##oh +zoom +e2 +trip +sb +##nba +rcep +342 +aspx +ci +080 +gc +gnu +める +##count +advanced +dance +dv +##url +##ging +367 +8591 +am09 +shadow +battle +346 +##ｉ +##cia +##という +emily +##のてす +##tation +host +ff +techorz +sars +##mini +##mporary +##ering +nc +4200 +798 +##next +cma +##mbps +##gas +##ift +##dot +##ィ +455 +##～17 +amana +##りの +426 +##ros +ir +00㎡1 +##eet +##ible +##↓ +710 +ˋ▽ˊ +##aka +dcs +iq +##ｖ +l1 +##lor +maggie +##011 +##iu +588 +##～1 +830 +##gt +1tb +articles +create +##burg +##iki +database +fantasy +##rex +##cam +dlc +dean +##you +hard +path +gaming +victoria +maps +cb +##lee +##itor +overchicstoretvhome +systems +##xt +416 +p3 +sarah +760 +##nan +407 +486 +x9 +install +second +626 +##ann +##ph +##rcle +##nic +860 +##nar +ec +##とう +768 +metro +chocolate +##rian +～4 +##table +##しています +skin +##sn +395 +mountain +##0mm +inparadise +6m +7x24 +ib +4800 +##jia +eeworld +creative +g5 +g3 +357 +parker +ecfa +village +からの +18000 +sylvia +サーヒス +hbl +##ques +##onsored +##x2 +##きます +##v4 +##tein +ie6 +383 +##stack +389 +ver +##ads +##baby +sound +bbe +##110 +##lone +##uid +ads +022 +gundam +351 +thinkpad +006 +scrum +match +##ave +mems +##470 +##oy +##なりました +##talk +glass +lamigo +span +##eme +job +##a5 +jay +wade +kde +498 +##lace +ocean +tvg +##covery +##r3 +##ners +##rea +junior +think +##aine +cover +##ision +##sia +↓↓ +##bow +msi +413 +458 +406 +##love +711 +801 +soft +z2 +##pl +456 +1840 +mobil +mind +##uy +427 +nginx +##oi +めた +##rr +6221 +##mple +##sson +##ーシてす +371 +##nts +91tv +comhd +crv3000 +##uard +1868 +397 +deep +lost +field +gallery +##bia +rate +spf +redis +traction +930 +icloud +011 +なら +fe +jose +372 +##tory +into +sohu +fx +899 +379 +kicstart2 +##hia +すく +##～3 +##sit +ra +２４ +##walk +##xure +500g +##pact +pacific +xa +natural +carlo +##250 +##walker +1850 +##can +cto +gigi +516 +##サー +pen +##hoo +ob +matlab +##ｂ +##yy +13913459 +##iti +mango +##bbs +sense +c5 +oxford +##ニア +walker +jennifer +##ola +course +##bre +701 +##pus +##rder +lucky +075 +##ぁ +ivy +なお +##nia +sotheby +side +##ugh +joy +##orage +##ush +##bat +##dt +364 +r9 +##2d +##gio +511 +country +wear +##lax +##～7 +##moon +393 +seven +study +411 +348 +lonzo +8k +##ェ +evolution +##イフ +##kk +gs +kd +##レス +arduino +344 +b12 +##lux +arpg +##rdon +cook +##x5 +dark +five +##als +##ida +とても +sign +362 +##ちの +something +20mm +##nda +387 +##posted +fresh +tf +1870 +422 +cam +##mine +##skip +##form +##ssion +education +394 +##tee +dyson +stage +##jie +want +##night +epson +pack +あります +##ppy +テリヘル +##█ +wd +##eh +##rence +left +##lvin +golden +mhz +discovery +##trix +##n2 +loft +##uch +##dra +##sse +speed +～1 +1mdb +sorry +welcome +##urn +wave +gaga +##lmer +teddy +##160 +トラックハック +せよ +611 +##f2016 +378 +rp +##sha +rar +##あなたに +##きた +840 +holiday +##ュー +373 +074 +##vg +##nos +##rail +gartner +gi +6p +##dium +kit +488 +b3 +eco +##ろう +20g +sean +##stone +autocad +nu +##np +f16 +write +029 +m5 +##ias +images +atp +##dk +fsm +504 +1350 +ve +52kb +##xxx +##のに +##cake +414 +unit +lim +ru +1v +##ification +published +angela +16g +analytics +ak +##ｑ +##nel +gmt +##icon +again +##₂ +##bby +ios11 +445 +かこさいます +waze +いてす +##ハ +9985 +##ust +##ティー +framework +##007 +iptv +delete +52sykb +cl +wwdc +027 +30cm +##fw +##ての +1389 +##xon +brandt +##ses +##dragon +tc +vetements +anne +monte +modern +official +##へて +##ere +##nne +##oud +もちろん +５０ +etnews +##a2 +##graphy +421 +863 +##ちゃん +444 +##rtex +##てお +l2 +##gma +mount +ccd +たと +archive +morning +tan +ddos +e7 +##ホ +day4 +##ウ +gis +453 +its +495 +factory +bruce +pg +##ito +ってくたさい +guest +cdma +##lling +536 +n3 +しかし +3～4 +mega +eyes +ro +１３ +women +dac +church +##jun +singapore +##facebook +6991 +starbucks +##tos +##stin +##shine +zen +##mu +tina +20℃ +1893 +##たけて +503 +465 +request +##gence +qt +##っ +1886 +347 +363 +q7 +##zzi +diary +##tore +409 +##ead +468 +cst +##osa +canada +agent +va +##jiang +##ちは +##ーク +##lam +sg +##nix +##sday +##よって +g6 +##master +bing +##zl +charlie +１６ +8mm +nb40 +##ーン +thai +##ルフ +ln284ct +##itz +##2f +bonnie +##food +##lent +originals +##stro +##lts +418 +∟∣ +##bscribe +children +ntd +yesstyle +##かも +hmv +##tment +d5 +2cm +arts +sms +##pn +##я +##いい +topios9 +539 +lifestyle +virtual +##ague +xz +##deo +muji +024 +unt +##nnis +##ᅩ +faq1 +1884 +396 +##ette +fly +64㎡ +はしめまして +441 +curry +##pop +のこ +release +##← +##◆◆ +##cast +073 +ありな +500ml +##ews +5c +##stle +ios7 +##ima +787 +dog +lenovo +##r4 +roger +013 +cbs +vornado +100m +417 +##desk +##クok +##ald +1867 +9595 +2900 +##van +oil +##ｘ +some +break +common +##jy +##lines +g7 +twice +419 +ella +nano +belle +にこ +##mes +##self +##note +jb +##ことかてきます +benz +##との +##ova +451 +save +##wing +##ますのて +kai +りは +##hua +##rect +rainer +##unge +448 +##0m +adsl +##かな +guestname +##uma +##kins +##zu +tokichoi +##price +county +##med +##mus +rmk +391 +address +vm +えて +openload +##group +##hin +##iginal +amg +urban +##oz +jobs +emi +##public +beautiful +##sch +album +##dden +##bell +jerry +works +hostel +miller +##drive +##rmin +##１０ +376 +boot +828 +##370 +##fx +##cm～ +1885 +##nome +##ctionary +##oman +##lish +##cr +##hm +433 +##how +432 +francis +xi +c919 +b5 +evernote +##uc +vga +##3000 +coupe +##urg +##cca +##uality +019 +6g +れる +multi +##また +##ett +em +hey +##ani +##tax +##rma +inside +than +740 +leonnhurt +##jin +ict +れた +bird +notes +200mm +くの +##dical +##lli +result +442 +iu +ee +438 +smap +gopro +##last +yin +pure +998 +32g +けた +5kg +##dan +##rame +mama +##oot +bean +marketing +##hur +2l +bella +sync +xuite +##ground +515 +discuz +##getrelax +##ince +##bay +##5s +cj +##イス +gmat +apt +##pass +jing +##rix +c4 +rich +##とても +niusnews +##ello +bag +770 +##eting +##mobile +１８ +culture +015 +##のてすか +377 +1020 +area +##ience +616 +details +gp +universal +silver +dit +はお +private +ddd +u11 +kanshu +##ified +fung +##nny +dx +##520 +tai +475 +023 +##fr +##lean +3s +##pin +429 +##rin +25000 +ly +rick +##bility +usb3 +banner +##baru +##gion +metal +dt +vdf +1871 +karl +qualcomm +bear +1010 +oldid +ian +jo +##tors +population +##ernel +1882 +mmorpg +##mv +##bike +603 +##© +ww +friend +##ager +exhibition +##del +##pods +fpx +structure +##free +##tings +kl +##rley +##copyright +##mma +california +3400 +orange +yoga +4l +canmake +honey +##anda +##コメント +595 +nikkie +##ルハイト +dhl +publishing +##mall +##gnet +20cm +513 +##クセス +##┅ +e88 +970 +##dog +fishbase +##! +##" +### +##$ +##% +##& +##' +##( +##) +##* +##+ +##, +##- +##. +##/ +##: +##; +##< +##= +##> +##? +##@ +##[ +##\ +##] +##^ +##_ +##{ +##| +##} +##~ +##£ +##¤ +##¥ +##§ +##« +##± +##³ +##µ +##· +##¹ +##º +##» +##¼ +##ß +##æ +##÷ +##ø +##đ +##ŋ +##ɔ +##ə +##ɡ +##ʰ +##ˇ +##ˈ +##ˊ +##ˋ +##ˍ +##ː +##˙ +##˚ +##ˢ +##α +##β +##γ +##δ +##ε +##η +##θ +##ι +##κ +##λ +##μ +##ν +##ο +##π +##ρ +##ς +##σ +##τ +##υ +##φ +##χ +##ψ +##б +##в +##г +##д +##е +##ж +##з +##к +##л +##м +##н +##о +##п +##р +##с +##т +##у +##ф +##х +##ц +##ч +##ш +##ы +##ь +##і +##ا +##ب +##ة +##ت +##د +##ر +##س +##ع +##ل +##م +##ن +##ه +##و +##ي +##۩ +##ก +##ง +##น +##ม +##ย +##ร +##อ +##า +##เ +##๑ +##་ +##ღ +##ᄀ +##ᄁ +##ᄂ +##ᄃ +##ᄅ +##ᄆ +##ᄇ +##ᄈ +##ᄉ +##ᄋ +##ᄌ +##ᄎ +##ᄏ +##ᄐ +##ᄑ +##ᄒ +##ᅢ +##ᅣ +##ᅥ +##ᅦ +##ᅧ +##ᅨ +##ᅪ +##ᅬ +##ᅭ +##ᅮ +##ᅯ +##ᅲ +##ᅳ +##ᅴ +##ᆷ +##ᆸ +##ᆺ +##ᆻ +##ᗜ +##ᵃ +##ᵉ +##ᵍ +##ᵏ +##ᵐ +##ᵒ +##ᵘ +##‖ +##„ +##† +##• +##‥ +##‧ +##
 +##‰ +##′ +##″ +##‹ +##› +##※ +##‿ +##⁄ +##ⁱ +##⁺ +##ⁿ +##₁ +##₃ +##₄ +##€ +##№ +##ⅰ +##ⅱ +##ⅲ +##ⅳ +##ⅴ +##↔ +##↗ +##↘ +##⇒ +##∀ +##− +##∕ +##∙ +##√ +##∞ +##∟ +##∠ +##∣ +##∩ +##∮ +##∶ +##∼ +##∽ +##≈ +##≒ +##≡ +##≤ +##≥ +##≦ +##≧ +##≪ +##≫ +##⊙ +##⋅ +##⋈ +##⋯ +##⌒ +##① +##② +##③ +##④ +##⑤ +##⑥ +##⑦ +##⑧ +##⑨ +##⑩ +##⑴ +##⑵ +##⑶ +##⑷ +##⑸ +##⒈ +##⒉ +##⒊ +##⒋ +##ⓒ +##ⓔ +##ⓘ +##━ +##┃ +##┆ +##┊ +##┌ +##└ +##├ +##┣ +##═ +##║ +##╚ +##╞ +##╠ +##╭ +##╮ +##╯ +##╰ +##╱ +##╳ +##▂ +##▃ +##▅ +##▇ +##▉ +##▋ +##▌ +##▍ +##▎ +##□ +##▪ +##▫ +##▬ +##△ +##▶ +##► +##▽ +##◇ +##◕ +##◠ +##◢ +##◤ +##☀ +##☕ +##☞ +##☺ +##☼ +##♀ +##♂ +##♠ +##♡ +##♣ +##♦ +##♫ +##♬ +##✈ +##✔ +##✕ +##✖ +##✦ +##✨ +##✪ +##✰ +##✿ +##❀ +##➜ +##➤ +##⦿ +##、 +##。 +##〃 +##々 +##〇 +##〈 +##〉 +##《 +##》 +##「 +##」 +##『 +##』 +##【 +##】 +##〓 +##〔 +##〕 +##〖 +##〗 +##〜 +##〝 +##〞 +##ぃ +##ぇ +##ぬ +##ふ +##ほ +##む +##ゃ +##ゅ +##ゆ +##ょ +##゜ +##ゝ +##ァ +##ゥ +##エ +##ォ +##ケ +##サ +##セ +##ソ +##ッ +##ニ +##ヌ +##ネ +##ノ +##ヘ +##モ +##ャ +##ヤ +##ュ +##ユ +##ョ +##ヨ +##ワ +##ヲ +##・ +##ヽ +##ㄅ +##ㄆ +##ㄇ +##ㄉ +##ㄋ +##ㄌ +##ㄍ +##ㄎ +##ㄏ +##ㄒ +##ㄚ +##ㄛ +##ㄞ +##ㄟ +##ㄢ +##ㄤ +##ㄥ +##ㄧ +##ㄨ +##ㆍ +##㈦ +##㊣ +##㗎 +##一 +##丁 +##七 +##万 +##丈 +##三 +##上 +##下 +##不 +##与 +##丐 +##丑 +##专 +##且 +##丕 +##世 +##丘 +##丙 +##业 +##丛 +##东 +##丝 +##丞 +##丟 +##両 +##丢 +##两 +##严 +##並 +##丧 +##丨 +##个 +##丫 +##中 +##丰 +##串 +##临 +##丶 +##丸 +##丹 +##为 +##主 +##丼 +##丽 +##举 +##丿 +##乂 +##乃 +##久 +##么 +##义 +##之 +##乌 +##乍 +##乎 +##乏 +##乐 +##乒 +##乓 +##乔 +##乖 +##乗 +##乘 +##乙 +##乜 +##九 +##乞 +##也 +##习 +##乡 +##书 +##乩 +##买 +##乱 +##乳 +##乾 +##亀 +##亂 +##了 +##予 +##争 +##事 +##二 +##于 +##亏 +##云 +##互 +##五 +##井 +##亘 +##亙 +##亚 +##些 +##亜 +##亞 +##亟 +##亡 +##亢 +##交 +##亥 +##亦 +##产 +##亨 +##亩 +##享 +##京 +##亭 +##亮 +##亲 +##亳 +##亵 +##人 +##亿 +##什 +##仁 +##仃 +##仄 +##仅 +##仆 +##仇 +##今 +##介 +##仍 +##从 +##仏 +##仑 +##仓 +##仔 +##仕 +##他 +##仗 +##付 +##仙 +##仝 +##仞 +##仟 +##代 +##令 +##以 +##仨 +##仪 +##们 +##仮 +##仰 +##仲 +##件 +##价 +##任 +##份 +##仿 +##企 +##伉 +##伊 +##伍 +##伎 +##伏 +##伐 +##休 +##伕 +##众 +##优 +##伙 +##会 +##伝 +##伞 +##伟 +##传 +##伢 +##伤 +##伦 +##伪 +##伫 +##伯 +##估 +##伴 +##伶 +##伸 +##伺 +##似 +##伽 +##佃 +##但 +##佇 +##佈 +##位 +##低 +##住 +##佐 +##佑 +##体 +##佔 +##何 +##佗 +##佘 +##余 +##佚 +##佛 +##作 +##佝 +##佞 +##佟 +##你 +##佢 +##佣 +##佤 +##佥 +##佩 +##佬 +##佯 +##佰 +##佳 +##併 +##佶 +##佻 +##佼 +##使 +##侃 +##侄 +##來 +##侈 +##例 +##侍 +##侏 +##侑 +##侖 +##侗 +##供 +##依 +##侠 +##価 +##侣 +##侥 +##侦 +##侧 +##侨 +##侬 +##侮 +##侯 +##侵 +##侶 +##侷 +##便 +##係 +##促 +##俄 +##俊 +##俎 +##俏 +##俐 +##俑 +##俗 +##俘 +##俚 +##保 +##俞 +##俟 +##俠 +##信 +##俨 +##俩 +##俪 +##俬 +##俭 +##修 +##俯 +##俱 +##俳 +##俸 +##俺 +##俾 +##倆 +##倉 +##個 +##倌 +##倍 +##倏 +##們 +##倒 +##倔 +##倖 +##倘 +##候 +##倚 +##倜 +##借 +##倡 +##値 +##倦 +##倩 +##倪 +##倫 +##倬 +##倭 +##倶 +##债 +##值 +##倾 +##偃 +##假 +##偈 +##偉 +##偌 +##偎 +##偏 +##偕 +##做 +##停 +##健 +##側 +##偵 +##偶 +##偷 +##偻 +##偽 +##偿 +##傀 +##傅 +##傍 +##傑 +##傘 +##備 +##傚 +##傢 +##傣 +##傥 +##储 +##傩 +##催 +##傭 +##傲 +##傳 +##債 +##傷 +##傻 +##傾 +##僅 +##働 +##像 +##僑 +##僕 +##僖 +##僚 +##僥 +##僧 +##僭 +##僮 +##僱 +##僵 +##價 +##僻 +##儀 +##儂 +##億 +##儆 +##儉 +##儋 +##儒 +##儕 +##儘 +##償 +##儡 +##優 +##儲 +##儷 +##儼 +##儿 +##兀 +##允 +##元 +##兄 +##充 +##兆 +##兇 +##先 +##光 +##克 +##兌 +##免 +##児 +##兑 +##兒 +##兔 +##兖 +##党 +##兜 +##兢 +##入 +##內 +##全 +##兩 +##八 +##公 +##六 +##兮 +##兰 +##共 +##兲 +##关 +##兴 +##兵 +##其 +##具 +##典 +##兹 +##养 +##兼 +##兽 +##冀 +##内 +##円 +##冇 +##冈 +##冉 +##冊 +##册 +##再 +##冏 +##冒 +##冕 +##冗 +##写 +##军 +##农 +##冠 +##冢 +##冤 +##冥 +##冨 +##冪 +##冬 +##冯 +##冰 +##冲 +##决 +##况 +##冶 +##冷 +##冻 +##冼 +##冽 +##冾 +##净 +##凄 +##准 +##凇 +##凈 +##凉 +##凋 +##凌 +##凍 +##减 +##凑 +##凛 +##凜 +##凝 +##几 +##凡 +##凤 +##処 +##凪 +##凭 +##凯 +##凰 +##凱 +##凳 +##凶 +##凸 +##凹 +##出 +##击 +##函 +##凿 +##刀 +##刁 +##刃 +##分 +##切 +##刈 +##刊 +##刍 +##刎 +##刑 +##划 +##列 +##刘 +##则 +##刚 +##创 +##初 +##删 +##判 +##別 +##刨 +##利 +##刪 +##别 +##刮 +##到 +##制 +##刷 +##券 +##刹 +##刺 +##刻 +##刽 +##剁 +##剂 +##剃 +##則 +##剉 +##削 +##剋 +##剌 +##前 +##剎 +##剐 +##剑 +##剔 +##剖 +##剛 +##剜 +##剝 +##剣 +##剤 +##剥 +##剧 +##剩 +##剪 +##副 +##割 +##創 +##剷 +##剽 +##剿 +##劃 +##劇 +##劈 +##劉 +##劊 +##劍 +##劏 +##劑 +##力 +##劝 +##办 +##功 +##加 +##务 +##劣 +##动 +##助 +##努 +##劫 +##劭 +##励 +##劲 +##劳 +##労 +##劵 +##効 +##劾 +##势 +##勁 +##勃 +##勇 +##勉 +##勋 +##勐 +##勒 +##動 +##勖 +##勘 +##務 +##勛 +##勝 +##勞 +##募 +##勢 +##勤 +##勧 +##勳 +##勵 +##勸 +##勺 +##勻 +##勾 +##勿 +##匀 +##包 +##匆 +##匈 +##匍 +##匐 +##匕 +##化 +##北 +##匙 +##匝 +##匠 +##匡 +##匣 +##匪 +##匮 +##匯 +##匱 +##匹 +##区 +##医 +##匾 +##匿 +##區 +##十 +##千 +##卅 +##升 +##午 +##卉 +##半 +##卍 +##华 +##协 +##卑 +##卒 +##卓 +##協 +##单 +##卖 +##南 +##単 +##博 +##卜 +##卞 +##卟 +##占 +##卡 +##卢 +##卤 +##卦 +##卧 +##卫 +##卮 +##卯 +##印 +##危 +##即 +##却 +##卵 +##卷 +##卸 +##卻 +##卿 +##厂 +##厄 +##厅 +##历 +##厉 +##压 +##厌 +##厕 +##厘 +##厚 +##厝 +##原 +##厢 +##厥 +##厦 +##厨 +##厩 +##厭 +##厮 +##厲 +##厳 +##去 +##县 +##叁 +##参 +##參 +##又 +##叉 +##及 +##友 +##双 +##反 +##収 +##发 +##叔 +##取 +##受 +##变 +##叙 +##叛 +##叟 +##叠 +##叡 +##叢 +##口 +##古 +##句 +##另 +##叨 +##叩 +##只 +##叫 +##召 +##叭 +##叮 +##可 +##台 +##叱 +##史 +##右 +##叵 +##叶 +##号 +##司 +##叹 +##叻 +##叼 +##叽 +##吁 +##吃 +##各 +##吆 +##合 +##吉 +##吊 +##吋 +##同 +##名 +##后 +##吏 +##吐 +##向 +##吒 +##吓 +##吕 +##吖 +##吗 +##君 +##吝 +##吞 +##吟 +##吠 +##吡 +##否 +##吧 +##吨 +##吩 +##含 +##听 +##吭 +##吮 +##启 +##吱 +##吳 +##吴 +##吵 +##吶 +##吸 +##吹 +##吻 +##吼 +##吽 +##吾 +##呀 +##呂 +##呃 +##呆 +##呈 +##告 +##呋 +##呎 +##呐 +##呓 +##呕 +##呗 +##员 +##呛 +##呜 +##呢 +##呤 +##呦 +##周 +##呱 +##呲 +##味 +##呵 +##呷 +##呸 +##呻 +##呼 +##命 +##咀 +##咁 +##咂 +##咄 +##咆 +##咋 +##和 +##咎 +##咏 +##咐 +##咒 +##咔 +##咕 +##咖 +##咗 +##咘 +##咙 +##咚 +##咛 +##咣 +##咤 +##咦 +##咧 +##咨 +##咩 +##咪 +##咫 +##咬 +##咭 +##咯 +##咱 +##咲 +##咳 +##咸 +##咻 +##咽 +##咿 +##哀 +##品 +##哂 +##哄 +##哆 +##哇 +##哈 +##哉 +##哋 +##哌 +##响 +##哎 +##哏 +##哐 +##哑 +##哒 +##哔 +##哗 +##哟 +##員 +##哥 +##哦 +##哧 +##哨 +##哩 +##哪 +##哭 +##哮 +##哲 +##哺 +##哼 +##哽 +##唁 +##唄 +##唆 +##唇 +##唉 +##唏 +##唐 +##唑 +##唔 +##唠 +##唤 +##唧 +##唬 +##售 +##唯 +##唰 +##唱 +##唳 +##唷 +##唸 +##唾 +##啃 +##啄 +##商 +##啉 +##啊 +##問 +##啓 +##啕 +##啖 +##啜 +##啞 +##啟 +##啡 +##啤 +##啥 +##啦 +##啧 +##啪 +##啫 +##啬 +##啮 +##啰 +##啱 +##啲 +##啵 +##啶 +##啷 +##啸 +##啻 +##啼 +##啾 +##喀 +##喂 +##喃 +##善 +##喆 +##喇 +##喉 +##喊 +##喋 +##喎 +##喏 +##喔 +##喘 +##喙 +##喚 +##喜 +##喝 +##喟 +##喧 +##喪 +##喫 +##喬 +##單 +##喰 +##喱 +##喲 +##喳 +##喵 +##営 +##喷 +##喹 +##喺 +##喻 +##喽 +##嗅 +##嗆 +##嗇 +##嗎 +##嗑 +##嗒 +##嗓 +##嗔 +##嗖 +##嗚 +##嗜 +##嗝 +##嗟 +##嗡 +##嗣 +##嗤 +##嗦 +##嗨 +##嗪 +##嗬 +##嗯 +##嗰 +##嗲 +##嗳 +##嗶 +##嗷 +##嗽 +##嘀 +##嘅 +##嘆 +##嘈 +##嘉 +##嘌 +##嘍 +##嘎 +##嘔 +##嘖 +##嘗 +##嘘 +##嘚 +##嘛 +##嘜 +##嘞 +##嘟 +##嘢 +##嘣 +##嘤 +##嘧 +##嘩 +##嘭 +##嘮 +##嘯 +##嘰 +##嘱 +##嘲 +##嘴 +##嘶 +##嘸 +##嘹 +##嘻 +##嘿 +##噁 +##噌 +##噎 +##噓 +##噔 +##噗 +##噙 +##噜 +##噠 +##噢 +##噤 +##器 +##噩 +##噪 +##噬 +##噱 +##噴 +##噶 +##噸 +##噹 +##噻 +##噼 +##嚀 +##嚇 +##嚎 +##嚏 +##嚐 +##嚓 +##嚕 +##嚟 +##嚣 +##嚥 +##嚨 +##嚮 +##嚴 +##嚷 +##嚼 +##囂 +##囉 +##囊 +##囍 +##囑 +##囔 +##囗 +##囚 +##四 +##囝 +##回 +##囟 +##因 +##囡 +##团 +##団 +##囤 +##囧 +##囪 +##囫 +##园 +##困 +##囱 +##囲 +##図 +##围 +##囹 +##固 +##国 +##图 +##囿 +##圃 +##圄 +##圆 +##圈 +##國 +##圍 +##圏 +##園 +##圓 +##圖 +##團 +##圜 +##土 +##圣 +##圧 +##在 +##圩 +##圭 +##地 +##圳 +##场 +##圻 +##圾 +##址 +##坂 +##均 +##坊 +##坍 +##坎 +##坏 +##坐 +##坑 +##块 +##坚 +##坛 +##坝 +##坞 +##坟 +##坠 +##坡 +##坤 +##坦 +##坨 +##坪 +##坯 +##坳 +##坵 +##坷 +##垂 +##垃 +##垄 +##型 +##垒 +##垚 +##垛 +##垠 +##垢 +##垣 +##垦 +##垩 +##垫 +##垭 +##垮 +##垵 +##埂 +##埃 +##埋 +##城 +##埔 +##埕 +##埗 +##域 +##埠 +##埤 +##埵 +##執 +##埸 +##培 +##基 +##埼 +##堀 +##堂 +##堃 +##堅 +##堆 +##堇 +##堑 +##堕 +##堙 +##堡 +##堤 +##堪 +##堯 +##堰 +##報 +##場 +##堵 +##堺 +##堿 +##塊 +##塌 +##塑 +##塔 +##塗 +##塘 +##塚 +##塞 +##塢 +##塩 +##填 +##塬 +##塭 +##塵 +##塾 +##墀 +##境 +##墅 +##墉 +##墊 +##墒 +##墓 +##増 +##墘 +##墙 +##墜 +##增 +##墟 +##墨 +##墩 +##墮 +##墳 +##墻 +##墾 +##壁 +##壅 +##壆 +##壇 +##壊 +##壑 +##壓 +##壕 +##壘 +##壞 +##壟 +##壢 +##壤 +##壩 +##士 +##壬 +##壮 +##壯 +##声 +##売 +##壳 +##壶 +##壹 +##壺 +##壽 +##处 +##备 +##変 +##复 +##夏 +##夔 +##夕 +##外 +##夙 +##多 +##夜 +##够 +##夠 +##夢 +##夥 +##大 +##天 +##太 +##夫 +##夭 +##央 +##夯 +##失 +##头 +##夷 +##夸 +##夹 +##夺 +##夾 +##奂 +##奄 +##奇 +##奈 +##奉 +##奋 +##奎 +##奏 +##奐 +##契 +##奔 +##奕 +##奖 +##套 +##奘 +##奚 +##奠 +##奢 +##奥 +##奧 +##奪 +##奬 +##奮 +##女 +##奴 +##奶 +##奸 +##她 +##好 +##如 +##妃 +##妄 +##妆 +##妇 +##妈 +##妊 +##妍 +##妒 +##妓 +##妖 +##妘 +##妙 +##妝 +##妞 +##妣 +##妤 +##妥 +##妨 +##妩 +##妪 +##妮 +##妲 +##妳 +##妹 +##妻 +##妾 +##姆 +##姉 +##姊 +##始 +##姍 +##姐 +##姑 +##姒 +##姓 +##委 +##姗 +##姚 +##姜 +##姝 +##姣 +##姥 +##姦 +##姨 +##姪 +##姫 +##姬 +##姹 +##姻 +##姿 +##威 +##娃 +##娄 +##娅 +##娆 +##娇 +##娉 +##娑 +##娓 +##娘 +##娛 +##娜 +##娟 +##娠 +##娣 +##娥 +##娩 +##娱 +##娲 +##娴 +##娶 +##娼 +##婀 +##婁 +##婆 +##婉 +##婊 +##婕 +##婚 +##婢 +##婦 +##婧 +##婪 +##婭 +##婴 +##婵 +##婶 +##婷 +##婺 +##婿 +##媒 +##媚 +##媛 +##媞 +##媧 +##媲 +##媳 +##媽 +##媾 +##嫁 +##嫂 +##嫉 +##嫌 +##嫑 +##嫔 +##嫖 +##嫘 +##嫚 +##嫡 +##嫣 +##嫦 +##嫩 +##嫲 +##嫵 +##嫻 +##嬅 +##嬉 +##嬌 +##嬗 +##嬛 +##嬢 +##嬤 +##嬪 +##嬰 +##嬴 +##嬷 +##嬸 +##嬿 +##孀 +##孃 +##子 +##孑 +##孔 +##孕 +##孖 +##字 +##存 +##孙 +##孚 +##孛 +##孜 +##孝 +##孟 +##孢 +##季 +##孤 +##学 +##孩 +##孪 +##孫 +##孬 +##孰 +##孱 +##孳 +##孵 +##學 +##孺 +##孽 +##孿 +##宁 +##它 +##宅 +##宇 +##守 +##安 +##宋 +##完 +##宏 +##宓 +##宕 +##宗 +##官 +##宙 +##定 +##宛 +##宜 +##宝 +##实 +##実 +##宠 +##审 +##客 +##宣 +##室 +##宥 +##宦 +##宪 +##宫 +##宮 +##宰 +##害 +##宴 +##宵 +##家 +##宸 +##容 +##宽 +##宾 +##宿 +##寂 +##寄 +##寅 +##密 +##寇 +##富 +##寐 +##寒 +##寓 +##寛 +##寝 +##寞 +##察 +##寡 +##寢 +##寥 +##實 +##寧 +##寨 +##審 +##寫 +##寬 +##寮 +##寰 +##寵 +##寶 +##寸 +##对 +##寺 +##寻 +##导 +##対 +##寿 +##封 +##専 +##射 +##将 +##將 +##專 +##尉 +##尊 +##尋 +##對 +##導 +##小 +##少 +##尔 +##尕 +##尖 +##尘 +##尚 +##尝 +##尤 +##尧 +##尬 +##就 +##尴 +##尷 +##尸 +##尹 +##尺 +##尻 +##尼 +##尽 +##尾 +##尿 +##局 +##屁 +##层 +##屄 +##居 +##屆 +##屈 +##屉 +##届 +##屋 +##屌 +##屍 +##屎 +##屏 +##屐 +##屑 +##展 +##屜 +##属 +##屠 +##屡 +##屢 +##層 +##履 +##屬 +##屯 +##山 +##屹 +##屿 +##岀 +##岁 +##岂 +##岌 +##岐 +##岑 +##岔 +##岖 +##岗 +##岘 +##岙 +##岚 +##岛 +##岡 +##岩 +##岫 +##岬 +##岭 +##岱 +##岳 +##岷 +##岸 +##峇 +##峋 +##峒 +##峙 +##峡 +##峤 +##峥 +##峦 +##峨 +##峪 +##峭 +##峯 +##峰 +##峴 +##島 +##峻 +##峽 +##崁 +##崂 +##崆 +##崇 +##崎 +##崑 +##崔 +##崖 +##崗 +##崙 +##崛 +##崧 +##崩 +##崭 +##崴 +##崽 +##嵇 +##嵊 +##嵋 +##嵌 +##嵐 +##嵘 +##嵩 +##嵬 +##嵯 +##嶂 +##嶄 +##嶇 +##嶋 +##嶙 +##嶺 +##嶼 +##嶽 +##巅 +##巍 +##巒 +##巔 +##巖 +##川 +##州 +##巡 +##巢 +##工 +##左 +##巧 +##巨 +##巩 +##巫 +##差 +##己 +##已 +##巳 +##巴 +##巷 +##巻 +##巽 +##巾 +##巿 +##币 +##市 +##布 +##帅 +##帆 +##师 +##希 +##帐 +##帑 +##帕 +##帖 +##帘 +##帚 +##帛 +##帜 +##帝 +##帥 +##带 +##帧 +##師 +##席 +##帮 +##帯 +##帰 +##帳 +##帶 +##帷 +##常 +##帼 +##帽 +##幀 +##幂 +##幄 +##幅 +##幌 +##幔 +##幕 +##幟 +##幡 +##幢 +##幣 +##幫 +##干 +##平 +##年 +##并 +##幸 +##幹 +##幺 +##幻 +##幼 +##幽 +##幾 +##广 +##庁 +##広 +##庄 +##庆 +##庇 +##床 +##序 +##庐 +##库 +##应 +##底 +##庖 +##店 +##庙 +##庚 +##府 +##庞 +##废 +##庠 +##度 +##座 +##庫 +##庭 +##庵 +##庶 +##康 +##庸 +##庹 +##庾 +##廁 +##廂 +##廃 +##廈 +##廉 +##廊 +##廓 +##廖 +##廚 +##廝 +##廟 +##廠 +##廢 +##廣 +##廬 +##廳 +##延 +##廷 +##建 +##廿 +##开 +##弁 +##异 +##弃 +##弄 +##弈 +##弊 +##弋 +##式 +##弑 +##弒 +##弓 +##弔 +##引 +##弗 +##弘 +##弛 +##弟 +##张 +##弥 +##弦 +##弧 +##弩 +##弭 +##弯 +##弱 +##張 +##強 +##弹 +##强 +##弼 +##弾 +##彅 +##彆 +##彈 +##彌 +##彎 +##归 +##当 +##录 +##彗 +##彙 +##彝 +##形 +##彤 +##彥 +##彦 +##彧 +##彩 +##彪 +##彫 +##彬 +##彭 +##彰 +##影 +##彷 +##役 +##彻 +##彼 +##彿 +##往 +##征 +##径 +##待 +##徇 +##很 +##徉 +##徊 +##律 +##後 +##徐 +##徑 +##徒 +##従 +##徕 +##得 +##徘 +##徙 +##徜 +##從 +##徠 +##御 +##徨 +##復 +##循 +##徬 +##微 +##徳 +##徴 +##徵 +##德 +##徹 +##徼 +##徽 +##心 +##必 +##忆 +##忌 +##忍 +##忏 +##忐 +##忑 +##忒 +##忖 +##志 +##忘 +##忙 +##応 +##忠 +##忡 +##忤 +##忧 +##忪 +##快 +##忱 +##念 +##忻 +##忽 +##忿 +##怀 +##态 +##怂 +##怅 +##怆 +##怎 +##怏 +##怒 +##怔 +##怕 +##怖 +##怙 +##怜 +##思 +##怠 +##怡 +##急 +##怦 +##性 +##怨 +##怪 +##怯 +##怵 +##总 +##怼 +##恁 +##恃 +##恆 +##恋 +##恍 +##恐 +##恒 +##恕 +##恙 +##恚 +##恢 +##恣 +##恤 +##恥 +##恨 +##恩 +##恪 +##恫 +##恬 +##恭 +##息 +##恰 +##恳 +##恵 +##恶 +##恸 +##恺 +##恻 +##恼 +##恿 +##悄 +##悅 +##悉 +##悌 +##悍 +##悔 +##悖 +##悚 +##悟 +##悠 +##患 +##悦 +##您 +##悩 +##悪 +##悬 +##悯 +##悱 +##悲 +##悴 +##悵 +##悶 +##悸 +##悻 +##悼 +##悽 +##情 +##惆 +##惇 +##惊 +##惋 +##惑 +##惕 +##惘 +##惚 +##惜 +##惟 +##惠 +##惡 +##惦 +##惧 +##惨 +##惩 +##惫 +##惬 +##惭 +##惮 +##惯 +##惰 +##惱 +##想 +##惴 +##惶 +##惹 +##惺 +##愁 +##愆 +##愈 +##愉 +##愍 +##意 +##愕 +##愚 +##愛 +##愜 +##感 +##愣 +##愤 +##愧 +##愫 +##愷 +##愿 +##慄 +##慈 +##態 +##慌 +##慎 +##慑 +##慕 +##慘 +##慚 +##慟 +##慢 +##慣 +##慧 +##慨 +##慫 +##慮 +##慰 +##慳 +##慵 +##慶 +##慷 +##慾 +##憂 +##憊 +##憋 +##憎 +##憐 +##憑 +##憔 +##憚 +##憤 +##憧 +##憨 +##憩 +##憫 +##憬 +##憲 +##憶 +##憾 +##懂 +##懇 +##懈 +##應 +##懊 +##懋 +##懑 +##懒 +##懦 +##懲 +##懵 +##懶 +##懷 +##懸 +##懺 +##懼 +##懾 +##懿 +##戀 +##戈 +##戊 +##戌 +##戍 +##戎 +##戏 +##成 +##我 +##戒 +##戕 +##或 +##战 +##戚 +##戛 +##戟 +##戡 +##戦 +##截 +##戬 +##戮 +##戰 +##戲 +##戳 +##戴 +##戶 +##户 +##戸 +##戻 +##戾 +##房 +##所 +##扁 +##扇 +##扈 +##扉 +##手 +##才 +##扎 +##扑 +##扒 +##打 +##扔 +##払 +##托 +##扛 +##扣 +##扦 +##执 +##扩 +##扪 +##扫 +##扬 +##扭 +##扮 +##扯 +##扰 +##扱 +##扳 +##扶 +##批 +##扼 +##找 +##承 +##技 +##抄 +##抉 +##把 +##抑 +##抒 +##抓 +##投 +##抖 +##抗 +##折 +##抚 +##抛 +##抜 +##択 +##抟 +##抠 +##抡 +##抢 +##护 +##报 +##抨 +##披 +##抬 +##抱 +##抵 +##抹 +##押 +##抽 +##抿 +##拂 +##拄 +##担 +##拆 +##拇 +##拈 +##拉 +##拋 +##拌 +##拍 +##拎 +##拐 +##拒 +##拓 +##拔 +##拖 +##拗 +##拘 +##拙 +##拚 +##招 +##拜 +##拟 +##拡 +##拢 +##拣 +##拥 +##拦 +##拧 +##拨 +##择 +##括 +##拭 +##拮 +##拯 +##拱 +##拳 +##拴 +##拷 +##拼 +##拽 +##拾 +##拿 +##持 +##挂 +##指 +##挈 +##按 +##挎 +##挑 +##挖 +##挙 +##挚 +##挛 +##挝 +##挞 +##挟 +##挠 +##挡 +##挣 +##挤 +##挥 +##挨 +##挪 +##挫 +##振 +##挲 +##挹 +##挺 +##挽 +##挾 +##捂 +##捅 +##捆 +##捉 +##捋 +##捌 +##捍 +##捎 +##捏 +##捐 +##捕 +##捞 +##损 +##捡 +##换 +##捣 +##捧 +##捨 +##捩 +##据 +##捱 +##捲 +##捶 +##捷 +##捺 +##捻 +##掀 +##掂 +##掃 +##掇 +##授 +##掉 +##掌 +##掏 +##掐 +##排 +##掖 +##掘 +##掙 +##掛 +##掠 +##採 +##探 +##掣 +##接 +##控 +##推 +##掩 +##措 +##掬 +##掰 +##掲 +##掳 +##掴 +##掷 +##掸 +##掺 +##揀 +##揃 +##揄 +##揆 +##揉 +##揍 +##描 +##提 +##插 +##揖 +##揚 +##換 +##握 +##揣 +##揩 +##揪 +##揭 +##揮 +##援 +##揶 +##揸 +##揹 +##揽 +##搀 +##搁 +##搂 +##搅 +##損 +##搏 +##搐 +##搓 +##搔 +##搖 +##搗 +##搜 +##搞 +##搡 +##搪 +##搬 +##搭 +##搵 +##搶 +##携 +##搽 +##摀 +##摁 +##摄 +##摆 +##摇 +##摈 +##摊 +##摒 +##摔 +##摘 +##摞 +##摟 +##摧 +##摩 +##摯 +##摳 +##摸 +##摹 +##摺 +##摻 +##撂 +##撃 +##撅 +##撇 +##撈 +##撐 +##撑 +##撒 +##撓 +##撕 +##撚 +##撞 +##撤 +##撥 +##撩 +##撫 +##撬 +##播 +##撮 +##撰 +##撲 +##撵 +##撷 +##撸 +##撻 +##撼 +##撿 +##擀 +##擁 +##擂 +##擄 +##擅 +##擇 +##擊 +##擋 +##操 +##擎 +##擒 +##擔 +##擘 +##據 +##擞 +##擠 +##擡 +##擢 +##擦 +##擬 +##擰 +##擱 +##擲 +##擴 +##擷 +##擺 +##擼 +##擾 +##攀 +##攏 +##攒 +##攔 +##攘 +##攙 +##攜 +##攝 +##攞 +##攢 +##攣 +##攤 +##攥 +##攪 +##攫 +##攬 +##支 +##收 +##攸 +##改 +##攻 +##放 +##政 +##故 +##效 +##敌 +##敍 +##敎 +##敏 +##救 +##敕 +##敖 +##敗 +##敘 +##教 +##敛 +##敝 +##敞 +##敢 +##散 +##敦 +##敬 +##数 +##敲 +##整 +##敵 +##敷 +##數 +##斂 +##斃 +##文 +##斋 +##斌 +##斎 +##斐 +##斑 +##斓 +##斗 +##料 +##斛 +##斜 +##斟 +##斡 +##斤 +##斥 +##斧 +##斩 +##斫 +##斬 +##断 +##斯 +##新 +##斷 +##方 +##於 +##施 +##旁 +##旃 +##旅 +##旋 +##旌 +##旎 +##族 +##旖 +##旗 +##无 +##既 +##日 +##旦 +##旧 +##旨 +##早 +##旬 +##旭 +##旮 +##旱 +##时 +##旷 +##旺 +##旻 +##昀 +##昂 +##昆 +##昇 +##昉 +##昊 +##昌 +##明 +##昏 +##易 +##昔 +##昕 +##昙 +##星 +##映 +##春 +##昧 +##昨 +##昭 +##是 +##昱 +##昴 +##昵 +##昶 +##昼 +##显 +##晁 +##時 +##晃 +##晉 +##晋 +##晌 +##晏 +##晒 +##晓 +##晔 +##晕 +##晖 +##晗 +##晚 +##晝 +##晞 +##晟 +##晤 +##晦 +##晨 +##晩 +##普 +##景 +##晰 +##晴 +##晶 +##晷 +##智 +##晾 +##暂 +##暄 +##暇 +##暈 +##暉 +##暌 +##暐 +##暑 +##暖 +##暗 +##暝 +##暢 +##暧 +##暨 +##暫 +##暮 +##暱 +##暴 +##暸 +##暹 +##曄 +##曆 +##曇 +##曉 +##曖 +##曙 +##曜 +##曝 +##曠 +##曦 +##曬 +##曰 +##曲 +##曳 +##更 +##書 +##曹 +##曼 +##曾 +##替 +##最 +##會 +##月 +##有 +##朋 +##服 +##朐 +##朔 +##朕 +##朗 +##望 +##朝 +##期 +##朦 +##朧 +##木 +##未 +##末 +##本 +##札 +##朮 +##术 +##朱 +##朴 +##朵 +##机 +##朽 +##杀 +##杂 +##权 +##杆 +##杈 +##杉 +##李 +##杏 +##材 +##村 +##杓 +##杖 +##杜 +##杞 +##束 +##杠 +##条 +##来 +##杨 +##杭 +##杯 +##杰 +##東 +##杳 +##杵 +##杷 +##杼 +##松 +##板 +##极 +##构 +##枇 +##枉 +##枋 +##析 +##枕 +##林 +##枚 +##果 +##枝 +##枢 +##枣 +##枪 +##枫 +##枭 +##枯 +##枰 +##枱 +##枳 +##架 +##枷 +##枸 +##柄 +##柏 +##某 +##柑 +##柒 +##染 +##柔 +##柘 +##柚 +##柜 +##柞 +##柠 +##柢 +##查 +##柩 +##柬 +##柯 +##柱 +##柳 +##柴 +##柵 +##査 +##柿 +##栀 +##栃 +##栄 +##栅 +##标 +##栈 +##栉 +##栋 +##栎 +##栏 +##树 +##栓 +##栖 +##栗 +##校 +##栩 +##株 +##样 +##核 +##根 +##格 +##栽 +##栾 +##桀 +##桁 +##桂 +##桃 +##桅 +##框 +##案 +##桉 +##桌 +##桎 +##桐 +##桑 +##桓 +##桔 +##桜 +##桠 +##桡 +##桢 +##档 +##桥 +##桦 +##桧 +##桨 +##桩 +##桶 +##桿 +##梁 +##梅 +##梆 +##梏 +##梓 +##梗 +##條 +##梟 +##梢 +##梦 +##梧 +##梨 +##梭 +##梯 +##械 +##梳 +##梵 +##梶 +##检 +##棂 +##棄 +##棉 +##棋 +##棍 +##棒 +##棕 +##棗 +##棘 +##棚 +##棟 +##棠 +##棣 +##棧 +##森 +##棱 +##棲 +##棵 +##棹 +##棺 +##椁 +##椅 +##椋 +##植 +##椎 +##椒 +##検 +##椪 +##椭 +##椰 +##椹 +##椽 +##椿 +##楂 +##楊 +##楓 +##楔 +##楚 +##楝 +##楞 +##楠 +##楣 +##楨 +##楫 +##業 +##楮 +##極 +##楷 +##楸 +##楹 +##楼 +##楽 +##概 +##榄 +##榆 +##榈 +##榉 +##榔 +##榕 +##榖 +##榛 +##榜 +##榨 +##榫 +##榭 +##榮 +##榱 +##榴 +##榷 +##榻 +##槁 +##槃 +##構 +##槌 +##槍 +##槎 +##槐 +##槓 +##様 +##槛 +##槟 +##槤 +##槭 +##槲 +##槳 +##槻 +##槽 +##槿 +##樁 +##樂 +##樊 +##樑 +##樓 +##標 +##樞 +##樟 +##模 +##樣 +##権 +##横 +##樫 +##樯 +##樱 +##樵 +##樸 +##樹 +##樺 +##樽 +##樾 +##橄 +##橇 +##橋 +##橐 +##橘 +##橙 +##機 +##橡 +##橢 +##橫 +##橱 +##橹 +##橼 +##檀 +##檄 +##檎 +##檐 +##檔 +##檗 +##檜 +##檢 +##檬 +##檯 +##檳 +##檸 +##檻 +##櫃 +##櫚 +##櫛 +##櫥 +##櫸 +##櫻 +##欄 +##權 +##欒 +##欖 +##欠 +##次 +##欢 +##欣 +##欧 +##欲 +##欸 +##欺 +##欽 +##款 +##歆 +##歇 +##歉 +##歌 +##歎 +##歐 +##歓 +##歙 +##歛 +##歡 +##止 +##正 +##此 +##步 +##武 +##歧 +##歩 +##歪 +##歯 +##歲 +##歳 +##歴 +##歷 +##歸 +##歹 +##死 +##歼 +##殁 +##殃 +##殆 +##殇 +##殉 +##殊 +##残 +##殒 +##殓 +##殖 +##殘 +##殞 +##殡 +##殤 +##殭 +##殯 +##殲 +##殴 +##段 +##殷 +##殺 +##殼 +##殿 +##毀 +##毁 +##毂 +##毅 +##毆 +##毋 +##母 +##毎 +##每 +##毒 +##毓 +##比 +##毕 +##毗 +##毘 +##毙 +##毛 +##毡 +##毫 +##毯 +##毽 +##氈 +##氏 +##氐 +##民 +##氓 +##气 +##氖 +##気 +##氙 +##氛 +##氟 +##氡 +##氢 +##氣 +##氤 +##氦 +##氧 +##氨 +##氪 +##氫 +##氮 +##氯 +##氰 +##氲 +##水 +##氷 +##永 +##氹 +##氾 +##汀 +##汁 +##求 +##汆 +##汇 +##汉 +##汎 +##汐 +##汕 +##汗 +##汙 +##汛 +##汝 +##汞 +##江 +##池 +##污 +##汤 +##汨 +##汩 +##汪 +##汰 +##汲 +##汴 +##汶 +##汹 +##決 +##汽 +##汾 +##沁 +##沂 +##沃 +##沅 +##沈 +##沉 +##沌 +##沏 +##沐 +##沒 +##沓 +##沖 +##沙 +##沛 +##沟 +##没 +##沢 +##沣 +##沥 +##沦 +##沧 +##沪 +##沫 +##沭 +##沮 +##沱 +##河 +##沸 +##油 +##治 +##沼 +##沽 +##沾 +##沿 +##況 +##泄 +##泉 +##泊 +##泌 +##泓 +##法 +##泗 +##泛 +##泞 +##泠 +##泡 +##波 +##泣 +##泥 +##注 +##泪 +##泫 +##泮 +##泯 +##泰 +##泱 +##泳 +##泵 +##泷 +##泸 +##泻 +##泼 +##泽 +##泾 +##洁 +##洄 +##洋 +##洒 +##洗 +##洙 +##洛 +##洞 +##津 +##洩 +##洪 +##洮 +##洱 +##洲 +##洵 +##洶 +##洸 +##洹 +##活 +##洼 +##洽 +##派 +##流 +##浃 +##浄 +##浅 +##浆 +##浇 +##浊 +##测 +##济 +##浏 +##浑 +##浒 +##浓 +##浔 +##浙 +##浚 +##浜 +##浣 +##浦 +##浩 +##浪 +##浬 +##浮 +##浯 +##浴 +##海 +##浸 +##涂 +##涅 +##涇 +##消 +##涉 +##涌 +##涎 +##涓 +##涔 +##涕 +##涙 +##涛 +##涝 +##涞 +##涟 +##涠 +##涡 +##涣 +##涤 +##润 +##涧 +##涨 +##涩 +##涪 +##涮 +##涯 +##液 +##涵 +##涸 +##涼 +##涿 +##淀 +##淄 +##淅 +##淆 +##淇 +##淋 +##淌 +##淑 +##淒 +##淖 +##淘 +##淙 +##淚 +##淞 +##淡 +##淤 +##淦 +##淨 +##淩 +##淪 +##淫 +##淬 +##淮 +##深 +##淳 +##淵 +##混 +##淹 +##淺 +##添 +##淼 +##清 +##済 +##渉 +##渊 +##渋 +##渍 +##渎 +##渐 +##渔 +##渗 +##渙 +##渚 +##減 +##渝 +##渠 +##渡 +##渣 +##渤 +##渥 +##渦 +##温 +##測 +##渭 +##港 +##渲 +##渴 +##游 +##渺 +##渾 +##湃 +##湄 +##湊 +##湍 +##湖 +##湘 +##湛 +##湟 +##湧 +##湫 +##湮 +##湯 +##湳 +##湾 +##湿 +##満 +##溃 +##溅 +##溉 +##溏 +##源 +##準 +##溜 +##溝 +##溟 +##溢 +##溥 +##溧 +##溪 +##溫 +##溯 +##溱 +##溴 +##溶 +##溺 +##溼 +##滁 +##滂 +##滄 +##滅 +##滇 +##滋 +##滌 +##滑 +##滓 +##滔 +##滕 +##滙 +##滚 +##滝 +##滞 +##滟 +##满 +##滢 +##滤 +##滥 +##滦 +##滨 +##滩 +##滬 +##滯 +##滲 +##滴 +##滷 +##滸 +##滾 +##滿 +##漁 +##漂 +##漆 +##漉 +##漏 +##漓 +##演 +##漕 +##漠 +##漢 +##漣 +##漩 +##漪 +##漫 +##漬 +##漯 +##漱 +##漲 +##漳 +##漸 +##漾 +##漿 +##潆 +##潇 +##潋 +##潍 +##潑 +##潔 +##潘 +##潛 +##潜 +##潞 +##潟 +##潢 +##潤 +##潦 +##潧 +##潭 +##潮 +##潰 +##潴 +##潸 +##潺 +##潼 +##澀 +##澄 +##澆 +##澈 +##澍 +##澎 +##澗 +##澜 +##澡 +##澤 +##澧 +##澱 +##澳 +##澹 +##激 +##濁 +##濂 +##濃 +##濑 +##濒 +##濕 +##濘 +##濛 +##濟 +##濠 +##濡 +##濤 +##濫 +##濬 +##濮 +##濯 +##濱 +##濺 +##濾 +##瀅 +##瀆 +##瀉 +##瀋 +##瀏 +##瀑 +##瀕 +##瀘 +##瀚 +##瀛 +##瀝 +##瀞 +##瀟 +##瀧 +##瀨 +##瀬 +##瀰 +##瀾 +##灌 +##灏 +##灑 +##灘 +##灝 +##灞 +##灣 +##火 +##灬 +##灭 +##灯 +##灰 +##灵 +##灶 +##灸 +##灼 +##災 +##灾 +##灿 +##炀 +##炁 +##炅 +##炉 +##炊 +##炎 +##炒 +##炔 +##炕 +##炖 +##炙 +##炜 +##炫 +##炬 +##炭 +##炮 +##炯 +##炳 +##炷 +##炸 +##点 +##為 +##炼 +##炽 +##烁 +##烂 +##烃 +##烈 +##烊 +##烏 +##烘 +##烙 +##烛 +##烟 +##烤 +##烦 +##烧 +##烨 +##烩 +##烫 +##烬 +##热 +##烯 +##烷 +##烹 +##烽 +##焉 +##焊 +##焕 +##焖 +##焗 +##焘 +##焙 +##焚 +##焜 +##無 +##焦 +##焯 +##焰 +##焱 +##然 +##焼 +##煅 +##煉 +##煊 +##煌 +##煎 +##煒 +##煖 +##煙 +##煜 +##煞 +##煤 +##煥 +##煦 +##照 +##煨 +##煩 +##煮 +##煲 +##煸 +##煽 +##熄 +##熊 +##熏 +##熒 +##熔 +##熙 +##熟 +##熠 +##熨 +##熬 +##熱 +##熵 +##熹 +##熾 +##燁 +##燃 +##燄 +##燈 +##燉 +##燊 +##燎 +##燒 +##燔 +##燕 +##燙 +##燜 +##營 +##燥 +##燦 +##燧 +##燭 +##燮 +##燴 +##燻 +##燼 +##燿 +##爆 +##爍 +##爐 +##爛 +##爪 +##爬 +##爭 +##爰 +##爱 +##爲 +##爵 +##父 +##爷 +##爸 +##爹 +##爺 +##爻 +##爽 +##爾 +##牆 +##片 +##版 +##牌 +##牍 +##牒 +##牙 +##牛 +##牝 +##牟 +##牠 +##牡 +##牢 +##牦 +##牧 +##物 +##牯 +##牲 +##牴 +##牵 +##特 +##牺 +##牽 +##犀 +##犁 +##犄 +##犊 +##犍 +##犒 +##犢 +##犧 +##犬 +##犯 +##状 +##犷 +##犸 +##犹 +##狀 +##狂 +##狄 +##狈 +##狎 +##狐 +##狒 +##狗 +##狙 +##狞 +##狠 +##狡 +##狩 +##独 +##狭 +##狮 +##狰 +##狱 +##狸 +##狹 +##狼 +##狽 +##猎 +##猕 +##猖 +##猗 +##猙 +##猛 +##猜 +##猝 +##猥 +##猩 +##猪 +##猫 +##猬 +##献 +##猴 +##猶 +##猷 +##猾 +##猿 +##獄 +##獅 +##獎 +##獐 +##獒 +##獗 +##獠 +##獣 +##獨 +##獭 +##獰 +##獲 +##獵 +##獷 +##獸 +##獺 +##獻 +##獼 +##獾 +##玄 +##率 +##玉 +##王 +##玑 +##玖 +##玛 +##玟 +##玠 +##玥 +##玩 +##玫 +##玮 +##环 +##现 +##玲 +##玳 +##玷 +##玺 +##玻 +##珀 +##珂 +##珅 +##珈 +##珉 +##珊 +##珍 +##珏 +##珐 +##珑 +##珙 +##珞 +##珠 +##珣 +##珥 +##珩 +##珪 +##班 +##珮 +##珲 +##珺 +##現 +##球 +##琅 +##理 +##琇 +##琉 +##琊 +##琍 +##琏 +##琐 +##琛 +##琢 +##琥 +##琦 +##琨 +##琪 +##琬 +##琮 +##琰 +##琲 +##琳 +##琴 +##琵 +##琶 +##琺 +##琼 +##瑀 +##瑁 +##瑄 +##瑋 +##瑕 +##瑗 +##瑙 +##瑚 +##瑛 +##瑜 +##瑞 +##瑟 +##瑠 +##瑣 +##瑤 +##瑩 +##瑪 +##瑯 +##瑰 +##瑶 +##瑾 +##璀 +##璁 +##璃 +##璇 +##璉 +##璋 +##璎 +##璐 +##璜 +##璞 +##璟 +##璧 +##璨 +##環 +##璽 +##璿 +##瓊 +##瓏 +##瓒 +##瓜 +##瓢 +##瓣 +##瓤 +##瓦 +##瓮 +##瓯 +##瓴 +##瓶 +##瓷 +##甄 +##甌 +##甕 +##甘 +##甙 +##甚 +##甜 +##生 +##產 +##産 +##甥 +##甦 +##用 +##甩 +##甫 +##甬 +##甭 +##甯 +##田 +##由 +##甲 +##申 +##电 +##男 +##甸 +##町 +##画 +##甾 +##畀 +##畅 +##界 +##畏 +##畑 +##畔 +##留 +##畜 +##畝 +##畢 +##略 +##畦 +##番 +##畫 +##異 +##畲 +##畳 +##畴 +##當 +##畸 +##畹 +##畿 +##疆 +##疇 +##疊 +##疏 +##疑 +##疔 +##疖 +##疗 +##疙 +##疚 +##疝 +##疟 +##疡 +##疣 +##疤 +##疥 +##疫 +##疮 +##疯 +##疱 +##疲 +##疳 +##疵 +##疸 +##疹 +##疼 +##疽 +##疾 +##痂 +##病 +##症 +##痈 +##痉 +##痊 +##痍 +##痒 +##痔 +##痕 +##痘 +##痙 +##痛 +##痞 +##痠 +##痢 +##痣 +##痤 +##痧 +##痨 +##痪 +##痫 +##痰 +##痱 +##痴 +##痹 +##痺 +##痼 +##痿 +##瘀 +##瘁 +##瘋 +##瘍 +##瘓 +##瘘 +##瘙 +##瘟 +##瘠 +##瘡 +##瘢 +##瘤 +##瘦 +##瘧 +##瘩 +##瘪 +##瘫 +##瘴 +##瘸 +##瘾 +##療 +##癇 +##癌 +##癒 +##癖 +##癜 +##癞 +##癡 +##癢 +##癣 +##癥 +##癫 +##癬 +##癮 +##癱 +##癲 +##癸 +##発 +##登 +##發 +##白 +##百 +##皂 +##的 +##皆 +##皇 +##皈 +##皋 +##皎 +##皑 +##皓 +##皖 +##皙 +##皚 +##皮 +##皰 +##皱 +##皴 +##皺 +##皿 +##盂 +##盃 +##盅 +##盆 +##盈 +##益 +##盎 +##盏 +##盐 +##监 +##盒 +##盔 +##盖 +##盗 +##盘 +##盛 +##盜 +##盞 +##盟 +##盡 +##監 +##盤 +##盥 +##盧 +##盪 +##目 +##盯 +##盱 +##盲 +##直 +##相 +##盹 +##盼 +##盾 +##省 +##眈 +##眉 +##看 +##県 +##眙 +##眞 +##真 +##眠 +##眦 +##眨 +##眩 +##眯 +##眶 +##眷 +##眸 +##眺 +##眼 +##眾 +##着 +##睁 +##睇 +##睏 +##睐 +##睑 +##睛 +##睜 +##睞 +##睡 +##睢 +##督 +##睥 +##睦 +##睨 +##睪 +##睫 +##睬 +##睹 +##睽 +##睾 +##睿 +##瞄 +##瞅 +##瞇 +##瞋 +##瞌 +##瞎 +##瞑 +##瞒 +##瞓 +##瞞 +##瞟 +##瞠 +##瞥 +##瞧 +##瞩 +##瞪 +##瞬 +##瞭 +##瞰 +##瞳 +##瞻 +##瞼 +##瞿 +##矇 +##矍 +##矗 +##矚 +##矛 +##矜 +##矢 +##矣 +##知 +##矩 +##矫 +##短 +##矮 +##矯 +##石 +##矶 +##矽 +##矾 +##矿 +##码 +##砂 +##砌 +##砍 +##砒 +##研 +##砖 +##砗 +##砚 +##砝 +##砣 +##砥 +##砧 +##砭 +##砰 +##砲 +##破 +##砷 +##砸 +##砺 +##砼 +##砾 +##础 +##硅 +##硐 +##硒 +##硕 +##硝 +##硫 +##硬 +##确 +##硯 +##硼 +##碁 +##碇 +##碉 +##碌 +##碍 +##碎 +##碑 +##碓 +##碗 +##碘 +##碚 +##碛 +##碟 +##碣 +##碧 +##碩 +##碰 +##碱 +##碳 +##碴 +##確 +##碼 +##碾 +##磁 +##磅 +##磊 +##磋 +##磐 +##磕 +##磚 +##磡 +##磨 +##磬 +##磯 +##磲 +##磷 +##磺 +##礁 +##礎 +##礙 +##礡 +##礦 +##礪 +##礫 +##礴 +##示 +##礼 +##社 +##祀 +##祁 +##祂 +##祇 +##祈 +##祉 +##祎 +##祐 +##祕 +##祖 +##祗 +##祚 +##祛 +##祜 +##祝 +##神 +##祟 +##祠 +##祢 +##祥 +##票 +##祭 +##祯 +##祷 +##祸 +##祺 +##祿 +##禀 +##禁 +##禄 +##禅 +##禍 +##禎 +##福 +##禛 +##禦 +##禧 +##禪 +##禮 +##禱 +##禹 +##禺 +##离 +##禽 +##禾 +##禿 +##秀 +##私 +##秃 +##秆 +##秉 +##秋 +##种 +##科 +##秒 +##秘 +##租 +##秣 +##秤 +##秦 +##秧 +##秩 +##秭 +##积 +##称 +##秸 +##移 +##秽 +##稀 +##稅 +##程 +##稍 +##税 +##稔 +##稗 +##稚 +##稜 +##稞 +##稟 +##稠 +##稣 +##種 +##稱 +##稲 +##稳 +##稷 +##稹 +##稻 +##稼 +##稽 +##稿 +##穀 +##穂 +##穆 +##穌 +##積 +##穎 +##穗 +##穢 +##穩 +##穫 +##穴 +##究 +##穷 +##穹 +##空 +##穿 +##突 +##窃 +##窄 +##窈 +##窍 +##窑 +##窒 +##窓 +##窕 +##窖 +##窗 +##窘 +##窜 +##窝 +##窟 +##窠 +##窥 +##窦 +##窨 +##窩 +##窪 +##窮 +##窯 +##窺 +##窿 +##竄 +##竅 +##竇 +##竊 +##立 +##竖 +##站 +##竜 +##竞 +##竟 +##章 +##竣 +##童 +##竭 +##端 +##競 +##竹 +##竺 +##竽 +##竿 +##笃 +##笆 +##笈 +##笋 +##笏 +##笑 +##笔 +##笙 +##笛 +##笞 +##笠 +##符 +##笨 +##第 +##笹 +##笺 +##笼 +##筆 +##等 +##筊 +##筋 +##筍 +##筏 +##筐 +##筑 +##筒 +##答 +##策 +##筛 +##筝 +##筠 +##筱 +##筲 +##筵 +##筷 +##筹 +##签 +##简 +##箇 +##箋 +##箍 +##箏 +##箐 +##箔 +##箕 +##算 +##箝 +##管 +##箩 +##箫 +##箭 +##箱 +##箴 +##箸 +##節 +##篁 +##範 +##篆 +##篇 +##築 +##篑 +##篓 +##篙 +##篝 +##篠 +##篡 +##篤 +##篩 +##篪 +##篮 +##篱 +##篷 +##簇 +##簌 +##簍 +##簡 +##簦 +##簧 +##簪 +##簫 +##簷 +##簸 +##簽 +##簾 +##簿 +##籁 +##籃 +##籌 +##籍 +##籐 +##籟 +##籠 +##籤 +##籬 +##籮 +##籲 +##米 +##类 +##籼 +##籽 +##粄 +##粉 +##粑 +##粒 +##粕 +##粗 +##粘 +##粟 +##粤 +##粥 +##粧 +##粪 +##粮 +##粱 +##粲 +##粳 +##粵 +##粹 +##粼 +##粽 +##精 +##粿 +##糅 +##糊 +##糍 +##糕 +##糖 +##糗 +##糙 +##糜 +##糞 +##糟 +##糠 +##糧 +##糬 +##糯 +##糰 +##糸 +##系 +##糾 +##紀 +##紂 +##約 +##紅 +##紉 +##紊 +##紋 +##納 +##紐 +##紓 +##純 +##紗 +##紘 +##紙 +##級 +##紛 +##紜 +##素 +##紡 +##索 +##紧 +##紫 +##紮 +##累 +##細 +##紳 +##紹 +##紺 +##終 +##絃 +##組 +##絆 +##経 +##結 +##絕 +##絞 +##絡 +##絢 +##給 +##絨 +##絮 +##統 +##絲 +##絳 +##絵 +##絶 +##絹 +##綁 +##綏 +##綑 +##經 +##継 +##続 +##綜 +##綠 +##綢 +##綦 +##綫 +##綬 +##維 +##綱 +##網 +##綴 +##綵 +##綸 +##綺 +##綻 +##綽 +##綾 +##綿 +##緊 +##緋 +##総 +##緑 +##緒 +##緘 +##線 +##緝 +##緞 +##締 +##緣 +##編 +##緩 +##緬 +##緯 +##練 +##緹 +##緻 +##縁 +##縄 +##縈 +##縛 +##縝 +##縣 +##縫 +##縮 +##縱 +##縴 +##縷 +##總 +##績 +##繁 +##繃 +##繆 +##繇 +##繋 +##織 +##繕 +##繚 +##繞 +##繡 +##繩 +##繪 +##繫 +##繭 +##繳 +##繹 +##繼 +##繽 +##纂 +##續 +##纍 +##纏 +##纓 +##纔 +##纖 +##纜 +##纠 +##红 +##纣 +##纤 +##约 +##级 +##纨 +##纪 +##纫 +##纬 +##纭 +##纯 +##纰 +##纱 +##纲 +##纳 +##纵 +##纶 +##纷 +##纸 +##纹 +##纺 +##纽 +##纾 +##线 +##绀 +##练 +##组 +##绅 +##细 +##织 +##终 +##绊 +##绍 +##绎 +##经 +##绑 +##绒 +##结 +##绔 +##绕 +##绘 +##给 +##绚 +##绛 +##络 +##绝 +##绞 +##统 +##绡 +##绢 +##绣 +##绥 +##绦 +##继 +##绩 +##绪 +##绫 +##续 +##绮 +##绯 +##绰 +##绳 +##维 +##绵 +##绶 +##绷 +##绸 +##绻 +##综 +##绽 +##绾 +##绿 +##缀 +##缄 +##缅 +##缆 +##缇 +##缈 +##缉 +##缎 +##缓 +##缔 +##缕 +##编 +##缘 +##缙 +##缚 +##缜 +##缝 +##缠 +##缢 +##缤 +##缥 +##缨 +##缩 +##缪 +##缭 +##缮 +##缰 +##缱 +##缴 +##缸 +##缺 +##缽 +##罂 +##罄 +##罌 +##罐 +##网 +##罔 +##罕 +##罗 +##罚 +##罡 +##罢 +##罩 +##罪 +##置 +##罰 +##署 +##罵 +##罷 +##罹 +##羁 +##羅 +##羈 +##羊 +##羌 +##美 +##羔 +##羚 +##羞 +##羟 +##羡 +##羣 +##群 +##羥 +##羧 +##羨 +##義 +##羯 +##羲 +##羸 +##羹 +##羽 +##羿 +##翁 +##翅 +##翊 +##翌 +##翎 +##習 +##翔 +##翘 +##翟 +##翠 +##翡 +##翦 +##翩 +##翰 +##翱 +##翳 +##翹 +##翻 +##翼 +##耀 +##老 +##考 +##耄 +##者 +##耆 +##耋 +##而 +##耍 +##耐 +##耒 +##耕 +##耗 +##耘 +##耙 +##耦 +##耨 +##耳 +##耶 +##耷 +##耸 +##耻 +##耽 +##耿 +##聂 +##聆 +##聊 +##聋 +##职 +##聒 +##联 +##聖 +##聘 +##聚 +##聞 +##聪 +##聯 +##聰 +##聲 +##聳 +##聴 +##聶 +##職 +##聽 +##聾 +##聿 +##肃 +##肄 +##肅 +##肆 +##肇 +##肉 +##肋 +##肌 +##肏 +##肓 +##肖 +##肘 +##肚 +##肛 +##肝 +##肠 +##股 +##肢 +##肤 +##肥 +##肩 +##肪 +##肮 +##肯 +##肱 +##育 +##肴 +##肺 +##肽 +##肾 +##肿 +##胀 +##胁 +##胃 +##胄 +##胆 +##背 +##胍 +##胎 +##胖 +##胚 +##胛 +##胜 +##胝 +##胞 +##胡 +##胤 +##胥 +##胧 +##胫 +##胭 +##胯 +##胰 +##胱 +##胳 +##胴 +##胶 +##胸 +##胺 +##能 +##脂 +##脅 +##脆 +##脇 +##脈 +##脉 +##脊 +##脍 +##脏 +##脐 +##脑 +##脓 +##脖 +##脘 +##脚 +##脛 +##脣 +##脩 +##脫 +##脯 +##脱 +##脲 +##脳 +##脸 +##脹 +##脾 +##腆 +##腈 +##腊 +##腋 +##腌 +##腎 +##腐 +##腑 +##腓 +##腔 +##腕 +##腥 +##腦 +##腩 +##腫 +##腭 +##腮 +##腰 +##腱 +##腳 +##腴 +##腸 +##腹 +##腺 +##腻 +##腼 +##腾 +##腿 +##膀 +##膈 +##膊 +##膏 +##膑 +##膘 +##膚 +##膛 +##膜 +##膝 +##膠 +##膦 +##膨 +##膩 +##膳 +##膺 +##膻 +##膽 +##膾 +##膿 +##臀 +##臂 +##臃 +##臆 +##臉 +##臊 +##臍 +##臓 +##臘 +##臟 +##臣 +##臥 +##臧 +##臨 +##自 +##臬 +##臭 +##至 +##致 +##臺 +##臻 +##臼 +##臾 +##舀 +##舂 +##舅 +##舆 +##與 +##興 +##舉 +##舊 +##舌 +##舍 +##舎 +##舐 +##舒 +##舔 +##舖 +##舗 +##舛 +##舜 +##舞 +##舟 +##航 +##舫 +##般 +##舰 +##舱 +##舵 +##舶 +##舷 +##舸 +##船 +##舺 +##舾 +##艇 +##艋 +##艘 +##艙 +##艦 +##艮 +##良 +##艰 +##艱 +##色 +##艳 +##艷 +##艹 +##艺 +##艾 +##节 +##芃 +##芈 +##芊 +##芋 +##芍 +##芎 +##芒 +##芙 +##芜 +##芝 +##芡 +##芥 +##芦 +##芩 +##芪 +##芫 +##芬 +##芭 +##芮 +##芯 +##花 +##芳 +##芷 +##芸 +##芹 +##芻 +##芽 +##芾 +##苁 +##苄 +##苇 +##苋 +##苍 +##苏 +##苑 +##苒 +##苓 +##苔 +##苕 +##苗 +##苛 +##苜 +##苞 +##苟 +##苡 +##苣 +##若 +##苦 +##苫 +##苯 +##英 +##苷 +##苹 +##苻 +##茁 +##茂 +##范 +##茄 +##茅 +##茉 +##茎 +##茏 +##茗 +##茜 +##茧 +##茨 +##茫 +##茬 +##茭 +##茯 +##茱 +##茲 +##茴 +##茵 +##茶 +##茸 +##茹 +##茼 +##荀 +##荃 +##荆 +##草 +##荊 +##荏 +##荐 +##荒 +##荔 +##荖 +##荘 +##荚 +##荞 +##荟 +##荠 +##荡 +##荣 +##荤 +##荥 +##荧 +##荨 +##荪 +##荫 +##药 +##荳 +##荷 +##荸 +##荻 +##荼 +##荽 +##莅 +##莆 +##莉 +##莊 +##莎 +##莒 +##莓 +##莖 +##莘 +##莞 +##莠 +##莢 +##莧 +##莪 +##莫 +##莱 +##莲 +##莴 +##获 +##莹 +##莺 +##莽 +##莿 +##菀 +##菁 +##菅 +##菇 +##菈 +##菊 +##菌 +##菏 +##菓 +##菖 +##菘 +##菜 +##菟 +##菠 +##菡 +##菩 +##華 +##菱 +##菲 +##菸 +##菽 +##萁 +##萃 +##萄 +##萊 +##萋 +##萌 +##萍 +##萎 +##萘 +##萝 +##萤 +##营 +##萦 +##萧 +##萨 +##萩 +##萬 +##萱 +##萵 +##萸 +##萼 +##落 +##葆 +##葉 +##著 +##葚 +##葛 +##葡 +##董 +##葦 +##葩 +##葫 +##葬 +##葭 +##葯 +##葱 +##葳 +##葵 +##葷 +##葺 +##蒂 +##蒋 +##蒐 +##蒔 +##蒙 +##蒜 +##蒞 +##蒟 +##蒡 +##蒨 +##蒲 +##蒸 +##蒹 +##蒻 +##蒼 +##蒿 +##蓁 +##蓄 +##蓆 +##蓉 +##蓋 +##蓑 +##蓓 +##蓖 +##蓝 +##蓟 +##蓦 +##蓬 +##蓮 +##蓼 +##蓿 +##蔑 +##蔓 +##蔔 +##蔗 +##蔘 +##蔚 +##蔡 +##蔣 +##蔥 +##蔫 +##蔬 +##蔭 +##蔵 +##蔷 +##蔺 +##蔻 +##蔼 +##蔽 +##蕁 +##蕃 +##蕈 +##蕉 +##蕊 +##蕎 +##蕙 +##蕤 +##蕨 +##蕩 +##蕪 +##蕭 +##蕲 +##蕴 +##蕻 +##蕾 +##薄 +##薅 +##薇 +##薈 +##薊 +##薏 +##薑 +##薔 +##薙 +##薛 +##薦 +##薨 +##薩 +##薪 +##薬 +##薯 +##薰 +##薹 +##藉 +##藍 +##藏 +##藐 +##藓 +##藕 +##藜 +##藝 +##藤 +##藥 +##藩 +##藹 +##藻 +##藿 +##蘆 +##蘇 +##蘊 +##蘋 +##蘑 +##蘚 +##蘭 +##蘸 +##蘼 +##蘿 +##虎 +##虏 +##虐 +##虑 +##虔 +##處 +##虚 +##虛 +##虜 +##虞 +##號 +##虢 +##虧 +##虫 +##虬 +##虱 +##虹 +##虻 +##虽 +##虾 +##蚀 +##蚁 +##蚂 +##蚊 +##蚌 +##蚓 +##蚕 +##蚜 +##蚝 +##蚣 +##蚤 +##蚩 +##蚪 +##蚯 +##蚱 +##蚵 +##蛀 +##蛆 +##蛇 +##蛊 +##蛋 +##蛎 +##蛐 +##蛔 +##蛙 +##蛛 +##蛟 +##蛤 +##蛭 +##蛮 +##蛰 +##蛳 +##蛹 +##蛻 +##蛾 +##蜀 +##蜂 +##蜃 +##蜆 +##蜇 +##蜈 +##蜊 +##蜍 +##蜒 +##蜓 +##蜕 +##蜗 +##蜘 +##蜚 +##蜜 +##蜡 +##蜢 +##蜥 +##蜱 +##蜴 +##蜷 +##蜻 +##蜿 +##蝇 +##蝈 +##蝉 +##蝌 +##蝎 +##蝕 +##蝗 +##蝙 +##蝟 +##蝠 +##蝦 +##蝨 +##蝴 +##蝶 +##蝸 +##蝼 +##螂 +##螃 +##融 +##螞 +##螢 +##螨 +##螯 +##螳 +##螺 +##蟀 +##蟄 +##蟆 +##蟋 +##蟎 +##蟑 +##蟒 +##蟠 +##蟬 +##蟲 +##蟹 +##蟻 +##蟾 +##蠅 +##蠍 +##蠔 +##蠕 +##蠛 +##蠟 +##蠡 +##蠢 +##蠣 +##蠱 +##蠶 +##蠹 +##蠻 +##血 +##衄 +##衅 +##衆 +##行 +##衍 +##術 +##衔 +##街 +##衙 +##衛 +##衝 +##衞 +##衡 +##衢 +##衣 +##补 +##表 +##衩 +##衫 +##衬 +##衮 +##衰 +##衲 +##衷 +##衹 +##衾 +##衿 +##袁 +##袂 +##袄 +##袅 +##袈 +##袋 +##袍 +##袒 +##袖 +##袜 +##袞 +##袤 +##袪 +##被 +##袭 +##袱 +##裁 +##裂 +##装 +##裆 +##裊 +##裏 +##裔 +##裕 +##裘 +##裙 +##補 +##裝 +##裟 +##裡 +##裤 +##裨 +##裱 +##裳 +##裴 +##裸 +##裹 +##製 +##裾 +##褂 +##複 +##褐 +##褒 +##褓 +##褔 +##褚 +##褥 +##褪 +##褫 +##褲 +##褶 +##褻 +##襁 +##襄 +##襟 +##襠 +##襪 +##襬 +##襯 +##襲 +##西 +##要 +##覃 +##覆 +##覇 +##見 +##規 +##覓 +##視 +##覚 +##覦 +##覧 +##親 +##覬 +##観 +##覷 +##覺 +##覽 +##觀 +##见 +##观 +##规 +##觅 +##视 +##览 +##觉 +##觊 +##觎 +##觐 +##觑 +##角 +##觞 +##解 +##觥 +##触 +##觸 +##言 +##訂 +##計 +##訊 +##討 +##訓 +##訕 +##訖 +##託 +##記 +##訛 +##訝 +##訟 +##訣 +##訥 +##訪 +##設 +##許 +##訳 +##訴 +##訶 +##診 +##註 +##証 +##詆 +##詐 +##詔 +##評 +##詛 +##詞 +##詠 +##詡 +##詢 +##詣 +##試 +##詩 +##詫 +##詬 +##詭 +##詮 +##詰 +##話 +##該 +##詳 +##詹 +##詼 +##誅 +##誇 +##誉 +##誌 +##認 +##誓 +##誕 +##誘 +##語 +##誠 +##誡 +##誣 +##誤 +##誥 +##誦 +##誨 +##說 +##説 +##読 +##誰 +##課 +##誹 +##誼 +##調 +##諄 +##談 +##請 +##諏 +##諒 +##論 +##諗 +##諜 +##諡 +##諦 +##諧 +##諫 +##諭 +##諮 +##諱 +##諳 +##諷 +##諸 +##諺 +##諾 +##謀 +##謁 +##謂 +##謄 +##謊 +##謎 +##謐 +##謔 +##謗 +##謙 +##講 +##謝 +##謠 +##謨 +##謬 +##謹 +##謾 +##譁 +##證 +##譎 +##譏 +##識 +##譙 +##譚 +##譜 +##警 +##譬 +##譯 +##議 +##譲 +##譴 +##護 +##譽 +##讀 +##變 +##讓 +##讚 +##讞 +##计 +##订 +##认 +##讥 +##讧 +##讨 +##让 +##讪 +##讫 +##训 +##议 +##讯 +##记 +##讲 +##讳 +##讴 +##讶 +##讷 +##许 +##讹 +##论 +##讼 +##讽 +##设 +##访 +##诀 +##证 +##诃 +##评 +##诅 +##识 +##诈 +##诉 +##诊 +##诋 +##词 +##诏 +##译 +##试 +##诗 +##诘 +##诙 +##诚 +##诛 +##话 +##诞 +##诟 +##诠 +##诡 +##询 +##诣 +##诤 +##该 +##详 +##诧 +##诩 +##诫 +##诬 +##语 +##误 +##诰 +##诱 +##诲 +##说 +##诵 +##诶 +##请 +##诸 +##诺 +##读 +##诽 +##课 +##诿 +##谀 +##谁 +##调 +##谄 +##谅 +##谆 +##谈 +##谊 +##谋 +##谌 +##谍 +##谎 +##谏 +##谐 +##谑 +##谒 +##谓 +##谔 +##谕 +##谗 +##谘 +##谙 +##谚 +##谛 +##谜 +##谟 +##谢 +##谣 +##谤 +##谥 +##谦 +##谧 +##谨 +##谩 +##谪 +##谬 +##谭 +##谯 +##谱 +##谲 +##谴 +##谶 +##谷 +##豁 +##豆 +##豇 +##豈 +##豉 +##豊 +##豌 +##豎 +##豐 +##豔 +##豚 +##象 +##豢 +##豪 +##豫 +##豬 +##豹 +##豺 +##貂 +##貅 +##貌 +##貓 +##貔 +##貘 +##貝 +##貞 +##負 +##財 +##貢 +##貧 +##貨 +##販 +##貪 +##貫 +##責 +##貯 +##貰 +##貳 +##貴 +##貶 +##買 +##貸 +##費 +##貼 +##貽 +##貿 +##賀 +##賁 +##賂 +##賃 +##賄 +##資 +##賈 +##賊 +##賑 +##賓 +##賜 +##賞 +##賠 +##賡 +##賢 +##賣 +##賤 +##賦 +##質 +##賬 +##賭 +##賴 +##賺 +##購 +##賽 +##贅 +##贈 +##贊 +##贍 +##贏 +##贓 +##贖 +##贛 +##贝 +##贞 +##负 +##贡 +##财 +##责 +##贤 +##败 +##账 +##货 +##质 +##贩 +##贪 +##贫 +##贬 +##购 +##贮 +##贯 +##贰 +##贱 +##贲 +##贴 +##贵 +##贷 +##贸 +##费 +##贺 +##贻 +##贼 +##贾 +##贿 +##赁 +##赂 +##赃 +##资 +##赅 +##赈 +##赊 +##赋 +##赌 +##赎 +##赏 +##赐 +##赓 +##赔 +##赖 +##赘 +##赚 +##赛 +##赝 +##赞 +##赠 +##赡 +##赢 +##赣 +##赤 +##赦 +##赧 +##赫 +##赭 +##走 +##赳 +##赴 +##赵 +##赶 +##起 +##趁 +##超 +##越 +##趋 +##趕 +##趙 +##趟 +##趣 +##趨 +##足 +##趴 +##趵 +##趸 +##趺 +##趾 +##跃 +##跄 +##跆 +##跋 +##跌 +##跎 +##跑 +##跖 +##跚 +##跛 +##距 +##跟 +##跡 +##跤 +##跨 +##跩 +##跪 +##路 +##跳 +##践 +##跷 +##跹 +##跺 +##跻 +##踉 +##踊 +##踌 +##踏 +##踐 +##踝 +##踞 +##踟 +##踢 +##踩 +##踪 +##踮 +##踱 +##踴 +##踵 +##踹 +##蹂 +##蹄 +##蹇 +##蹈 +##蹉 +##蹊 +##蹋 +##蹑 +##蹒 +##蹙 +##蹟 +##蹣 +##蹤 +##蹦 +##蹩 +##蹬 +##蹭 +##蹲 +##蹴 +##蹶 +##蹺 +##蹼 +##蹿 +##躁 +##躇 +##躉 +##躊 +##躋 +##躍 +##躏 +##躪 +##身 +##躬 +##躯 +##躲 +##躺 +##軀 +##車 +##軋 +##軌 +##軍 +##軒 +##軟 +##転 +##軸 +##軼 +##軽 +##軾 +##較 +##載 +##輒 +##輓 +##輔 +##輕 +##輛 +##輝 +##輟 +##輩 +##輪 +##輯 +##輸 +##輻 +##輾 +##輿 +##轄 +##轅 +##轆 +##轉 +##轍 +##轎 +##轟 +##车 +##轧 +##轨 +##轩 +##转 +##轭 +##轮 +##软 +##轰 +##轲 +##轴 +##轶 +##轻 +##轼 +##载 +##轿 +##较 +##辄 +##辅 +##辆 +##辇 +##辈 +##辉 +##辊 +##辍 +##辐 +##辑 +##输 +##辕 +##辖 +##辗 +##辘 +##辙 +##辛 +##辜 +##辞 +##辟 +##辣 +##辦 +##辨 +##辩 +##辫 +##辭 +##辮 +##辯 +##辰 +##辱 +##農 +##边 +##辺 +##辻 +##込 +##辽 +##达 +##迁 +##迂 +##迄 +##迅 +##过 +##迈 +##迎 +##运 +##近 +##返 +##还 +##这 +##进 +##远 +##违 +##连 +##迟 +##迢 +##迤 +##迥 +##迦 +##迩 +##迪 +##迫 +##迭 +##述 +##迴 +##迷 +##迸 +##迹 +##迺 +##追 +##退 +##送 +##适 +##逃 +##逅 +##逆 +##选 +##逊 +##逍 +##透 +##逐 +##递 +##途 +##逕 +##逗 +##這 +##通 +##逛 +##逝 +##逞 +##速 +##造 +##逢 +##連 +##逮 +##週 +##進 +##逵 +##逶 +##逸 +##逻 +##逼 +##逾 +##遁 +##遂 +##遅 +##遇 +##遊 +##運 +##遍 +##過 +##遏 +##遐 +##遑 +##遒 +##道 +##達 +##違 +##遗 +##遙 +##遛 +##遜 +##遞 +##遠 +##遢 +##遣 +##遥 +##遨 +##適 +##遭 +##遮 +##遲 +##遴 +##遵 +##遶 +##遷 +##選 +##遺 +##遼 +##遽 +##避 +##邀 +##邁 +##邂 +##邃 +##還 +##邇 +##邈 +##邊 +##邋 +##邏 +##邑 +##邓 +##邕 +##邛 +##邝 +##邢 +##那 +##邦 +##邨 +##邪 +##邬 +##邮 +##邯 +##邰 +##邱 +##邳 +##邵 +##邸 +##邹 +##邺 +##邻 +##郁 +##郅 +##郊 +##郎 +##郑 +##郜 +##郝 +##郡 +##郢 +##郤 +##郦 +##郧 +##部 +##郫 +##郭 +##郴 +##郵 +##郷 +##郸 +##都 +##鄂 +##鄉 +##鄒 +##鄔 +##鄙 +##鄞 +##鄢 +##鄧 +##鄭 +##鄰 +##鄱 +##鄲 +##鄺 +##酉 +##酊 +##酋 +##酌 +##配 +##酐 +##酒 +##酗 +##酚 +##酝 +##酢 +##酣 +##酥 +##酩 +##酪 +##酬 +##酮 +##酯 +##酰 +##酱 +##酵 +##酶 +##酷 +##酸 +##酿 +##醃 +##醇 +##醉 +##醋 +##醍 +##醐 +##醒 +##醚 +##醛 +##醜 +##醞 +##醣 +##醪 +##醫 +##醬 +##醮 +##醯 +##醴 +##醺 +##釀 +##釁 +##采 +##釉 +##释 +##釋 +##里 +##重 +##野 +##量 +##釐 +##金 +##釗 +##釘 +##釜 +##針 +##釣 +##釦 +##釧 +##釵 +##鈀 +##鈉 +##鈍 +##鈎 +##鈔 +##鈕 +##鈞 +##鈣 +##鈦 +##鈪 +##鈴 +##鈺 +##鈾 +##鉀 +##鉄 +##鉅 +##鉉 +##鉑 +##鉗 +##鉚 +##鉛 +##鉤 +##鉴 +##鉻 +##銀 +##銃 +##銅 +##銑 +##銓 +##銖 +##銘 +##銜 +##銬 +##銭 +##銮 +##銳 +##銷 +##銹 +##鋁 +##鋅 +##鋒 +##鋤 +##鋪 +##鋰 +##鋸 +##鋼 +##錄 +##錐 +##錘 +##錚 +##錠 +##錢 +##錦 +##錨 +##錫 +##錮 +##錯 +##録 +##錳 +##錶 +##鍊 +##鍋 +##鍍 +##鍛 +##鍥 +##鍰 +##鍵 +##鍺 +##鍾 +##鎂 +##鎊 +##鎌 +##鎏 +##鎔 +##鎖 +##鎗 +##鎚 +##鎧 +##鎬 +##鎮 +##鎳 +##鏈 +##鏖 +##鏗 +##鏘 +##鏞 +##鏟 +##鏡 +##鏢 +##鏤 +##鏽 +##鐘 +##鐮 +##鐲 +##鐳 +##鐵 +##鐸 +##鐺 +##鑄 +##鑊 +##鑑 +##鑒 +##鑣 +##鑫 +##鑰 +##鑲 +##鑼 +##鑽 +##鑾 +##鑿 +##针 +##钉 +##钊 +##钎 +##钏 +##钒 +##钓 +##钗 +##钙 +##钛 +##钜 +##钝 +##钞 +##钟 +##钠 +##钡 +##钢 +##钣 +##钤 +##钥 +##钦 +##钧 +##钨 +##钩 +##钮 +##钯 +##钰 +##钱 +##钳 +##钴 +##钵 +##钺 +##钻 +##钼 +##钾 +##钿 +##铀 +##铁 +##铂 +##铃 +##铄 +##铅 +##铆 +##铉 +##铎 +##铐 +##铛 +##铜 +##铝 +##铠 +##铡 +##铢 +##铣 +##铤 +##铨 +##铩 +##铬 +##铭 +##铮 +##铰 +##铲 +##铵 +##银 +##铸 +##铺 +##链 +##铿 +##销 +##锁 +##锂 +##锄 +##锅 +##锆 +##锈 +##锉 +##锋 +##锌 +##锏 +##锐 +##锑 +##错 +##锚 +##锟 +##锡 +##锢 +##锣 +##锤 +##锥 +##锦 +##锭 +##键 +##锯 +##锰 +##锲 +##锵 +##锹 +##锺 +##锻 +##镀 +##镁 +##镂 +##镇 +##镉 +##镌 +##镍 +##镐 +##镑 +##镕 +##镖 +##镗 +##镛 +##镜 +##镣 +##镭 +##镯 +##镰 +##镳 +##镶 +##長 +##长 +##門 +##閃 +##閉 +##開 +##閎 +##閏 +##閑 +##閒 +##間 +##閔 +##閘 +##閡 +##関 +##閣 +##閥 +##閨 +##閩 +##閱 +##閲 +##閹 +##閻 +##閾 +##闆 +##闇 +##闊 +##闌 +##闍 +##闔 +##闕 +##闖 +##闘 +##關 +##闡 +##闢 +##门 +##闪 +##闫 +##闭 +##问 +##闯 +##闰 +##闲 +##间 +##闵 +##闷 +##闸 +##闹 +##闺 +##闻 +##闽 +##闾 +##阀 +##阁 +##阂 +##阅 +##阆 +##阇 +##阈 +##阉 +##阎 +##阐 +##阑 +##阔 +##阕 +##阖 +##阙 +##阚 +##阜 +##队 +##阡 +##阪 +##阮 +##阱 +##防 +##阳 +##阴 +##阵 +##阶 +##阻 +##阿 +##陀 +##陂 +##附 +##际 +##陆 +##陇 +##陈 +##陋 +##陌 +##降 +##限 +##陕 +##陛 +##陝 +##陞 +##陟 +##陡 +##院 +##陣 +##除 +##陨 +##险 +##陪 +##陰 +##陲 +##陳 +##陵 +##陶 +##陷 +##陸 +##険 +##陽 +##隅 +##隆 +##隈 +##隊 +##隋 +##隍 +##階 +##随 +##隐 +##隔 +##隕 +##隘 +##隙 +##際 +##障 +##隠 +##隣 +##隧 +##隨 +##險 +##隱 +##隴 +##隶 +##隸 +##隻 +##隼 +##隽 +##难 +##雀 +##雁 +##雄 +##雅 +##集 +##雇 +##雉 +##雋 +##雌 +##雍 +##雎 +##雏 +##雑 +##雒 +##雕 +##雖 +##雙 +##雛 +##雜 +##雞 +##離 +##難 +##雨 +##雪 +##雯 +##雰 +##雲 +##雳 +##零 +##雷 +##雹 +##電 +##雾 +##需 +##霁 +##霄 +##霆 +##震 +##霈 +##霉 +##霊 +##霍 +##霎 +##霏 +##霑 +##霓 +##霖 +##霜 +##霞 +##霧 +##霭 +##霰 +##露 +##霸 +##霹 +##霽 +##霾 +##靂 +##靄 +##靈 +##青 +##靓 +##靖 +##静 +##靚 +##靛 +##靜 +##非 +##靠 +##靡 +##面 +##靥 +##靦 +##革 +##靳 +##靴 +##靶 +##靼 +##鞅 +##鞋 +##鞍 +##鞏 +##鞑 +##鞘 +##鞠 +##鞣 +##鞦 +##鞭 +##韆 +##韋 +##韌 +##韓 +##韜 +##韦 +##韧 +##韩 +##韬 +##韭 +##音 +##韵 +##韶 +##韻 +##響 +##頁 +##頂 +##頃 +##項 +##順 +##須 +##頌 +##預 +##頑 +##頒 +##頓 +##頗 +##領 +##頜 +##頡 +##頤 +##頫 +##頭 +##頰 +##頷 +##頸 +##頹 +##頻 +##頼 +##顆 +##題 +##額 +##顎 +##顏 +##顔 +##願 +##顛 +##類 +##顧 +##顫 +##顯 +##顱 +##顴 +##页 +##顶 +##顷 +##项 +##顺 +##须 +##顼 +##顽 +##顾 +##顿 +##颁 +##颂 +##预 +##颅 +##领 +##颇 +##颈 +##颉 +##颊 +##颌 +##颍 +##颐 +##频 +##颓 +##颔 +##颖 +##颗 +##题 +##颚 +##颛 +##颜 +##额 +##颞 +##颠 +##颡 +##颢 +##颤 +##颦 +##颧 +##風 +##颯 +##颱 +##颳 +##颶 +##颼 +##飄 +##飆 +##风 +##飒 +##飓 +##飕 +##飘 +##飙 +##飚 +##飛 +##飞 +##食 +##飢 +##飨 +##飩 +##飪 +##飯 +##飲 +##飼 +##飽 +##飾 +##餃 +##餅 +##餉 +##養 +##餌 +##餐 +##餒 +##餓 +##餘 +##餚 +##餛 +##餞 +##餡 +##館 +##餮 +##餵 +##餾 +##饅 +##饈 +##饋 +##饌 +##饍 +##饑 +##饒 +##饕 +##饗 +##饞 +##饥 +##饨 +##饪 +##饬 +##饭 +##饮 +##饯 +##饰 +##饱 +##饲 +##饴 +##饵 +##饶 +##饷 +##饺 +##饼 +##饽 +##饿 +##馀 +##馁 +##馄 +##馅 +##馆 +##馈 +##馋 +##馍 +##馏 +##馒 +##馔 +##首 +##馗 +##香 +##馥 +##馨 +##馬 +##馭 +##馮 +##馳 +##馴 +##駁 +##駄 +##駅 +##駆 +##駐 +##駒 +##駕 +##駛 +##駝 +##駭 +##駱 +##駿 +##騁 +##騎 +##騏 +##験 +##騙 +##騨 +##騰 +##騷 +##驀 +##驅 +##驊 +##驍 +##驒 +##驕 +##驗 +##驚 +##驛 +##驟 +##驢 +##驥 +##马 +##驭 +##驮 +##驯 +##驰 +##驱 +##驳 +##驴 +##驶 +##驷 +##驸 +##驹 +##驻 +##驼 +##驾 +##驿 +##骁 +##骂 +##骄 +##骅 +##骆 +##骇 +##骈 +##骊 +##骋 +##验 +##骏 +##骐 +##骑 +##骗 +##骚 +##骛 +##骜 +##骞 +##骠 +##骡 +##骤 +##骥 +##骧 +##骨 +##骯 +##骰 +##骶 +##骷 +##骸 +##骼 +##髂 +##髅 +##髋 +##髏 +##髒 +##髓 +##體 +##髖 +##高 +##髦 +##髪 +##髮 +##髯 +##髻 +##鬃 +##鬆 +##鬍 +##鬓 +##鬚 +##鬟 +##鬢 +##鬣 +##鬥 +##鬧 +##鬱 +##鬼 +##魁 +##魂 +##魄 +##魅 +##魇 +##魍 +##魏 +##魔 +##魘 +##魚 +##魯 +##魷 +##鮑 +##鮨 +##鮪 +##鮭 +##鮮 +##鯉 +##鯊 +##鯖 +##鯛 +##鯨 +##鯰 +##鯽 +##鰍 +##鰓 +##鰭 +##鰲 +##鰻 +##鰾 +##鱈 +##鱉 +##鱔 +##鱗 +##鱷 +##鱸 +##鱼 +##鱿 +##鲁 +##鲈 +##鲍 +##鲑 +##鲛 +##鲜 +##鲟 +##鲢 +##鲤 +##鲨 +##鲫 +##鲱 +##鲲 +##鲶 +##鲷 +##鲸 +##鳃 +##鳄 +##鳅 +##鳌 +##鳍 +##鳕 +##鳖 +##鳗 +##鳝 +##鳞 +##鳥 +##鳩 +##鳳 +##鳴 +##鳶 +##鴉 +##鴕 +##鴛 +##鴦 +##鴨 +##鴻 +##鴿 +##鵑 +##鵜 +##鵝 +##鵡 +##鵬 +##鵰 +##鵲 +##鶘 +##鶩 +##鶯 +##鶴 +##鷗 +##鷲 +##鷹 +##鷺 +##鸚 +##鸞 +##鸟 +##鸠 +##鸡 +##鸢 +##鸣 +##鸥 +##鸦 +##鸨 +##鸪 +##鸭 +##鸯 +##鸳 +##鸵 +##鸽 +##鸾 +##鸿 +##鹂 +##鹃 +##鹄 +##鹅 +##鹈 +##鹉 +##鹊 +##鹌 +##鹏 +##鹑 +##鹕 +##鹘 +##鹜 +##鹞 +##鹤 +##鹦 +##鹧 +##鹫 +##鹭 +##鹰 +##鹳 +##鹵 +##鹹 +##鹼 +##鹽 +##鹿 +##麂 +##麋 +##麒 +##麓 +##麗 +##麝 +##麟 +##麥 +##麦 +##麩 +##麴 +##麵 +##麸 +##麺 +##麻 +##麼 +##麽 +##麾 +##黃 +##黄 +##黍 +##黎 +##黏 +##黑 +##黒 +##黔 +##默 +##黛 +##黜 +##黝 +##點 +##黠 +##黨 +##黯 +##黴 +##鼋 +##鼎 +##鼐 +##鼓 +##鼠 +##鼬 +##鼹 +##鼻 +##鼾 +##齁 +##齊 +##齋 +##齐 +##齒 +##齡 +##齢 +##齣 +##齦 +##齿 +##龄 +##龅 +##龈 +##龊 +##龋 +##龌 +##龍 +##龐 +##龔 +##龕 +##龙 +##龚 +##龛 +##龜 +##龟 +##︰ +##︱ +##︶ +##︿ +##﹁ +##﹂ +##﹍ +##﹏ +##﹐ +##﹑ +##﹒ +##﹔ +##﹕ +##﹖ +##﹗ +##﹙ +##﹚ +##﹝ +##﹞ +##﹡ +##﹣ +##！ +##＂ +##＃ +##＄ +##％ +##＆ +##＇ +##（ +##） +##＊ +##， +##－ +##． +##／ +##： +##； +##＜ +##？ +##＠ +##［ +##＼ +##］ +##＾ +##＿ +##｀ +##ｆ +##ｈ +##ｊ +##ｕ +##ｗ +##ｚ +##｛ +##｝ +##｡ +##｢ +##｣ +##､ +##･ +##ｯ +##ｰ +##ｲ +##ｸ +##ｼ +##ｽ +##ﾄ +##ﾉ +##ﾌ +##ﾗ +##ﾙ +##ﾝ +##ﾞ +##ﾟ +##￣ +##￥ +##👍 +##🔥 +##😂 +##😎 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/config.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a9f8cc780dd2b8a4cd8af99b90263c3dc41a842a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/config.json @@ -0,0 +1,28 @@ +{ + "_name_or_path": "/home/patrick/t5/mt5-xl", + "architectures": [ + "MT5ForConditionalGeneration" + ], + "d_ff": 5120, + "d_kv": 64, + "d_model": 2048, + "decoder_start_token_id": 0, + "dropout_rate": 0.1, + "eos_token_id": 1, + "feed_forward_proj": "gated-gelu", + "initializer_factor": 1.0, + "is_encoder_decoder": true, + "layer_norm_epsilon": 1e-06, + "model_type": "mt5", + "num_decoder_layers": 24, + "num_heads": 32, + "num_layers": 24, + "output_past": true, + "pad_token_id": 0, + "relative_attention_num_buckets": 32, + "tie_word_embeddings": false, + "tokenizer_class": "T5Tokenizer", + "transformers_version": "4.10.0.dev0", + "use_cache": true, + "vocab_size": 250112 +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/special_tokens_map.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..6dc4d430ddbd24171268d73da061ce9f0b092911 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/special_tokens_map.json @@ -0,0 +1 @@ +{"eos_token": "", "unk_token": "", "pad_token": ""} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/tokenizer_config.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..712e82c8fde9e07d534178c2dfbdd063572d00c4 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/tokenizer_mt5/tokenizer_config.json @@ -0,0 +1 @@ +{"eos_token": "", "unk_token": "", "pad_token": "", "extra_ids": 0, "additional_special_tokens": null, "special_tokens_map_file": "/home/patrick/.cache/torch/transformers/685ac0ca8568ec593a48b61b0a3c272beee9bc194a3c7241d15dcadb5f875e53.f76030f3ec1b96a8199b2593390c610e76ca8028ef3d24680000619ffb646276", "tokenizer_file": null, "name_or_path": "google/mt5-small"} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/utils.py b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b41cb3520316350d273c12a87769ac3e4b4aeef0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/utils.py @@ -0,0 +1,215 @@ +import io +import torch +import requests +import numpy as np +from PIL import Image +from omegaconf import OmegaConf +from torchvision.transforms import ToTensor +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import ( + assign_to_checkpoint, + conv_attn_to_linear, + create_vae_diffusers_config, + renew_vae_attention_paths, + renew_vae_resnet_paths, +) +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DDPMScheduler, + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + KDPM2AncestralDiscreteScheduler, + KDPM2DiscreteScheduler, + UniPCMultistepScheduler, +) + +SCHEDULERS = { + 'DDIM' : DDIMScheduler, + 'DDPM' : DDPMScheduler, + 'DEISMultistep' : DEISMultistepScheduler, + 'DPMSolverMultistep' : DPMSolverMultistepScheduler, + 'DPMSolverSinglestep' : DPMSolverSinglestepScheduler, + 'EulerAncestralDiscrete' : EulerAncestralDiscreteScheduler, + 'EulerDiscrete' : EulerDiscreteScheduler, + 'HeunDiscrete' : HeunDiscreteScheduler, + 'KDPM2AncestralDiscrete' : KDPM2AncestralDiscreteScheduler, + 'KDPM2Discrete' : KDPM2DiscreteScheduler, + 'UniPCMultistep' : UniPCMultistepScheduler +} + +SCHEDULERS_hunyuan = ["ddpm", "ddim", "dpmms"] + +def token_auto_concat_embeds(pipe, positive, negative): + max_length = pipe.tokenizer.model_max_length + positive_length = pipe.tokenizer(positive, return_tensors="pt").input_ids.shape[-1] + negative_length = pipe.tokenizer(negative, return_tensors="pt").input_ids.shape[-1] + + print(f'Token length is model maximum: {max_length}, positive length: {positive_length}, negative length: {negative_length}.') + if max_length < positive_length or max_length < negative_length: + print('Concatenated embedding.') + if positive_length > negative_length: + positive_ids = pipe.tokenizer(positive, return_tensors="pt").input_ids.to("cuda") + negative_ids = pipe.tokenizer(negative, truncation=False, padding="max_length", max_length=positive_ids.shape[-1], return_tensors="pt").input_ids.to("cuda") + else: + negative_ids = pipe.tokenizer(negative, return_tensors="pt").input_ids.to("cuda") + positive_ids = pipe.tokenizer(positive, truncation=False, padding="max_length", max_length=negative_ids.shape[-1], return_tensors="pt").input_ids.to("cuda") + else: + positive_ids = pipe.tokenizer(positive, truncation=False, padding="max_length", max_length=max_length, return_tensors="pt").input_ids.to("cuda") + negative_ids = pipe.tokenizer(negative, truncation=False, padding="max_length", max_length=max_length, return_tensors="pt").input_ids.to("cuda") + + positive_concat_embeds = [] + negative_concat_embeds = [] + for i in range(0, positive_ids.shape[-1], max_length): + positive_concat_embeds.append(pipe.text_encoder(positive_ids[:, i: i + max_length])[0]) + negative_concat_embeds.append(pipe.text_encoder(negative_ids[:, i: i + max_length])[0]) + + positive_prompt_embeds = torch.cat(positive_concat_embeds, dim=1) + negative_prompt_embeds = torch.cat(negative_concat_embeds, dim=1) + return positive_prompt_embeds, negative_prompt_embeds + +# Reference from : https://github.com/huggingface/diffusers/blob/main/scripts/convert_vae_pt_to_diffusers.py +def custom_convert_ldm_vae_checkpoint(checkpoint, config): + vae_state_dict = checkpoint + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + +# Reference from : https://github.com/huggingface/diffusers/blob/main/scripts/convert_vae_pt_to_diffusers.py +def vae_pt_to_vae_diffuser( + checkpoint_path: str, + output_path: str, +): + # Only support V1 + r = requests.get( + " https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml" + ) + io_obj = io.BytesIO(r.content) + + original_config = OmegaConf.load(io_obj) + image_size = 512 + device = "cuda" if torch.cuda.is_available() else "cpu" + if checkpoint_path.endswith("safetensors"): + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + checkpoint = torch.load(checkpoint_path, map_location=device)["state_dict"] + + # Convert the VAE model. + vae_config = create_vae_diffusers_config(original_config, image_size=image_size) + converted_vae_checkpoint = custom_convert_ldm_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + vae.save_pretrained(output_path) + + +def convert_images_to_tensors(images: list[Image.Image]): + return torch.stack([np.transpose(ToTensor()(image), (1, 2, 0)) for image in images]) + +def convert_tensors_to_images(images: torch.tensor): + return [Image.fromarray(np.clip(255. * image.cpu().numpy(), 0, 255).astype(np.uint8)) for image in images] + +def resize_images(images: list[Image.Image], size: tuple[int, int]): + return [image.resize(size) for image in images] \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_diffusers.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_diffusers.json new file mode 100644 index 0000000000000000000000000000000000000000..dc0c58f37a7bcc759309d61b9e7b3991a965ce25 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_diffusers.json @@ -0,0 +1,307 @@ +{ + "last_node_id": 30, + "last_link_id": 17, + "nodes": [ + { + "id": 16, + "type": "DiffusersModelMakeup", + "pos": [ + 600, + 130 + ], + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "PIPELINE", + "link": 2 + }, + { + "name": "scheduler", + "type": "SCHEDULER", + "link": 3 + }, + { + "name": "controlnet", + "type": "CTRL", + "link": null + } + ], + "outputs": [ + { + "name": "MAKED_PIPELINE", + "type": "MAKED_PIPELINE", + "links": [ + 4 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersModelMakeup" + } + }, + { + "id": 18, + "type": "DiffusersSampler", + "pos": [ + 741, + 256 + ], + "size": { + "0": 315, + "1": 286 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "maked_pipeline", + "type": "MAKED_PIPELINE", + "link": 4 + }, + { + "name": "positive", + "type": "STRINGC", + "link": 5 + }, + { + "name": "negative", + "type": "STRINGC", + "link": 6 + }, + { + "name": "image", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersSampler" + }, + "widgets_values": [ + 1, + 1024, + 1024, + 30, + 1, + 5.99, + 214826175, + "fixed" + ] + }, + { + "id": 15, + "type": "DiffusersSchedulerLoader", + "pos": [ + 229, + 139 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "SCHEDULER", + "type": "SCHEDULER", + "links": [ + 3 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersSchedulerLoader" + }, + "widgets_values": [ + "ddim" + ] + }, + { + "id": 19, + "type": "DiffusersClipTextEncode", + "pos": [ + 299, + 265 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "positive", + "type": "STRINGC", + "links": [ + 5 + ], + "shape": 3 + }, + { + "name": "negative", + "type": "STRINGC", + "links": [ + 6 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersClipTextEncode" + }, + "widgets_values": [ + "一只小狗在追逐蝴蝶。", + "左右，四宫格，九宫格，错误的眼睛，糟糕的人脸，裸体，画布框架，毁容，糟糕的艺术，变形，多余的肢体，特写，黑白，模糊的颜色，模糊，重复，病态，残缺，框架" + ] + }, + { + "id": 6, + "type": "PreviewImage", + "pos": [ + 1088, + 137 + ], + "size": [ + 334.9803761718754, + 427.6778060729981 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 1 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 21, + "type": "DiffusersPipelineLoader", + "pos": [ + 346, + 510 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "lora", + "type": "lora_path", + "link": null + } + ], + "outputs": [ + { + "name": "PIPELINE", + "type": "PIPELINE", + "links": [ + 2 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersPipelineLoader" + }, + "widgets_values": [ + "ckpts" + ] + } + ], + "links": [ + [ + 1, + 18, + 0, + 6, + 0, + "IMAGE" + ], + [ + 2, + 21, + 0, + 16, + 0, + "PIPELINE" + ], + [ + 3, + 15, + 0, + 16, + 1, + "SCHEDULER" + ], + [ + 4, + 16, + 0, + 18, + 0, + "MAKED_PIPELINE" + ], + [ + 5, + 19, + 0, + 18, + 1, + "STRINGC" + ], + [ + 6, + 19, + 1, + 18, + 2, + "STRINGC" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650706, + "offset": { + "0": 652.930419921875, + "1": 130.0147705078125 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_ksampler.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_ksampler.json new file mode 100644 index 0000000000000000000000000000000000000000..e0e8a9fae4035728d5c17200c51ca1a2739af88f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_ksampler.json @@ -0,0 +1,424 @@ +{ + "last_node_id": 33, + "last_link_id": 9, + "nodes": [ + { + "id": 33, + "type": "DiffusersCLIPLoader", + "pos": [ + -77, + 221 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 8, + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DiffusersCLIPLoader" + }, + "widgets_values": [ + "pytorch_model.bin", + "pytorch_model.bin" + ] + }, + { + "id": 25, + "type": "CLIPTextEncode", + "pos": [ + 260, + 108 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 8 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 1 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "描绘的风格是写实，画面主要描述一双泥泞的靴子在雨天里，靴子颜色是棕色，泥沙溅在 Boot 的表面，背景是湿漉漉的地板，泥泞的环境，天气是阴沉的雨天，镜头是近景" + ] + }, + { + "id": 32, + "type": "DiffusersVAELoader", + "pos": [ + 359, + 498 + ], + "size": { + "0": 303.9019775390625, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "VAE", + "type": "VAE", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DiffusersVAELoader" + }, + "widgets_values": [ + "diffusion_pytorch_model.bin" + ] + }, + { + "id": 26, + "type": "CLIPTextEncode", + "pos": [ + -80, + 356 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 9 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 2 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "错误的眼睛，糟糕的人脸，毁容，糟糕的艺术，变形，多余的肢体，模糊的颜色，模糊，重复，病态，残缺，" + ] + }, + { + "id": 28, + "type": "VAEDecode", + "pos": [ + 731, + 159 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 4 + }, + { + "name": "vae", + "type": "VAE", + "link": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 5 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 24, + "type": "KSampler", + "pos": [ + 681, + 290 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 6, + "slot_index": 0 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 1 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 2 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 3 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 4 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 123, + "fixed", + 30, + 6, + "ddim", + "ddim_uniform", + 1 + ] + }, + { + "id": 31, + "type": "DiffusersCheckpointLoader", + "pos": [ + -79, + 107 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 6 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DiffusersCheckpointLoader" + }, + "widgets_values": [ + "pytorch_model_distill.pt" + ] + }, + { + "id": 27, + "type": "EmptyLatentImage", + "pos": [ + 351, + 350 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 3 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 1280, + 2 + ] + }, + { + "id": 29, + "type": "PreviewImage", + "pos": [ + 1031, + 175 + ], + "size": [ + 357.9370410924005, + 264.0938661488955 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 5 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + } + ], + "links": [ + [ + 1, + 25, + 0, + 24, + 1, + "CONDITIONING" + ], + [ + 2, + 26, + 0, + 24, + 2, + "CONDITIONING" + ], + [ + 3, + 27, + 0, + 24, + 3, + "LATENT" + ], + [ + 4, + 24, + 0, + 28, + 0, + "LATENT" + ], + [ + 5, + 28, + 0, + 29, + 0, + "IMAGE" + ], + [ + 6, + 31, + 0, + 24, + 0, + "MODEL" + ], + [ + 7, + 32, + 0, + 28, + 1, + "VAE" + ], + [ + 8, + 33, + 0, + 25, + 0, + "CLIP" + ], + [ + 9, + 33, + 0, + 26, + 0, + "CLIP" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.8954302432552415, + "offset": [ + 370.7466007037628, + 90.3483300454219 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_lora_controlnet.json b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_lora_controlnet.json new file mode 100644 index 0000000000000000000000000000000000000000..160c01c95ad32d289f77a774d37f1cd4a11a4881 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/comfyui-hydit/workflow/workflow_lora_controlnet.json @@ -0,0 +1,516 @@ +{ + "last_node_id": 30, + "last_link_id": 16, + "nodes": [ + { + "id": 16, + "type": "DiffusersModelMakeup", + "pos": [ + 600, + 130 + ], + "size": { + "0": 304.79998779296875, + "1": 66 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "pipeline", + "type": "PIPELINE", + "link": 2 + }, + { + "name": "scheduler", + "type": "SCHEDULER", + "link": 3 + }, + { + "name": "controlnet", + "type": "CTRL", + "link": 9 + } + ], + "outputs": [ + { + "name": "MAKED_PIPELINE", + "type": "MAKED_PIPELINE", + "links": [ + 4 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersModelMakeup" + } + }, + { + "id": 15, + "type": "DiffusersSchedulerLoader", + "pos": [ + 49, + 105 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "SCHEDULER", + "type": "SCHEDULER", + "links": [ + 3 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersSchedulerLoader" + }, + "widgets_values": [ + "ddim" + ] + }, + { + "id": 19, + "type": "DiffusersClipTextEncode", + "pos": [ + -41, + 233 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "positive", + "type": "STRINGC", + "links": [ + 5 + ], + "shape": 3 + }, + { + "name": "negative", + "type": "STRINGC", + "links": [ + 6 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersClipTextEncode" + }, + "widgets_values": [ + "玉石风格，一只小狗在追逐蝴蝶。", + "左右，四宫格，九宫格，错误的眼睛，糟糕的人脸，裸体，画布框架，毁容，糟糕的艺术，变形，多余的肢体，特写，黑白，模糊的颜色，模糊，重复，病态，残缺，框架" + ] + }, + { + "id": 25, + "type": "DiffusersControlNetLoader", + "pos": [ + 40, + 488 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "CTRL", + "type": "CTRL", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DiffusersControlNetLoader" + }, + "widgets_values": [ + "pytorch_model_canny_distill.pt" + ] + }, + { + "id": 21, + "type": "DiffusersPipelineLoader", + "pos": [ + 30, + 597 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "lora", + "type": "lora_path", + "link": 15 + } + ], + "outputs": [ + { + "name": "PIPELINE", + "type": "PIPELINE", + "links": [ + 2 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersPipelineLoader" + }, + "widgets_values": [ + "ckpts" + ] + }, + { + "id": 23, + "type": "LoadImage", + "pos": [ + 373, + 254 + ], + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ComfyUI_temp_calve_00004_.png", + "image" + ] + }, + { + "id": 6, + "type": "PreviewImage", + "pos": [ + 1086, + 132 + ], + "size": [ + 253.2063696289065, + 295.8073356216431 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 1 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 18, + "type": "DiffusersSampler", + "pos": [ + 741, + 256 + ], + "size": { + "0": 315, + "1": 286 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "maked_pipeline", + "type": "MAKED_PIPELINE", + "link": 4 + }, + { + "name": "positive", + "type": "STRINGC", + "link": 5 + }, + { + "name": "negative", + "type": "STRINGC", + "link": 6 + }, + { + "name": "image", + "type": "IMAGE", + "link": 12 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "DiffusersSampler" + }, + "widgets_values": [ + 1, + 1024, + 1024, + 30, + 1, + 5.99, + 214826175, + "fixed" + ] + }, + { + "id": 26, + "type": "Canny", + "pos": [ + 429, + 630 + ], + "size": { + "0": 315, + "1": 82 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": 10 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 12, + 16 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Canny" + }, + "widgets_values": [ + 0.2, + 0.5 + ] + }, + { + "id": 30, + "type": "PreviewImage", + "pos": [ + 1095, + 478 + ], + "size": [ + 240.53468134765626, + 248.44370163726808 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 16 + } + ], + "properties": { + "Node name for S&R": "PreviewImage" + } + }, + { + "id": 22, + "type": "DiffusersLoraLoader", + "pos": [ + -312, + 484 + ], + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "lora_path", + "type": "lora_path", + "links": [ + 15 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DiffusersLoraLoader" + }, + "widgets_values": [ + "jade" + ] + } + ], + "links": [ + [ + 1, + 18, + 0, + 6, + 0, + "IMAGE" + ], + [ + 2, + 21, + 0, + 16, + 0, + "PIPELINE" + ], + [ + 3, + 15, + 0, + 16, + 1, + "SCHEDULER" + ], + [ + 4, + 16, + 0, + 18, + 0, + "MAKED_PIPELINE" + ], + [ + 5, + 19, + 0, + 18, + 1, + "STRINGC" + ], + [ + 6, + 19, + 1, + 18, + 2, + "STRINGC" + ], + [ + 9, + 25, + 0, + 16, + 2, + "CTRL" + ], + [ + 10, + 23, + 0, + 26, + 0, + "IMAGE" + ], + [ + 12, + 26, + 0, + 18, + 3, + "IMAGE" + ], + [ + 15, + 22, + 0, + 21, + 0, + "lora_path" + ], + [ + 16, + 26, + 0, + 30, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650706, + "offset": { + "0": 893.4832153320312, + "1": 18.211286544799805 + } + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/controlnet/README.md b/PyTorch/built-in/mlm/HunyuanDiT/controlnet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c59dae511b6d2b2c3739dedd460e6d045dd624e8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/controlnet/README.md @@ -0,0 +1,171 @@ + +## Using HunyuanDiT ControlNet + + +### Instructions + + The dependencies and installation are basically the same as the [**base model**](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1). + + We provide three types of ControlNet weights for you to test: canny, depth and pose ControlNet. + + Download the model using the following commands: + +```bash +cd HunyuanDiT +# Use the huggingface-cli tool to download the model. +# We recommend using distilled weights as the base model for ControlNet inference, as our provided pretrained weights are trained on them. +huggingface-cli download Tencent-Hunyuan/HYDiT-ControlNet --local-dir ./ckpts/t2i/controlnet +huggingface-cli download Tencent-Hunyuan/Distillation-v1.1 ./pytorch_model_distill.pt --local-dir ./ckpts/t2i/model + +# Quick start +python3 sample_controlnet.py --no-enhance --load-key distill --infer-steps 50 --control-type canny --prompt "在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围" --condition-image-path controlnet/asset/input/canny.jpg --control-weight 1.0 +``` + +Examples of condition input and ControlNet results are as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Condition Input
Canny ControlNet Depth ControlNet Pose ControlNet
在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围
（At night, an ancient Chinese-style lion statue stands in front of the hotel, its eyes gleaming as if guarding the building. The background is the hotel entrance at night, with a close-up, eye-level, and centered composition. This photo presents a realistic photographic style, embodies Chinese sculpture culture, and reveals a mysterious atmosphere.） 		在茂密的森林中，一只黑白相间的熊猫静静地坐在绿树红花中，周围是山川和海洋。背景是白天的森林，光线充足
（In the dense forest, a black and white panda sits quietly in green trees and red flowers, surrounded by mountains, rivers, and the ocean. The background is the forest in a bright environment.） 		一位亚洲女性，身穿绿色上衣，戴着紫色头巾和紫色围巾，站在黑板前。背景是黑板。照片采用近景、平视和居中构图的方式呈现真实摄影风格
（An Asian woman, dressed in a green top, wearing a purple headscarf and a purple scarf, stands in front of a blackboard. The background is the blackboard. The photo is presented in a close-up, eye-level, and centered composition, adopting a realistic photographic style）
Image 0Image 1Image 2
ControlNet Output
Image 3Image 4Image 5
+ + +### Training + +We utilize [**DWPose**](https://github.com/IDEA-Research/DWPose) for pose extraction. Please follow their guidelines to download the checkpoints and save them to `hydit/annotator/ckpts` directory. We provide serveral commands to quick install: +```bash +mkdir ./hydit/annotator/ckpts +wget -O ./hydit/annotator/ckpts/dwpose.zip https://dit.hunyuan.tencent.com/download/HunyuanDiT/dwpose.zip +unzip ./hydit/annotator/ckpts/dwpose.zip -d ./hydit/annotator/ckpts/ +``` +Additionally, ensure that you install the related dependencies. +```bash +pip install matplotlib==3.7.5 +pip install onnxruntime_gpu==1.16.3 +pip install opencv-python==4.8.1.78 +``` + + +We provide three types of weights for ControlNet training, `ema`, `module` and `distill`, and you can choose according to the actual effects. By default, we use `distill` weights. + +Here is an example, we load the `distill` weights into the main model and conduct ControlNet training. + +If you want to load the `module` weights into the main model, just remove the `--ema-to-module` parameter. + +If apply multiple resolution training, you need to add the `--multireso` and `--reso-step 64` parameter. + +```bash +task_flag="canny_controlnet" # task flag is used to identify folders. +control_type=canny +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=path/to/your/index_file +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=2 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + + +sh $(dirname "$0")/run_g_controlnet.sh \ + --task-flag ${task_flag} \ + --control-type ${control_type} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --multireso \ + --reso-step 64 \ + --ema-to-module \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + "$@" +``` + +Recommended parameter settings + +| Parameter | Description | Recommended Parameter Value | Note| +|:---------------:|:---------:|:---------------------------------------------------:|:--:| +| `--batch-size` | Training batch size | 1 | Depends on GPU memory| +| `--grad-accu-steps` | Size of gradient accumulation | 2 | - | +| `--lr` | Learning rate | 0.0001 | - | +| `--control-type` | ControlNet condition type, support 3 types now (canny, depth and pose) | / | - | + + +### Inference +You can use the following command line for inference. + +a. You can use a float to specify the weight for all layers, **or use a list to separately specify the weight for each layer**, for example, '[1.0 * (0.825 ** float(19 - i)) for i in range(19)]' +```bash +python3 sample_controlnet.py --control-weight [1.0 * (0.825 ** float(19 - i)) for i in range(19)] --no-enhance --load-key distill --infer-steps 50 --control-type canny --prompt "在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围" --condition-image-path controlnet/asset/input/canny.jpg +``` + +b. Using canny ControlNet during inference + +```bash +python3 sample_controlnet.py --no-enhance --load-key distill --infer-steps 50 --control-type canny --prompt "在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围" --condition-image-path controlnet/asset/input/canny.jpg --control-weight 1.0 +``` + +c. Using pose ControlNet during inference + +```bash +python3 sample_controlnet.py --no-enhance --load-key distill --infer-steps 50 --control-type depth --prompt "在茂密的森林中，一只黑白相间的熊猫静静地坐在绿树红花中，周围是山川和海洋。背景是白天的森林，光线充足" --condition-image-path controlnet/asset/input/depth.jpg --control-weight 1.0 +``` + +d. Using depth ControlNet during inference + +```bash +python3 sample_controlnet.py --no-enhance --load-key distill --infer-steps 50 --control-type pose --prompt "一位亚洲女性，身穿绿色上衣，戴着紫色头巾和紫色围巾，站在黑板前。背景是黑板。照片采用近景、平视和居中构图的方式呈现真实摄影风格" --condition-image-path controlnet/asset/input/pose.jpg --control-weight 1.0 +``` + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain.yaml b/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95e8c65ce52316778728c27d62967bbd31a6dc49 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain.yaml @@ -0,0 +1,15 @@ +source: + - ./dataset/porcelain/arrows/00000.arrow + +filter: + column: + - name: height + type: int + action: ge + target: 1024 + default: 1024 + - name: width + type: int + action: ge + target: 1024 + default: 1024 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain_mt.yaml b/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain_mt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96ca828f9cb3d30d3c6a6a4698dc8a369a27709d --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/dataset/yamls/porcelain_mt.yaml @@ -0,0 +1,5 @@ +src: + - ./dataset/porcelain/jsons/porcelain.json +base_size: 1024 +reso_step: 64 +min_size: 1024 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/diffusers/README.md b/PyTorch/built-in/mlm/HunyuanDiT/diffusers/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3a45574203ef91c31654cfa49e6c750cbfb9d658 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/diffusers/README.md @@ -0,0 +1,131 @@ +# Hunyuan-DiT + 🤗 Diffusers + +You can use Hunyuan-DiT in 🤗 Diffusers library. Before using the pipelines, please install the latest version of 🤗 Diffusers with +```bash +pip install git+https://github.com/huggingface/diffusers.git +``` + +## Inference with th Base Model + +You can generate images with both Chinese and English prompts using the following Python script: +```py +import torch +from diffusers import HunyuanDiTPipeline + +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers", torch_dtype=torch.float16) +pipe.to("cuda") + +# You may also use English prompt as HunyuanDiT supports both English and Chinese +# prompt = "An astronaut riding a horse" +prompt = "一个宇航员在骑马" +image = pipe(prompt).images[0] +``` +You can use our distilled model to generate images even faster: + +```py +import torch +from diffusers import HunyuanDiTPipeline + +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled", torch_dtype=torch.float16) +pipe.to("cuda") + +# You may also use English prompt as HunyuanDiT supports both English and Chinese +# prompt = "An astronaut riding a horse" +prompt = "一个宇航员在骑马" +image = pipe(prompt, num_inference_steps=25).images[0] +``` +More details can be found in [HunyuanDiT-Diffusers-Distilled](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-Diffusers-Distilled) + +## LoRA +LoRA can be integrated with Hunyuan-DiT inside the 🤗 Diffusers framework. +The following example loads and uses the pre-trained LoRA. To try it, please start by downloading our pre-trained LoRA checkpoints, +```bash +huggingface-cli download Tencent-Hunyuan/HYDiT-LoRA --local-dir ./ckpts/t2i/lora +``` +Then run the following code snippet to use the jade LoRA: +```python +import torch +from diffusers import HunyuanDiTPipeline + +### convert checkpoint to diffusers format +num_layers = 40 +def load_hunyuan_dit_lora(transformer_state_dict, lora_state_dict, lora_scale): + for i in range(num_layers): + Wqkv = torch.matmul(lora_state_dict[f"blocks.{i}.attn1.Wqkv.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn1.Wqkv.lora_A.weight"]) + q, k, v = torch.chunk(Wqkv, 3, dim=0) + transformer_state_dict[f"blocks.{i}.attn1.to_q.weight"] += lora_scale * q + transformer_state_dict[f"blocks.{i}.attn1.to_k.weight"] += lora_scale * k + transformer_state_dict[f"blocks.{i}.attn1.to_v.weight"] += lora_scale * v + + out_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn1.out_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn1.out_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn1.to_out.0.weight"] += lora_scale * out_proj + + q_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.q_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.q_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn2.to_q.weight"] += lora_scale * q_proj + + kv_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.kv_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.kv_proj.lora_A.weight"]) + k, v = torch.chunk(kv_proj, 2, dim=0) + transformer_state_dict[f"blocks.{i}.attn2.to_k.weight"] += lora_scale * k + transformer_state_dict[f"blocks.{i}.attn2.to_v.weight"] += lora_scale * v + + out_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.out_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.out_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn2.to_out.0.weight"] += lora_scale * out_proj + + q_proj = torch.matmul(lora_state_dict["pooler.q_proj.lora_B.weight"], lora_state_dict["pooler.q_proj.lora_A.weight"]) + transformer_state_dict["time_extra_emb.pooler.q_proj.weight"] += lora_scale * q_proj + + return transformer_state_dict + +### use the diffusers pipeline with lora +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers", torch_dtype=torch.float16) +pipe.to("cuda") + +from safetensors import safe_open + +lora_state_dict = {} +with safe_open("./ckpts/t2i/lora/jade/adapter_model.safetensors", framework="pt", device=0) as f: + for k in f.keys(): + lora_state_dict[k[17:]] = f.get_tensor(k) # remove 'basemodel.model' + +transformer_state_dict = pipe.transformer.state_dict() +transformer_state_dict = load_hunyuan_dit_lora(transformer_state_dict, lora_state_dict, lora_scale=1.0) +pipe.transformer.load_state_dict(transformer_state_dict) + +prompt = "玉石绘画风格，一只猫在追蝴蝶" +image = pipe( + prompt, + num_inference_steps=100, + guidance_scale=6.0, +).images[0] +image.save('img.png') +``` + +You can control the strength of LoRA by changing the `lora_scale` parameter. + +## ControlNet +Hunyuan-DiT + ControlNet is supported in 🤗 Diffusers. The following example shows how to use Hunyuan-DiT + Canny ControlNet. +```py +from diffusers import HunyuanDiT2DControlNetModel, HunyuanDiTControlNetPipeline +import torch +controlnet = HunyuanDiT2DControlNetModel.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-ControlNet-Diffusers-Canny", torch_dtype=torch.float16) + +pipe = HunyuanDiTControlNetPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers", controlnet=controlnet, torch_dtype=torch.float16) +pipe.to("cuda") + +from diffusers.utils import load_image +cond_image = load_image('https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1-ControlNet-Diffusers-Canny/resolve/main/canny.jpg?download=true') + +## You may also use English prompt as HunyuanDiT supports both English and Chinese +prompt="在夜晚的酒店门前，一座古老的中国风格的狮子雕像矗立着，它的眼睛闪烁着光芒，仿佛在守护着这座建筑。背景是夜晚的酒店前，构图方式是特写，平视，居中构图。这张照片呈现了真实摄影风格，蕴含了中国雕塑文化，同时展现了神秘氛围" +#prompt="At night, an ancient Chinese-style lion statue stands in front of the hotel, its eyes gleaming as if guarding the building. The background is the hotel entrance at night, with a close-up, eye-level, and centered composition. This photo presents a realistic photographic style, embodies Chinese sculpture culture, and reveals a mysterious atmosphere." +image = pipe( + prompt, + height=1024, + width=1024, + control_image=cond_image, + num_inference_steps=50, +).images[0] +``` + +There are other pre-trained ControlNets available. Please have a look at [the official huggingface website of Tencent Hunyuan Team](https://huggingface.co/Tencent-Hunyuan) + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/environment.yml b/PyTorch/built-in/mlm/HunyuanDiT/environment.yml new file mode 100644 index 0000000000000000000000000000000000000000..f43b45b7fbd182dc2cb6c82dec2cbecc562b1a17 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/environment.yml @@ -0,0 +1,8 @@ +name: HunyuanDiT +channels: + - pytorch + - nvidia +dependencies: + - python=3.8.12 + - pytorch=1.13.1 + - pip diff --git a/PyTorch/built-in/mlm/HunyuanDiT/example_prompts.txt b/PyTorch/built-in/mlm/HunyuanDiT/example_prompts.txt new file mode 100644 index 0000000000000000000000000000000000000000..f590be43c46e4c50f91a28f84821047241066df2 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/example_prompts.txt @@ -0,0 +1,28 @@ +一只聪明的狐狸走在阔叶树林里, 旁边是一条小溪, 细节真实, 摄影 +湖水清澈，天空湛蓝，阳光灿烂。一只优雅的白天鹅在湖边游泳。它周围有几只小鸭子，看起来非常可爱，整个画面给人一种宁静祥和的感觉。 +太阳微微升起，花园里的玫瑰花瓣上露珠晶莹剔透，一只瓢虫正在爬向露珠，背景是清晨的花园，微距镜头 +一位女明星，中国人，头发是黑色，衣服是纯白色短袖，人物风格清新，城市背景 +后印象主义风格，一条古老的石板路上面散落着金黄色的树叶。路旁的风车在静谧地转动，后面竖着两个风车。背景是一片向日葵田，蓝天上飘着几朵白云 +一幅细致的油画描绘了一只年轻獾轻轻嗅着一朵明亮的黄色玫瑰时错综复杂的皮毛。背景是一棵大树干的粗糙纹理，獾的爪子轻轻地挖进树皮。在柔和的背景中，一个宁静的瀑布倾泻而下，它的水在绿色植物中闪烁着蓝色。 +渔舟唱晚 +请将杞人忧天的样子画出来 +一只长靴猫手持亮银色的宝剑，身着铠甲，眼神坚毅，站在一堆金币上，背景是暗色调的洞穴，图像上有金币的光影点缀。 +插画风格，一只狐狸和一只刺猬坐在水边的石头上，刺猬手里拿着一杯茶，狐狸旁边放着一个玻璃杯。周围是茂密的绿色植物和树木，阳光透过树叶洒在水面上，画面宁静温馨。 +泥塑风格，一座五彩斑斓的花园在画面中展现，各种各样的花朵，绿色的叶子和一只正在嬉戏的小猫形成了一幅生动的图像，背景是蓝天和白云 +枯藤老树昏鸦，小桥流水人家 +一张细致的照片捕捉到了一尊雕像的形象，这尊雕像酷似一位古代法老，头上出人意料地戴着一副青铜蒸汽朋克护目镜。这座雕像穿着复古时髦，一件清爽的白色T恤和一件合身的黑色皮夹克，与传统的头饰形成鲜明对比。背景是简单的纯色，突出了雕像的非传统服装和蒸汽朋克眼镜的复杂细节。 +一朵鲜艳的红色玫瑰花，花瓣撒有一些水珠，晶莹剔透，特写镜头， +一只可爱的猫, 细节真实, 摄影 +飞流直下三千尺，疑是银河落九天 +成语“鲤鱼跃龙门” +一颗新鲜的草莓特写，红色的外表，表面布满许多种子，背景是淡绿色的叶子 +九寨沟 +摄影风格，在画面中心是一盘热气腾腾的麻婆豆腐，豆腐呈白色，上面撒着一层红色的辣酱，有些许绿色的葱花点缀，背景是深色木质餐桌，桌子上放有辣椒和葱花作为点缀。 +一位年轻女子站在春季的火车站月台上。她身着蓝灰色长风衣，白色衬衫。她的深棕色头发扎成低马尾，几缕碎发随风飘扬。她的眼神充满期待，阳光洒在她温暖的脸庞上。 +一只优雅的白鹤在湖边静静地站立，它的身体纯白色，翅膀轻轻展开，背景是湖面和远处的山脉 +国画风格，苏州园林中的小桥流水，周围是郁郁葱葱的树，池塘里有几朵绽放的荷花，背景是宁静的江南水乡 +现实主义风格，画面主要描述一个巴洛克风格的花瓶，带有金色的装饰边框，花瓶上盛开着各种色彩鲜艳的花，白色背景 +醉后不知天在水，满船清梦压星河 +长城 +一个亚洲中年男士在夕阳下的公园长椅上静坐。他穿着一件深蓝色的针织毛衣和灰色裤子。他的头发略显花白，手中拿着一本敞开的书。面带微笑，眼神温和，周围是落日余晖和四周的绿树。 +风格是写实，画面主要描述一个亚洲戏曲艺术家正在表演，她穿着华丽的戏服，脸上戴着精致的面具，身姿优雅，背景是古色古香的舞台，镜头是近景 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/gpu_sample.sh b/PyTorch/built-in/mlm/HunyuanDiT/gpu_sample.sh new file mode 100644 index 0000000000000000000000000000000000000000..f7721e48aa57c29a43307c8d61075bb89412e2d3 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/gpu_sample.sh @@ -0,0 +1 @@ +python sample_t2i.py --infer-mode fa --prompt "渔舟唱晚" --image-size 1280 768 --no-enhance \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/gpu_train.sh b/PyTorch/built-in/mlm/HunyuanDiT/gpu_train.sh new file mode 100644 index 0000000000000000000000000000000000000000..95005fbcdcc032dfd647d57b054500c435722772 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/gpu_train.sh @@ -0,0 +1,2 @@ +export CUBLAS_WORKSPACE_CONFIG=:4096:8 +PYTHONPATH=./ sh hydit/train.sh \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d17d0033b243cad48f26f5555d3dadac266a8f6 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/__init__.py @@ -0,0 +1,143 @@ +# Openpose +# Original from CMU https://github.com/CMU-Perceptual-Computing-Lab/openpose +# 2nd Edited by https://github.com/Hzzone/pytorch-openpose +# 3rd Edited by ControlNet +# 4th Edited by ControlNet (added face and correct hands) + +import os +import random + +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + +import torch +import numpy as np +from . import util +from .wholebody import Wholebody + + +def draw_pose(pose, H, W, draw_body=True): + bodies = pose['bodies'] + faces = pose['faces'] + hands = pose['hands'] + candidate = bodies['candidate'] + subset = bodies['subset'] + canvas = np.zeros(shape=(H, W, 3), dtype=np.uint8) + + if draw_body: + canvas = util.draw_bodypose(canvas, candidate, subset) + + canvas = util.draw_handpose(canvas, hands) + + canvas = util.draw_facepose(canvas, faces) + + return canvas + + +def keypoint2bbox(keypoints): + valid_keypoints = keypoints[keypoints[:, 0] >= 0] # Ignore keypoints with confidence 0 + if len(valid_keypoints) == 0: + return np.zeros(4) + x_min, y_min = np.min(valid_keypoints, axis=0) + x_max, y_max = np.max(valid_keypoints, axis=0) + + return np.array([x_min, y_min, x_max, y_max]) + +def expand_bboxes(bboxes, expansion_rate=0.5, image_shape=(0, 0)): + expanded_bboxes = [] + for bbox in bboxes: + x_min, y_min, x_max, y_max = map(int, bbox) + width = x_max - x_min + height = y_max - y_min + + # 扩展宽度和高度 + new_width = width * (1 + expansion_rate) + new_height = height * (1 + expansion_rate) + + # 计算新的边界框坐标 + x_min_new = max(0, x_min - (new_width - width) / 2) + x_max_new = min(image_shape[1], x_max + (new_width - width) / 2) + y_min_new = max(0, y_min - (new_height - height) / 2) + y_max_new = min(image_shape[0], y_max + (new_height - height) / 2) + + expanded_bboxes.append([x_min_new, y_min_new, x_max_new, y_max_new]) + + return expanded_bboxes + +def create_mask(image_width, image_height, bboxs): + mask = np.zeros((image_height, image_width), dtype=np.float32) + for bbox in bboxs: + x1, y1, x2, y2 = map(int, bbox) + mask[y1:y2+1, x1:x2+1] = 1.0 + return mask + +threshold = 0.4 +class DWposeDetector: + def __init__(self): + + self.pose_estimation = Wholebody() + + def __call__(self, oriImg, return_index=False, return_yolo=False, return_mask=False): + oriImg = oriImg.copy() + H, W, C = oriImg.shape + with torch.no_grad(): + candidate, subset = self.pose_estimation(oriImg) + candidate = np.zeros((1, 134, 2), dtype=np.float32) if candidate is None else candidate + subset = np.zeros((1, 134), dtype=np.float32) if subset is None else subset + nums, keys, locs = candidate.shape + candidate[..., 0] /= float(W) + candidate[..., 1] /= float(H) + # import pdb; pdb.set_trace() + if return_yolo: + candidate[subset < threshold] = -0.1 + subset = np.expand_dims(subset >= threshold, axis=-1) + keypoint = np.concatenate([candidate, subset], axis=-1) + + # return pose + hand + return np.concatenate([keypoint[:, :18], keypoint[:, 92:]], axis=1) + + body = candidate[:, :18].copy() + body = body.reshape(nums * 18, locs) + score = subset[:, :18] + for i in range(len(score)): + for j in range(len(score[i])): + if score[i][j] > threshold: + score[i][j] = int(18 * i + j) + else: + score[i][j] = -1 + + un_visible = subset < threshold + candidate[un_visible] = -1 + + foot = candidate[:, 18:24] + + faces = candidate[:, 24:92] + + hands1 = candidate[:, 92:113] + hands2 = candidate[:, 113:] + hands = np.vstack([hands1, hands2]) + + # import pdb; pdb.set_trace() + hands_ = hands[hands.max(axis=(1, 2)) > 0] + if len(hands_) == 0: + bbox = np.array([0, 0, 0, 0]).astype(int) + else: + hand_random = random.choice(hands_) + bbox = (keypoint2bbox(hand_random) * H).astype(int) # [0, 1] -> [h, w] + + + + bodies = dict(candidate=body, subset=score) + pose = dict(bodies=bodies, hands=hands, faces=faces) + + if return_mask: + bbox = [(keypoint2bbox(hand) * H).astype(int) for hand in hands_] + # bbox = expand_bboxes(bbox, expansion_rate=0.5, image_shape=(H, W)) + mask = create_mask(W, H, bbox) + return draw_pose(pose, H, W), mask + + if return_index: + return pose + else: + return draw_pose(pose, H, W), bbox + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxdet.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxdet.py new file mode 100644 index 0000000000000000000000000000000000000000..48056f54338de43cbd3430339b957922f37fc8c8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxdet.py @@ -0,0 +1,129 @@ +import cv2 +import numpy as np + +import onnxruntime + + +def nms(boxes, scores, nms_thr): + """Single class NMS implemented in Numpy.""" + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + + +def multiclass_nms(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-aware version.""" + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1 + ) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + + +def demo_postprocess(outputs, img_size, p6=False): + grids = [] + expanded_strides = [] + strides = [8, 16, 32] if not p6 else [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs + + +def preprocess(img, input_size, swap=(2, 0, 1)): + if len(img.shape) == 3: + padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(input_size, dtype=np.uint8) * 114 + + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + + +def inference_detector(session, oriImg): + input_shape = (640, 640) + img, ratio = preprocess(oriImg, input_shape) + + ort_inputs = {session.get_inputs()[0].name: img[None, :, :, :]} + output = session.run(None, ort_inputs) + predictions = demo_postprocess(output[0], input_shape)[0] + + boxes = predictions[:, :4] + scores = predictions[:, 4:5] * predictions[:, 5:] + + boxes_xyxy = np.ones_like(boxes) + boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2] / 2. + boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3] / 2. + boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2] / 2. + boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3] / 2. + boxes_xyxy /= ratio + dets = multiclass_nms(boxes_xyxy, scores, nms_thr=0.45, score_thr=0.1) + if dets is not None: + final_boxes, final_scores, final_cls_inds = dets[:, :4], dets[:, 4], dets[:, 5] + isscore = final_scores > 0.3 + iscat = final_cls_inds == 0 + isbbox = [i and j for (i, j) in zip(isscore, iscat)] + final_boxes = final_boxes[isbbox] + return final_boxes + else: + return None diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxpose.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxpose.py new file mode 100644 index 0000000000000000000000000000000000000000..79cd4a06241123af81ea22446a4ca8816716443f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/onnxpose.py @@ -0,0 +1,360 @@ +from typing import List, Tuple + +import cv2 +import numpy as np +import onnxruntime as ort + +def preprocess( + img: np.ndarray, out_bbox, input_size: Tuple[int, int] = (192, 256) +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Do preprocessing for RTMPose model inference. + + Args: + img (np.ndarray): Input image in shape. + input_size (tuple): Input image size in shape (w, h). + + Returns: + tuple: + - resized_img (np.ndarray): Preprocessed image. + - center (np.ndarray): Center of image. + - scale (np.ndarray): Scale of image. + """ + # get shape of image + img_shape = img.shape[:2] + out_img, out_center, out_scale = [], [], [] + if len(out_bbox) == 0: + out_bbox = [[0, 0, img_shape[1], img_shape[0]]] + for i in range(len(out_bbox)): + x0 = out_bbox[i][0] + y0 = out_bbox[i][1] + x1 = out_bbox[i][2] + y1 = out_bbox[i][3] + bbox = np.array([x0, y0, x1, y1]) + + # get center and scale + center, scale = bbox_xyxy2cs(bbox, padding=1.25) + + # do affine transformation + resized_img, scale = top_down_affine(input_size, scale, center, img) + + # normalize image + mean = np.array([123.675, 116.28, 103.53]) + std = np.array([58.395, 57.12, 57.375]) + resized_img = (resized_img - mean) / std + + out_img.append(resized_img) + out_center.append(center) + out_scale.append(scale) + + return out_img, out_center, out_scale + + +def inference(sess: ort.InferenceSession, img: np.ndarray) -> np.ndarray: + """Inference RTMPose model. + + Args: + sess (ort.InferenceSession): ONNXRuntime session. + img (np.ndarray): Input image in shape. + + Returns: + outputs (np.ndarray): Output of RTMPose model. + """ + all_out = [] + # build input + for i in range(len(img)): + input = [img[i].transpose(2, 0, 1)] + + # build output + sess_input = {sess.get_inputs()[0].name: input} + sess_output = [] + for out in sess.get_outputs(): + sess_output.append(out.name) + + # run model + outputs = sess.run(sess_output, sess_input) + all_out.append(outputs) + + return all_out + + +def postprocess(outputs: List[np.ndarray], + model_input_size: Tuple[int, int], + center: Tuple[int, int], + scale: Tuple[int, int], + simcc_split_ratio: float = 2.0 + ) -> Tuple[np.ndarray, np.ndarray]: + """Postprocess for RTMPose model output. + + Args: + outputs (np.ndarray): Output of RTMPose model. + model_input_size (tuple): RTMPose model Input image size. + center (tuple): Center of bbox in shape (x, y). + scale (tuple): Scale of bbox in shape (w, h). + simcc_split_ratio (float): Split ratio of simcc. + + Returns: + tuple: + - keypoints (np.ndarray): Rescaled keypoints. + - scores (np.ndarray): Model predict scores. + """ + all_key = [] + all_score = [] + for i in range(len(outputs)): + # use simcc to decode + simcc_x, simcc_y = outputs[i] + keypoints, scores = decode(simcc_x, simcc_y, simcc_split_ratio) + + # rescale keypoints + keypoints = keypoints / model_input_size * scale[i] + center[i] - scale[i] / 2 + all_key.append(keypoints[0]) + all_score.append(scores[0]) + + return np.array(all_key), np.array(all_score) + + +def bbox_xyxy2cs(bbox: np.ndarray, + padding: float = 1.) -> Tuple[np.ndarray, np.ndarray]: + """Transform the bbox format from (x,y,w,h) into (center, scale) + + Args: + bbox (ndarray): Bounding box(es) in shape (4,) or (n, 4), formatted + as (left, top, right, bottom) + padding (float): BBox padding factor that will be multilied to scale. + Default: 1.0 + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: Center (x, y) of the bbox in shape (2,) or + (n, 2) + - np.ndarray[float32]: Scale (w, h) of the bbox in shape (2,) or + (n, 2) + """ + # convert single bbox from (4, ) to (1, 4) + dim = bbox.ndim + if dim == 1: + bbox = bbox[None, :] + + # get bbox center and scale + x1, y1, x2, y2 = np.hsplit(bbox, [1, 2, 3]) + center = np.hstack([x1 + x2, y1 + y2]) * 0.5 + scale = np.hstack([x2 - x1, y2 - y1]) * padding + + if dim == 1: + center = center[0] + scale = scale[0] + + return center, scale + + +def _fix_aspect_ratio(bbox_scale: np.ndarray, + aspect_ratio: float) -> np.ndarray: + """Extend the scale to match the given aspect ratio. + + Args: + scale (np.ndarray): The image scale (w, h) in shape (2, ) + aspect_ratio (float): The ratio of ``w/h`` + + Returns: + np.ndarray: The reshaped image scale in (2, ) + """ + w, h = np.hsplit(bbox_scale, [1]) + bbox_scale = np.where(w > h * aspect_ratio, + np.hstack([w, w / aspect_ratio]), + np.hstack([h * aspect_ratio, h])) + return bbox_scale + + +def _rotate_point(pt: np.ndarray, angle_rad: float) -> np.ndarray: + """Rotate a point by an angle. + + Args: + pt (np.ndarray): 2D point coordinates (x, y) in shape (2, ) + angle_rad (float): rotation angle in radian + + Returns: + np.ndarray: Rotated point in shape (2, ) + """ + sn, cs = np.sin(angle_rad), np.cos(angle_rad) + rot_mat = np.array([[cs, -sn], [sn, cs]]) + return rot_mat @ pt + + +def _get_3rd_point(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """To calculate the affine matrix, three pairs of points are required. This + function is used to get the 3rd point, given 2D points a & b. + + The 3rd point is defined by rotating vector `a - b` by 90 degrees + anticlockwise, using b as the rotation center. + + Args: + a (np.ndarray): The 1st point (x,y) in shape (2, ) + b (np.ndarray): The 2nd point (x,y) in shape (2, ) + + Returns: + np.ndarray: The 3rd point. + """ + direction = a - b + c = b + np.r_[-direction[1], direction[0]] + return c + + +def get_warp_matrix(center: np.ndarray, + scale: np.ndarray, + rot: float, + output_size: Tuple[int, int], + shift: Tuple[float, float] = (0., 0.), + inv: bool = False) -> np.ndarray: + """Calculate the affine transformation matrix that can warp the bbox area + in the input image to the output size. + + Args: + center (np.ndarray[2, ]): Center of the bounding box (x, y). + scale (np.ndarray[2, ]): Scale of the bounding box + wrt [width, height]. + rot (float): Rotation angle (degree). + output_size (np.ndarray[2, ] | list(2,)): Size of the + destination heatmaps. + shift (0-100%): Shift translation ratio wrt the width/height. + Default (0., 0.). + inv (bool): Option to inverse the affine transform direction. + (inv=False: src->dst or inv=True: dst->src) + + Returns: + np.ndarray: A 2x3 transformation matrix + """ + shift = np.array(shift) + src_w = scale[0] + dst_w = output_size[0] + dst_h = output_size[1] + + # compute transformation matrix + rot_rad = np.deg2rad(rot) + src_dir = _rotate_point(np.array([0., src_w * -0.5]), rot_rad) + dst_dir = np.array([0., dst_w * -0.5]) + + # get four corners of the src rectangle in the original image + src = np.zeros((3, 2), dtype=np.float32) + src[0, :] = center + scale * shift + src[1, :] = center + src_dir + scale * shift + src[2, :] = _get_3rd_point(src[0, :], src[1, :]) + + # get four corners of the dst rectangle in the input image + dst = np.zeros((3, 2), dtype=np.float32) + dst[0, :] = [dst_w * 0.5, dst_h * 0.5] + dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir + dst[2, :] = _get_3rd_point(dst[0, :], dst[1, :]) + + if inv: + warp_mat = cv2.getAffineTransform(np.float32(dst), np.float32(src)) + else: + warp_mat = cv2.getAffineTransform(np.float32(src), np.float32(dst)) + + return warp_mat + + +def top_down_affine(input_size: dict, bbox_scale: dict, bbox_center: dict, + img: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get the bbox image as the model input by affine transform. + + Args: + input_size (dict): The input size of the model. + bbox_scale (dict): The bbox scale of the img. + bbox_center (dict): The bbox center of the img. + img (np.ndarray): The original image. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: img after affine transform. + - np.ndarray[float32]: bbox scale after affine transform. + """ + w, h = input_size + warp_size = (int(w), int(h)) + + # reshape bbox to fixed aspect ratio + bbox_scale = _fix_aspect_ratio(bbox_scale, aspect_ratio=w / h) + + # get the affine matrix + center = bbox_center + scale = bbox_scale + rot = 0 + warp_mat = get_warp_matrix(center, scale, rot, output_size=(w, h)) + + # do affine transform + img = cv2.warpAffine(img, warp_mat, warp_size, flags=cv2.INTER_LINEAR) + + return img, bbox_scale + + +def get_simcc_maximum(simcc_x: np.ndarray, + simcc_y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get maximum response location and value from simcc representations. + + Note: + instance number: N + num_keypoints: K + heatmap height: H + heatmap width: W + + Args: + simcc_x (np.ndarray): x-axis SimCC in shape (K, Wx) or (N, K, Wx) + simcc_y (np.ndarray): y-axis SimCC in shape (K, Wy) or (N, K, Wy) + + Returns: + tuple: + - locs (np.ndarray): locations of maximum heatmap responses in shape + (K, 2) or (N, K, 2) + - vals (np.ndarray): values of maximum heatmap responses in shape + (K,) or (N, K) + """ + N, K, Wx = simcc_x.shape + simcc_x = simcc_x.reshape(N * K, -1) + simcc_y = simcc_y.reshape(N * K, -1) + + # get maximum value locations + x_locs = np.argmax(simcc_x, axis=1) + y_locs = np.argmax(simcc_y, axis=1) + locs = np.stack((x_locs, y_locs), axis=-1).astype(np.float32) + max_val_x = np.amax(simcc_x, axis=1) + max_val_y = np.amax(simcc_y, axis=1) + + # get maximum value across x and y axis + mask = max_val_x > max_val_y + max_val_x[mask] = max_val_y[mask] + vals = max_val_x + locs[vals <= 0.] = -1 + + # reshape + locs = locs.reshape(N, K, 2) + vals = vals.reshape(N, K) + + return locs, vals + + +def decode(simcc_x: np.ndarray, simcc_y: np.ndarray, + simcc_split_ratio) -> Tuple[np.ndarray, np.ndarray]: + """Modulate simcc distribution with Gaussian. + + Args: + simcc_x (np.ndarray[K, Wx]): model predicted simcc in x. + simcc_y (np.ndarray[K, Wy]): model predicted simcc in y. + simcc_split_ratio (int): The split ratio of simcc. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: keypoints in shape (K, 2) or (n, K, 2) + - np.ndarray[float32]: scores in shape (K,) or (n, K) + """ + keypoints, scores = get_simcc_maximum(simcc_x, simcc_y) + keypoints /= simcc_split_ratio + + return keypoints, scores + + +def inference_pose(session, out_bbox, oriImg): + h, w = session.get_inputs()[0].shape[2:] + model_input_size = (w, h) + resized_img, center, scale = preprocess(oriImg, out_bbox, model_input_size) + outputs = inference(session, resized_img) + keypoints, scores = postprocess(outputs, model_input_size, center, scale) + + return keypoints, scores \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/util.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/util.py new file mode 100644 index 0000000000000000000000000000000000000000..a1453ab66d3d7a2f161aa6fd45acf70729990863 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/util.py @@ -0,0 +1,298 @@ +import math +import numpy as np +import matplotlib +import cv2 + + +eps = 0.01 + + +def smart_resize(x, s): + Ht, Wt = s + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize(x[:, :, i], s) for i in range(Co)], axis=2) + + +def smart_resize_k(x, fx, fy): + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + Ht, Wt = Ho * fy, Wo * fx + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize_k(x[:, :, i], fx, fy) for i in range(Co)], axis=2) + + +def padRightDownCorner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join(weights_name.split('.')[1:])] + return transfered_model_weights + + +def draw_bodypose(canvas, candidate, subset): + H, W, C = canvas.shape + candidate = np.array(candidate) + subset = np.array(subset) + + stickwidth = 4 + + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], \ + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], \ + [1, 16], [16, 18], [3, 17], [6, 18]] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \ + [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \ + [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] + + for i in range(17): + for n in range(len(subset)): + index = subset[n][np.array(limbSeq[i]) - 1] + if -1 in index: + continue + Y = candidate[index.astype(int), 0] * float(W) + X = candidate[index.astype(int), 1] * float(H) + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) + # import pdb; pdb.set_trace() + cv2.fillConvexPoly(canvas, polygon, colors[i]) + + canvas = (canvas * 0.6).astype(np.uint8) + + for i in range(18): + for n in range(len(subset)): + index = int(subset[n][i]) + if index == -1: + continue + x, y = candidate[index][0:2] + x = int(x * W) + y = int(y * H) + cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1) + + return canvas + + +def draw_handpose(canvas, all_hand_peaks): + H, W, C = canvas.shape + + edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \ + [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] + + for peaks in all_hand_peaks: + peaks = np.array(peaks) + + for ie, e in enumerate(edges): + x1, y1 = peaks[e[0]] + x2, y2 = peaks[e[1]] + x1 = int(x1 * W) + y1 = int(y1 * H) + x2 = int(x2 * W) + y2 = int(y2 * H) + if x1 > eps and y1 > eps and x2 > eps and y2 > eps: + cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie / float(len(edges)), 1.0, 1.0]) * 255, thickness=2) + + for i, keyponit in enumerate(peaks): + x, y = keyponit + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1) + return canvas + + +def draw_facepose(canvas, all_lmks): + H, W, C = canvas.shape + for lmks in all_lmks: + lmks = np.array(lmks) + for lmk in lmks: + x, y = lmk + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 3, (255, 255, 255), thickness=-1) + return canvas + + +# detect hand according to body pose keypoints +# please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp +def handDetect(candidate, subset, oriImg): + # right hand: wrist 4, elbow 3, shoulder 2 + # left hand: wrist 7, elbow 6, shoulder 5 + ratioWristElbow = 0.33 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + # if any of three not detected + has_left = np.sum(person[[5, 6, 7]] == -1) == 0 + has_right = np.sum(person[[2, 3, 4]] == -1) == 0 + if not (has_left or has_right): + continue + hands = [] + #left hand + if has_left: + left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]] + x1, y1 = candidate[left_shoulder_index][:2] + x2, y2 = candidate[left_elbow_index][:2] + x3, y3 = candidate[left_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, True]) + # right hand + if has_right: + right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]] + x1, y1 = candidate[right_shoulder_index][:2] + x2, y2 = candidate[right_elbow_index][:2] + x3, y3 = candidate[right_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, False]) + + for x1, y1, x2, y2, x3, y3, is_left in hands: + # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox + # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]); + # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]); + # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow); + # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder); + # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder); + x = x3 + ratioWristElbow * (x3 - x2) + y = y3 + ratioWristElbow * (y3 - y2) + distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2) + distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) + width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder) + # x-y refers to the center --> offset to topLeft point + # handRectangle.x -= handRectangle.width / 2.f; + # handRectangle.y -= handRectangle.height / 2.f; + x -= width / 2 + y -= width / 2 # width = height + # overflow the image + if x < 0: x = 0 + if y < 0: y = 0 + width1 = width + width2 = width + if x + width > image_width: width1 = image_width - x + if y + width > image_height: width2 = image_height - y + width = min(width1, width2) + # the max hand box value is 20 pixels + if width >= 20: + detect_result.append([int(x), int(y), int(width), is_left]) + + ''' + return value: [[x, y, w, True if left hand else False]]. + width=height since the network require squared input. + x, y is the coordinate of top left + ''' + return detect_result + + +# Written by Lvmin +def faceDetect(candidate, subset, oriImg): + # left right eye ear 14 15 16 17 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + has_head = person[0] > -1 + if not has_head: + continue + + has_left_eye = person[14] > -1 + has_right_eye = person[15] > -1 + has_left_ear = person[16] > -1 + has_right_ear = person[17] > -1 + + if not (has_left_eye or has_right_eye or has_left_ear or has_right_ear): + continue + + head, left_eye, right_eye, left_ear, right_ear = person[[0, 14, 15, 16, 17]] + + width = 0.0 + x0, y0 = candidate[head][:2] + + if has_left_eye: + x1, y1 = candidate[left_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_right_eye: + x1, y1 = candidate[right_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_left_ear: + x1, y1 = candidate[left_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + if has_right_ear: + x1, y1 = candidate[right_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + x, y = x0, y0 + + x -= width + y -= width + + if x < 0: + x = 0 + + if y < 0: + y = 0 + + width1 = width * 2 + width2 = width * 2 + + if x + width > image_width: + width1 = image_width - x + + if y + width > image_height: + width2 = image_height - y + + width = min(width1, width2) + + if width >= 20: + detect_result.append([int(x), int(y), int(width)]) + + return detect_result + + +# get max index of 2d array +def npmax(array): + arrayindex = array.argmax(1) + arrayvalue = array.max(1) + i = arrayvalue.argmax() + j = arrayindex[i] + return i, j diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/wholebody.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/wholebody.py new file mode 100644 index 0000000000000000000000000000000000000000..108fa670ceaee0a263291b87674032ea3b663066 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/dwpose/wholebody.py @@ -0,0 +1,56 @@ +import os +import cv2 +import numpy as np + +import onnxruntime as ort +from .onnxdet import inference_detector +from .onnxpose import inference_pose + +class Wholebody: + def __init__(self): + rank = int(os.getenv('LOCAL_RANK', '0')) + device = f'cuda:{rank}' + providers = ['CPUExecutionProvider'] if device == 'cpu' else [("CUDAExecutionProvider", {"device_id": rank})] + onnx_det = 'hydit/annotator/ckpts/yolox_l.onnx' + onnx_pose = 'hydit/annotator/ckpts/dw-ll_ucoco_384.onnx' + + self.session_det = ort.InferenceSession(path_or_bytes=onnx_det, providers=providers) + self.session_pose = ort.InferenceSession(path_or_bytes=onnx_pose, providers=providers) + + def __call__(self, oriImg): + det_result = inference_detector(self.session_det, oriImg) + if det_result is None: + return None, None + keypoints, scores = inference_pose(self.session_pose, det_result, oriImg) + + keypoints_info = np.concatenate( + (keypoints, scores[..., None]), axis=-1) + # compute neck joint + neck = np.mean(keypoints_info[:, [5, 6]], axis=1) + # neck score when visualizing pred + neck[:, 2:4] = np.logical_and( + keypoints_info[:, 5, 2:4] > 0.3, + keypoints_info[:, 6, 2:4] > 0.3).astype(int) + new_keypoints_info = np.insert( + keypoints_info, 17, neck, axis=1) + mmpose_idx = [ + 17, 6, 8, 10, 7, 9, 12, 14, 16, 13, 15, 2, 1, 4, 3 + ] + openpose_idx = [ + 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17 + ] + new_keypoints_info[:, openpose_idx] = \ + new_keypoints_info[:, mmpose_idx] + keypoints_info = new_keypoints_info + + keypoints, scores = keypoints_info[ + ..., :2], keypoints_info[..., 2] + + return keypoints, scores + + def to(self, device): + self.session_det.set_providers([device]) + self.session_pose.set_providers([device]) + return self + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/glyph.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/glyph.py new file mode 100644 index 0000000000000000000000000000000000000000..8bb67813897e90e051a218fcf157ee7261dc59b5 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/glyph.py @@ -0,0 +1,249 @@ +# MIT License +# Copyright (c) 2023 AIGText +# https://github.com/AIGText/GlyphControl-release + +from PIL import Image, ImageFont, ImageDraw +import random +import numpy as np +import cv2 + + +# resize height to image_height first, then shrink or pad to image_width +def resize_and_pad_image(pil_image, image_size): + if isinstance(image_size, (tuple, list)) and len(image_size) == 2: + image_width, image_height = image_size + elif isinstance(image_size, int): + image_width = image_height = image_size + else: + raise ValueError(f"Image size should be int or list/tuple of int not {image_size}") + + while pil_image.size[1] >= 2 * image_height: + pil_image = pil_image.resize( + tuple(x // 2 for x in pil_image.size), resample=Image.BOX + ) + + scale = image_height / pil_image.size[1] + pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC) + + # shrink + if pil_image.size[0] > image_width: + pil_image = pil_image.resize((image_width, image_height), resample=Image.BICUBIC) + + # padding + if pil_image.size[0] < image_width: + img = Image.new(mode="RGBA", size=(image_width, image_height), color=(255, 255, 255, 0)) + width, _ = pil_image.size + img.paste(pil_image, ((image_width - width) // 2, 0)) + pil_image = img + + return pil_image + + +def resize_and_pad_image2(pil_image, image_size): + if isinstance(image_size, (tuple, list)) and len(image_size) == 2: + image_width, image_height = image_size + elif isinstance(image_size, int): + image_width = image_height = image_size + else: + raise ValueError(f"Image size should be int or list/tuple of int not {image_size}") + + while pil_image.size[1] >= 2 * image_height: + pil_image = pil_image.resize( + tuple(x // 2 for x in pil_image.size), resample=Image.BOX + ) + + scale = image_height / pil_image.size[1] + pil_image = pil_image.resize(tuple(round(x * scale) for x in pil_image.size), resample=Image.BICUBIC) + + # shrink + if pil_image.size[0] > image_width: + pil_image = pil_image.resize((image_width, image_height), resample=Image.BICUBIC) + + # padding + if pil_image.size[0] < image_width: + img = Image.new(mode="RGB", size=(image_width, image_height), color="white") + width, _ = pil_image.size + img.paste(pil_image, ((image_width - width) // 2, 0)) + pil_image = img + + return pil_image + + +def draw_visual_text(image_size, bboxes, rendered_txt_values, num_rows_values=None, align="center"): + # aligns = ["center", "left", "right"] + """Render text image based on the glyph instructions, i.e., the list of tuples (text, bbox, num_rows). + Currently we just use Calibri font to render glyph images. + """ + # print(image_size, bboxes, rendered_txt_values, num_rows_values, align) + background = Image.new("RGB", image_size, "white") + font = ImageFont.truetype("simfang.ttf", encoding='utf-8', size=512) + if num_rows_values is None: + num_rows_values = [1] * len(rendered_txt_values) + + text_list = [] + for text, bbox, num_rows in zip(rendered_txt_values, bboxes, num_rows_values): + + if len(text) == 0: + continue + + text = text.strip() + if num_rows != 1: + word_tokens = text.split() + num_tokens = len(word_tokens) + index_list = range(1, num_tokens + 1) + if num_tokens > num_rows: + index_list = random.sample(index_list, num_rows) + index_list.sort() + line_list = [] + start_idx = 0 + for index in index_list: + line_list.append( + " ".join(word_tokens + [start_idx: index] + ) + ) + start_idx = index + text = "\n".join(line_list) + + if 'ratio' not in bbox or bbox['ratio'] == 0 or bbox['ratio'] < 1e-4: + image4ratio = Image.new("RGB", (512, 512), "white") + draw = ImageDraw.Draw(image4ratio) + _, _, w, h = draw.textbbox(xy=(0, 0), text=text, font=font) + ratio = w / h + else: + ratio = bbox['ratio'] + + width = int(bbox['width'] * image_size[1]) + height = int(width / ratio) + top_left_x = int(bbox['top_left_x'] * image_size[0]) + top_left_y = int(bbox['top_left_y'] * image_size[1]) + yaw = bbox['yaw'] + + text_image = Image.new("RGB", (512, 512), "white") + draw = ImageDraw.Draw(text_image) + x, y, w, h = draw.textbbox(xy=(0, 0), text=text, font=font) + text_image = Image.new("RGBA", (w, h), (255, 255, 255, 0)) + draw = ImageDraw.Draw(text_image) + draw.text((-x / 2, -y / 2), text, (0, 0, 0, 255), font=font, align=align) + + text_image_ = resize_and_pad_image2(text_image.convert('RGB'), (288, 48)) + # import pdb; pdb.set_trace() + text_list.append(np.array(text_image_)) + + text_image = resize_and_pad_image(text_image, (width, height)) + text_image = text_image.rotate(angle=-yaw, expand=True, fillcolor=(255, 255, 255, 0)) + # image = Image.new("RGB", (w, h), "white") + # draw = ImageDraw.Draw(image) + background.paste(text_image, (top_left_x, top_left_y), mask=text_image) + + return background, text_list + + +# [{'width': 0.1601562201976776, 'ratio': 81.99999451637203, 'yaw': 0.0, 'top_left_x': 0.712890625, 'top_left_y': 0.0}, +# {'width': 0.134765625, 'ratio': 34.5, 'yaw': 0.0, 'top_left_x': 0.4453125, 'top_left_y': 0.0}, + + +def insert_spaces(string, nSpace): + if nSpace == 0: + return string + new_string = "" + for char in string: + new_string += char + " " * nSpace + return new_string[:-nSpace] + + +def draw_glyph(text, font='simfang.ttf'): + if isinstance(font, str): + font = ImageFont.truetype(font, encoding='utf-8', size=512) + g_size = 50 + W, H = (512, 80) + new_font = font.font_variant(size=g_size) + img = Image.new(mode='1', size=(W, H), color=0) + draw = ImageDraw.Draw(img) + left, top, right, bottom = new_font.getbbox(text) + text_width = max(right-left, 5) + text_height = max(bottom - top, 5) + ratio = min(W*0.9/text_width, H*0.9/text_height) + new_font = font.font_variant(size=int(g_size*ratio)) + + text_width, text_height = new_font.getsize(text) + offset_x, offset_y = new_font.getoffset(text) + x = (img.width - text_width) // 2 + y = (img.height - text_height) // 2 - offset_y//2 + draw.text((x, y), text, font=new_font, fill='white') + img = np.expand_dims(np.array(img), axis=2).astype(np.float64) + + return img + + +def draw_glyph2(text, polygon, font='simfang.ttf', vertAng=10, scale=1, width=1024, height=1024, add_space=True): + if isinstance(font, str): + font = ImageFont.truetype(font, encoding='utf-8', size=60) + enlarge_polygon = polygon*scale + rect = cv2.minAreaRect(enlarge_polygon) + box = cv2.boxPoints(rect) + box = np.int0(box) + w, h = rect[1] + angle = rect[2] + if angle < -45: + angle += 90 + angle = -angle + if w < h: + angle += 90 + + vert = False + if (abs(angle) % 90 < vertAng or abs(90-abs(angle) % 90) % 90 < vertAng): + _w = max(box[:, 0]) - min(box[:, 0]) + _h = max(box[:, 1]) - min(box[:, 1]) + if _h >= _w: + vert = True + angle = 0 + + img = np.zeros((height*scale, width*scale, 3), np.uint8) + img = Image.fromarray(img) + + # infer font size + image4ratio = Image.new("RGB", img.size, "white") + draw = ImageDraw.Draw(image4ratio) + _, _, _tw, _th = draw.textbbox(xy=(0, 0), text=text, font=font) + text_w = min(w, h) * (_tw / _th) + if text_w <= max(w, h): + # add space + if len(text) > 1 and not vert and add_space: + for i in range(1, 100): + text_space = insert_spaces(text, i) + _, _, _tw2, _th2 = draw.textbbox(xy=(0, 0), text=text_space, font=font) + if min(w, h) * (_tw2 / _th2) > max(w, h): + break + text = insert_spaces(text, i-1) + font_size = min(w, h)*0.80 + else: + # shrink = 0.75 if vert else 0.85 + shrink = 1.0 + font_size = min(w, h) / (text_w/max(w, h)) * shrink + new_font = font.font_variant(size=int(font_size)) + + left, top, right, bottom = new_font.getbbox(text) + text_width = right-left + text_height = bottom - top + + layer = Image.new('RGBA', img.size, (0, 0, 0, 0)) + draw = ImageDraw.Draw(layer) + if not vert: + draw.text((rect[0][0]-text_width//2, rect[0][1]-text_height//2-top), text, font=new_font, fill=(255, 255, 255, 255)) + else: + x_s = min(box[:, 0]) + _w//2 - text_height//2 + y_s = min(box[:, 1]) + for c in text: + draw.text((x_s, y_s), c, font=new_font, fill=(255, 255, 255, 255)) + _, _t, _, _b = new_font.getbbox(c) + y_s += _b + + rotated_layer = layer.rotate(angle, expand=1, center=(rect[0][0], rect[0][1])) + + x_offset = int((img.width - rotated_layer.width) / 2) + y_offset = int((img.height - rotated_layer.height) / 2) + img.paste(rotated_layer, (x_offset, y_offset), rotated_layer) + img = np.expand_dims(np.array(img.convert('1')), axis=2).astype(np.float64) + + return img diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/util.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/util.py new file mode 100644 index 0000000000000000000000000000000000000000..e0b217ef9adf92dd5b1fe0debcfb07d0f241a4cb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/annotator/util.py @@ -0,0 +1,98 @@ +import random + +import numpy as np +import cv2 +import os + + +annotator_ckpts_path = os.path.join(os.path.dirname(__file__), 'ckpts') + + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + + +def resize_image(input_image, resolution): + H, W, C = input_image.shape + H = float(H) + W = float(W) + k = float(resolution) / min(H, W) + H *= k + W *= k + H = int(np.round(H / 64.0)) * 64 + W = int(np.round(W / 64.0)) * 64 + img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA) + return img + + +def nms(x, t, s): + x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s) + + f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8) + f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8) + f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8) + + y = np.zeros_like(x) + + for f in [f1, f2, f3, f4]: + np.putmask(y, cv2.dilate(x, kernel=f) == x, x) + + z = np.zeros_like(y, dtype=np.uint8) + z[y > t] = 255 + return z + + +def make_noise_disk(H, W, C, F): + noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) + noise = cv2.resize(noise, (W + 2 * F, H + 2 * F), interpolation=cv2.INTER_CUBIC) + noise = noise[F: F + H, F: F + W] + noise -= np.min(noise) + noise /= np.max(noise) + if C == 1: + noise = noise[:, :, None] + return noise + + +def min_max_norm(x): + x -= np.min(x) + x /= np.maximum(np.max(x), 1e-5) + return x + + +def safe_step(x, step=2): + y = x.astype(np.float32) * float(step + 1) + y = y.astype(np.int32).astype(np.float32) / float(step) + return y + + +def img2mask(img, H, W, low=10, high=90): + assert img.ndim == 3 or img.ndim == 2 + assert img.dtype == np.uint8 + + if img.ndim == 3: + y = img[:, :, random.randrange(0, img.shape[2])] + else: + y = img + + y = cv2.resize(y, (W, H), interpolation=cv2.INTER_CUBIC) + + if random.uniform(0, 1) < 0.5: + y = 255 - y + + return y < np.percentile(y, random.randrange(low, high)) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/config.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/config.py new file mode 100644 index 0000000000000000000000000000000000000000..3310ae5e29869f0878eeac1ad2a3fa4d89ee3b3a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/config.py @@ -0,0 +1,213 @@ +import argparse + +from .constants import * +from .modules.models import HUNYUAN_DIT_CONFIG, HUNYUAN_DIT_MODELS +from .diffusion.gaussian_diffusion import ModelVarType + +import deepspeed + + +def model_var_type(value): + try: + return ModelVarType[value] + except KeyError: + raise ValueError(f"Invalid choice '{value}', valid choices are {[v.name for v in ModelVarType]}") + + +def get_args(default_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("--task-flag", type=str) + + # General Setting + parser.add_argument("--batch-size", type=int, default=1, help="Per-GPU batch size") + parser.add_argument('--seed', type=int, default=42, help="A seed for all the prompts.") + parser.add_argument("--use-fp16", action="store_true", help="Use FP16 precision.") + parser.add_argument("--no-fp16", dest="use_fp16", action="store_false") + parser.set_defaults(use_fp16=True) + parser.add_argument("--extra-fp16", action="store_true", help="Use extra fp16 for vae and text_encoder.") + + # HunYuan-DiT + parser.add_argument("--model", type=str, choices=list(HUNYUAN_DIT_CONFIG.keys()), default='DiT-g/2') + parser.add_argument("--image-size", type=int, nargs='+', default=[1024, 1024], + help='Image size (h, w). If a single value is provided, the image will be treated to ' + '(value, value).') + parser.add_argument("--qk-norm", action="store_true", + help="Query Key normalization. See http://arxiv.org/abs/2302.05442 for details.") + parser.set_defaults(qk_norm=True) + parser.add_argument("--norm", type=str, choices=["rms", "layer"], default="layer", help="Normalization layer type") + parser.add_argument("--text-states-dim", type=int, default=1024, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len", type=int, default=77, help="Token length of CLIP text encoder output.") + parser.add_argument("--text-states-dim-t5", type=int, default=2048, help="Hidden size of CLIP text encoder.") + parser.add_argument("--text-len-t5", type=int, default=256, help="Token length of T5 text encoder output.") + + # LoRA config + parser.add_argument("--training-parts", type=str, default='all', choices=['all', 'lora'], help="Training parts") + parser.add_argument("--rank", type=int, default=64, help="Rank of LoRA") + parser.add_argument("--lora-ckpt", type=str, default=None, help="LoRA checkpoint") + parser.add_argument('--target-modules', type=str, nargs='+', default=['Wqkv', 'q_proj', 'kv_proj', 'out_proj'], + help="Target modules for LoRA fine tune") + parser.add_argument("--output-merge-path", type=str, default=None, help="Output path for merged model") + + # controlnet config + parser.add_argument("--control-type", type=str, default='canny', choices=['canny', 'depth', 'pose'], + help="Controlnet condition type") + parser.add_argument("--control-weight", type=float, default=1.0, + help="Controlnet weight, You can use a float to specify the weight for all layers, or use a list to separately specify the weight for each layer, for example, '[1.0 * (0.825 ** float(19 - i)) for i in range(19)]'") + parser.add_argument("--condition-image-path", type=str, default=None, help="Inference condition image path") + + # Diffusion + parser.add_argument("--learn-sigma", action="store_true", help="Learn extra channels for sigma.") + parser.add_argument("--no-learn-sigma", dest="learn_sigma", action="store_false") + parser.set_defaults(learn_sigma=True) + parser.add_argument("--predict-type", type=str, choices=list(PREDICT_TYPE), default="v_prediction", + help="Diffusion predict type") + parser.add_argument("--noise-schedule", type=str, choices=list(NOISE_SCHEDULES), default="scaled_linear", + help="Noise schedule") + parser.add_argument("--beta-start", type=float, default=0.00085, help="Beta start value") + parser.add_argument("--beta-end", type=float, default=0.03, help="Beta end value") + parser.add_argument("--sigma-small", action="store_true") + parser.add_argument("--mse-loss-weight-type", type=str, default="constant", + help="Min-SNR-gamma. Can be constant or min_snr_ where gamma is a integer. 5 is recommended in the paper.") + parser.add_argument("--model-var-type", type=model_var_type, default=None, help="Specify the model variable type.") + parser.add_argument("--noise-offset", type=float, default=0.0, help="Add extra noise to the input image.") + + # ======================================================================================================== + # Inference + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--prompt", type=str, default="一只小猫", help="The prompt for generating images.") + parser.add_argument("--model-root", type=str, default="ckpts", help="Model root path.") + + # Model setting + parser.add_argument("--load-key", type=str, choices=["ema", "module", "distill", 'merge'], default="ema", + help="Load model key for HunYuanDiT checkpoint.") + parser.add_argument('--size-cond', type=int, nargs='+', default=[1024, 1024], + help="Size condition used in sampling. 2 values are required for height and width. " + "If a single value is provided, the image will be treated to (value, value).") + parser.add_argument('--target-ratios', type=str, nargs='+', default=None, + help="Target ratios for multi-resolution training.") + parser.add_argument("--cfg-scale", type=float, default=6.0, help="Guidance scale for classifier-free.") + parser.add_argument("--negative", type=str, default=None, help="Negative prompt.") + + # Acceleration + parser.add_argument("--infer-mode", type=str, choices=["fa", "torch", "trt"], default="torch", + help="Inference mode") + parser.add_argument("--onnx-workdir", type=str, default="onnx_model", help="Path to save ONNX model") + + # Sampling + parser.add_argument("--sampler", type=str, choices=SAMPLER_FACTORY, default="ddpm", help="Diffusion sampler") + parser.add_argument("--infer-steps", type=int, default=100, help="Inference steps") + + # Prompt enhancement + parser.add_argument("--enhance", action="store_true", help="Enhance prompt with mllm.") + parser.add_argument("--no-enhance", dest="enhance", action="store_false") + parser.add_argument("--load-4bit", help="load DialogGen model with 4bit quantization.", action="store_true") + parser.set_defaults(enhance=True) + + # App + parser.add_argument("--lang", type=str, default="zh", choices=["zh", "en"], help="Language") + + # ======================================================================================================== + # Training + # ======================================================================================================== + + # Basic Setting + parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--epochs", type=int, default=1400) + parser.add_argument("--max-training-steps", type=int, default=10_000_000) + parser.add_argument("--gc-interval", type=int, default=40, + help='To address the memory bottleneck encountered during the preprocessing of the dataset,' + ' memory fragments are reclaimed here by invoking the gc.collect() function.') + parser.add_argument("--log-every", type=int, default=100) + parser.add_argument("--ckpt-every", type=int, default=100_000, help="Create a ckpt every a few steps.") + parser.add_argument("--ckpt-latest-every", type=int, default=10_000, + help="Create a ckpt named `latest.pt` every a few steps.") + parser.add_argument("--num-workers", type=int, default=4) + parser.add_argument("--global-seed", type=int, default=1234) + parser.add_argument("--warmup-min-lr", type=float, default=1e-6) + parser.add_argument("--warmup-num-steps", type=float, default=0) + parser.add_argument("--weight-decay", type=float, default=0, help="weight-decay in optimizer") + parser.add_argument("--rope-img", type=str, default=None, choices=['extend', 'base512', 'base1024'], + help="Extend or interpolate the positional embedding of the image.") + parser.add_argument("--rope-real", action="store_true", + help="Use real part and imaginary part separately for RoPE.") + + # Classifier-free + parser.add_argument("--uncond-p", type=float, default=0.2, + help="The probability of dropping training text used for CLIP feature extraction") + parser.add_argument("--uncond-p-t5", type=float, default=0.2, + help="The probability of dropping training text used for mT5 feature extraction") + + # Directory + parser.add_argument("--results-dir", type=str, default="results") + parser.add_argument("--resume", type=str, default=None, help="Resume experiment from a checkpoint") + parser.add_argument("--no-strict", dest="strict", action="store_false", help="Strict loading of checkpoint") + parser.set_defaults(strict=True) + parser.add_argument("--resume-deepspeed", action="store_true", + help="Resume model and ema states from a checkpoint saved by Deepspeed version of DIT.") + parser.add_argument("--resume-split", action="store_true", + help="Resume model and ema states from two checkpoint separated from DeepSpeed ckpt.") + parser.add_argument("--ema-to-module", action="store_true", + help="If true, initialize the module with EMA weights.") + parser.add_argument("--module-to-ema", action="store_true", + help="if true, initialize the ema with Module weights.") + + # Dataset + parser.add_argument("--index-file", type=str, nargs='+', + help="During training, provide a JSON file with data indices.") + parser.add_argument("--random-flip", action="store_true", help="Random flip image") + parser.add_argument("--reset-loader", action="store_true", + help="Reset the data loader. It is useful when resuming from a checkpoint but switch to a new dataset.") + parser.add_argument("--multireso", action="store_true", help="Use multi-resolution training.") + parser.add_argument("--reso-step", type=int, default=None, help="Step size for multi-resolution training.") + + # Additional condition + parser.add_argument("--random-shrink-size-cond", action="store_true", + help="Randomly shrink the original size condition.") + parser.add_argument("--merge-src-cond", action="store_true", help="Merge the source condition into a single value.") + + # EMA Model + parser.add_argument("--use-ema", action="store_true", help="Use EMA model") + parser.add_argument("--ema-dtype", type=str, choices=['fp16', 'fp32', 'none'], default="none", + help="EMA data type. If none, use the same data type as the model.") + parser.add_argument("--ema-decay", type=float, default=None, + help="EMA decay rate. If None, use the default value of the model.") + parser.add_argument("--ema-warmup", action="store_true", + help="EMA warmup. If True, perform ema_decay warmup from 0 to ema_decay.") + parser.add_argument("--ema-warmup-power", type=float, default=None, + help="EMA power. If None, use the default value of the model.") + parser.add_argument("--ema-reset-decay", action="store_true", + help="Reset EMA decay to 0 and restart increasing the EMA decay." + "Only works when --ema-warmup is enabled.") + # Acceleration + parser.add_argument("--use-flash-attn", action="store_true", help="During training, " + "flash attention is used to accelerate training.") + parser.add_argument("--no-flash-attn", dest="use_flash_attn", + action="store_false", + help="During training, flash attention is not used to accelerate training.") + parser.add_argument("--use-zero-stage", type=int, default=1, help="Use AngelPTM zero stage. Support 2 and 3") + parser.add_argument("--grad-accu-steps", type=int, default=1, help="Gradient accumulation steps.") + + # ======================================================================================================== + # Deepspeed config + # ======================================================================================================== + parser = deepspeed.add_config_arguments(parser) + parser.add_argument('--local_rank', type=int, default=None, + help='local rank passed from distributed launcher.') + parser.add_argument('--deepspeed-optimizer', action='store_true', + help='Switching to the optimizers in DeepSpeed') + parser.add_argument('--remote-device', type=str, default='none', choices=['none', 'cpu', 'nvme'], + help='Remote device for ZeRO-3 initialized parameters.') + parser.add_argument('--zero-stage', type=int, default=1) + parser.add_argument("--async-ema", action="store_true", help="Whether to use multi stream to excut EMA.") + + # 模型迁移新增参数 + parser.add_argument("--cal-e2e", dest="cal_e2e", action="store_true") + parser.add_argument("--profiling", dest="profiling", action="store_true") + parser.add_argument("--seed-all", dest="seed_all", action="store_true") + parser.add_argument("--autocast-dtype", type=str, default="bf16", choices=['bf16', 'fp16'], help="the dtype to autocast") + + args = parser.parse_args(default_args) + + return args diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/constants.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..a46921578127b3c81ffea3d1bfb78d06a35421af --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/constants.py @@ -0,0 +1,82 @@ +import torch + +# ======================================================= + +NOISE_SCHEDULES = { + "linear", + "scaled_linear", + "squaredcos_cap_v2", +} + +PREDICT_TYPE = { + "epsilon", + "sample", + "v_prediction", +} + +# ======================================================= + +NEGATIVE_PROMPT = '错误的眼睛，糟糕的人脸，毁容，糟糕的艺术，变形，多余的肢体，模糊的颜色，模糊，重复，病态，残缺，' + +# ======================================================= +TRT_MAX_BATCH_SIZE = 1 +TRT_MAX_WIDTH = 1280 +TRT_MAX_HEIGHT = 1280 + +# ======================================================= +# Constants about models +# ======================================================= + +VAE_EMA_PATH = "ckpts/t2i/sdxl-vae-fp16-fix" +TOKENIZER = "ckpts/t2i/tokenizer" +TEXT_ENCODER = 'ckpts/t2i/clip_text_encoder' +T5_ENCODER = { + 'MT5': 'ckpts/t2i/mt5', + 'attention_mask': True, + 'layer_index': -1, + 'attention_pool': True, + 'torch_dtype': torch.float16, + 'learnable_replace': True +} + +SAMPLER_FACTORY = { + 'ddpm': { + 'scheduler': 'DDPMScheduler', + 'name': 'DDPM', + 'kwargs': { + 'steps_offset': 1, + 'clip_sample': False, + 'clip_sample_range': 1.0, + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + } + }, + 'ddim': { + 'scheduler': 'DDIMScheduler', + 'name': 'DDIM', + 'kwargs': { + 'steps_offset': 1, + 'clip_sample': False, + 'clip_sample_range': 1.0, + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + } + }, + 'dpmms': { + 'scheduler': 'DPMSolverMultistepScheduler', + 'name': 'DPMMS', + 'kwargs': { + 'beta_schedule': 'scaled_linear', + 'beta_start': 0.00085, + 'beta_end': 0.03, + 'prediction_type': 'v_prediction', + 'trained_betas': None, + 'solver_order': 2, + 'algorithm_type': 'dpmsolver++', + } + }, +} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/arrow_load_stream.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/arrow_load_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..ac2bd17ae04b0959ced080fae0c3345123bcc0e7 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/arrow_load_stream.py @@ -0,0 +1,255 @@ +import pickle +import random +from pathlib import Path +import ast +import numpy as np +import re +import json +import time +from functools import partial +from PIL import Image + +import torch +import torchvision.transforms as T +import torch.nn.functional as F +from torchvision.transforms import functional as TF +from torch.utils.data import Dataset + +from IndexKits.index_kits import ArrowIndexV2, MultiResolutionBucketIndexV2, MultiIndexV2 + + +class TextImageArrowStream(Dataset): + def __init__(self, + args, + resolution=512, + random_flip=None, + enable_CN=True, + log_fn=print, + index_file=None, + multireso=False, + batch_size=-1, + world_size=1, + random_shrink_size_cond=False, + merge_src_cond=False, + uncond_p=0.0, + text_ctx_len=77, + tokenizer=None, + uncond_p_t5=0.0, + text_ctx_len_t5=256, + tokenizer_t5=None, + ): + self.args = args + self.resolution = resolution + self.log_fn = lambda x: log_fn(f" {Path(__file__).stem} | " + x) + + self.random_flip = random_flip + # If true, the Chinese prompt from the `text_zh` column will be taken from the arrow file; + # otherwise, the English prompt from the `text_en` column will be taken, + # provided that `text_zh` or `text_en` exists in the arrow file. + self.enable_CN = enable_CN + self.index_file = index_file + self.multireso = multireso + self.batch_size = batch_size + self.world_size = world_size + self.index_manager = self.load_index() + + # clip params + self.uncond_p = uncond_p + self.text_ctx_len = text_ctx_len + self.tokenizer = tokenizer + + # t5 params + self.uncond_p_t5 = uncond_p_t5 + self.text_ctx_len_t5 = text_ctx_len_t5 + self.tokenizer_t5 = tokenizer_t5 + + # size condition + self.random_shrink_size_cond = random_shrink_size_cond + self.merge_src_cond = merge_src_cond + + assert isinstance(resolution, int), f"resolution must be an integer, got {resolution}" + self.flip_norm = T.Compose( + [ + T.RandomHorizontalFlip() if self.random_flip else T.Lambda(lambda x: x), + T.ToTensor(), + T.Normalize([0.5], [0.5]), + ] + ) + + # show info + if self.merge_src_cond: + self.log_fn("Enable merging src condition: (oriW, oriH) --> ((WH)**0.5, (WH)**0.5)") + + self.log_fn("Enable image_meta_size condition (original_size, target_size, crop_coords)") + self.log_fn(f"Image_transforms: {self.flip_norm}") + + def load_index(self): + multireso = self.multireso + index_file = self.index_file + batch_size = self.batch_size + world_size = self.world_size + + if multireso: + if isinstance(index_file, (list, tuple)): + if len(index_file) > 1: + raise ValueError(f"When enabling multireso, index_file should be a single file, but got {index_file}") + index_file = index_file[0] + index_manager = MultiResolutionBucketIndexV2(index_file, batch_size, world_size) + self.log_fn(f"Using MultiResolutionBucketIndexV2: {len(index_manager):,}") + else: + if isinstance(index_file, str): + index_file = [index_file] + if len(index_file) == 1: + index_manager = ArrowIndexV2(index_file[0]) + self.log_fn(f"Using ArrowIndexV2: {len(index_manager):,}") + else: + index_manager = MultiIndexV2(index_file) + self.log_fn(f"Using MultiIndexV2: {len(index_manager):,}") + + return index_manager + + def shuffle(self, seed, fast=False): + self.index_manager.shuffle(seed, fast=fast) + + def get_raw_image(self, index, image_key="image"): + try: + ret = self.index_manager.get_image(index, image_key) + except Exception as e: + self.log_fn(f'get_raw_image | Error: {e}') + ret = Image.new("RGB", (256, 256), (255, 255, 255)) + return ret + + @staticmethod + def random_crop_image(image, origin_size, target_size): + aspect_ratio = float(origin_size[0]) / float(origin_size[1]) + if origin_size[0] < origin_size[1]: + new_width = target_size[0] + new_height = int(new_width / aspect_ratio) + else: + new_height = target_size[1] + new_width = int(new_height * aspect_ratio) + + image = image.resize((new_width, new_height), Image.LANCZOS) + + if new_width > target_size[0]: + x_start = random.randint(0, new_width - target_size[0]) + y_start = 0 + else: + x_start = 0 + y_start = random.randint(0, new_height - target_size[1]) + image_crop = image.crop((x_start, y_start, x_start + target_size[0], y_start + target_size[1])) + crops_coords_top_left = (x_start, y_start) + return image_crop, crops_coords_top_left + + def get_style(self, index): + "Here we use a default learned embedder layer for future extension." + style = 0 + return style + + def get_image_with_hwxy(self, index, image_key="image"): + + image = self.get_raw_image(index, image_key=image_key) + origin_size = image.size + + if self.multireso: + target_size = self.index_manager.get_target_size(index) + image, crops_coords_top_left = self.index_manager.resize_and_crop( + image, target_size, resample=Image.LANCZOS, crop_type='random') + image_tensor = self.flip_norm(image) + else: + target_size = (self.resolution, self.resolution) + image_crop, crops_coords_top_left = self.random_crop_image(image, origin_size, target_size) + image_tensor = self.flip_norm(image_crop) + + if self.random_shrink_size_cond: + origin_size = (1024 if origin_size[0] < 1024 else origin_size[0], + 1024 if origin_size[1] < 1024 else origin_size[1]) + if self.merge_src_cond: + val = (origin_size[0] * origin_size[1]) ** 0.5 + origin_size = (val, val) + + image_meta_size = tuple(origin_size) + tuple(target_size) + tuple(crops_coords_top_left) + kwargs = { + 'image_meta_size': image_meta_size, + } + + style = self.get_style(index) + kwargs['style'] = style + + return image_tensor, kwargs + + def get_text_info_with_encoder(self, description): + pad_num = 0 + text_inputs = self.tokenizer( + description, + padding="max_length", + max_length=self.text_ctx_len, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids[0] + attention_mask = text_inputs.attention_mask[0].bool() + if pad_num > 0: + attention_mask[1:pad_num + 1] = False + return description, text_input_ids, attention_mask + + def fill_t5_token_mask(self, fill_tensor, fill_number, setting_length): + fill_length = setting_length - fill_tensor.shape[1] + if fill_length > 0: + fill_tensor = torch.cat((fill_tensor, fill_number * torch.ones(1, fill_length)), dim=1) + return fill_tensor + + def get_text_info_with_encoder_t5(self, description_t5): + text_tokens_and_mask = self.tokenizer_t5( + description_t5, + max_length=self.text_ctx_len_t5, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors='pt' + ) + text_input_ids_t5=self.fill_t5_token_mask(text_tokens_and_mask["input_ids"], fill_number=1, setting_length=self.text_ctx_len_t5).long() + attention_mask_t5=self.fill_t5_token_mask(text_tokens_and_mask["attention_mask"], fill_number=0, setting_length=self.text_ctx_len_t5).bool() + return description_t5, text_input_ids_t5, attention_mask_t5 + + def get_original_text(self, ind): + text = self.index_manager.get_attribute(ind, 'text_zh' if self.enable_CN else 'text_en') + text = str(text).strip() + return text + + def get_text(self, ind): + text = self.get_original_text(ind) + if text == '': + text = '随机生成一张图片' + return text + + def __getitem__(self, ind): + # Get text + if random.random() < self.uncond_p: + description = "" + else: + description = self.get_text(ind) + + # Get text for t5 + if random.random() < self.uncond_p_t5: + description_t5 = "" + else: + description_t5 = self.get_text(ind) + + original_pil_image, kwargs = self.get_image_with_hwxy(ind) + + # Use encoder to embed tokens online + text, text_embedding, text_embedding_mask = self.get_text_info_with_encoder(description) + + text_t5, text_embedding_t5, text_embedding_mask_t5 = self.get_text_info_with_encoder_t5(description_t5) + return ( + original_pil_image, + text_embedding.clone().detach(), + text_embedding_mask.clone().detach(), + text_embedding_t5.clone().detach(), + text_embedding_mask_t5.clone().detach(), + {k: torch.tensor(np.array(v)).clone().detach() for k, v in kwargs.items()}, + ) + + def __len__(self): + return len(self.index_manager) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/csv2arrow.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/csv2arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..1ffe81d7810d645297364ec13c40e1d62edc4088 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/data_loader/csv2arrow.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +import datetime +import gc +import os +import time +from multiprocessing import Pool +import subprocess +import pandas as pd +import pyarrow as pa +from tqdm import tqdm +import hashlib +from PIL import Image +import sys + + +def parse_data(data): + try: + img_path = data[0] + + with open(img_path, "rb") as fp: + image = fp.read() + md5 = hashlib.md5(image).hexdigest() + + with Image.open(img_path) as f: + width, height = f.size + + return [data[1], md5, width, height, image] + + except Exception as e: + print(f'error: {e}') + return + +def make_arrow(csv_root, dataset_root, start_id=0, end_id=-1): + print(csv_root) + arrow_dir = dataset_root + print(arrow_dir) + + if not os.path.exists(arrow_dir): + os.makedirs(arrow_dir) + + data = pd.read_csv(csv_root) + data = data[["image_path", "text_zh"]] + columns_list = data.columns.tolist() + columns_list.append("image") + + if end_id < 0: + end_id = len(data) + print(f'start_id:{start_id} end_id:{end_id}') + data = data[start_id:end_id] + num_slice = 5000 + start_sub = int(start_id / num_slice) + sub_len = int(len(data) // num_slice) # if int(len(data) // num_slice) else 1 + subs = list(range(sub_len + 1)) + for sub in tqdm(subs): + arrow_path = os.path.join(arrow_dir, '{}.arrow'.format(str(sub + start_sub).zfill(5))) + if os.path.exists(arrow_path): + continue + print(f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')} start {sub + start_sub}") + + sub_data = data[sub * num_slice: (sub + 1) * num_slice].values + + bs = pool.map(parse_data, sub_data) + bs = [b for b in bs if b] + print(f'length of this arrow:{len(bs)}') + + columns_list = ["text_zh", "md5", "width", "height", "image"] + dataframe = pd.DataFrame(bs, columns=columns_list) + table = pa.Table.from_pandas(dataframe) + + os.makedirs(dataset_root, exist_ok=True) + with pa.OSFile(arrow_path, "wb") as sink: + with pa.RecordBatchFileWriter(sink, table.schema) as writer: + writer.write_table(table) + del dataframe + del table + del bs + gc.collect() + + +if __name__ == '__main__': + + if len(sys.argv) != 4: + print("Usage: python hydit/data_loader/csv2arrow.py ${csv_root} ${output_arrow_data_path} ${pool_num}") + print("csv_root: The path to your created CSV file. For more details, see https://github.com/Tencent/HunyuanDiT?tab=readme-ov-file#truck-training") + print("output_arrow_data_path: The path for storing the created Arrow file") + print("pool_num: The number of processes, used for multiprocessing. If you encounter memory issues, you can set pool_num to 1") + sys.exit(1) + csv_root = sys.argv[1] + output_arrow_data_path = sys.argv[2] + pool_num = int(sys.argv[3]) + pool = Pool(pool_num) + make_arrow(csv_root, output_arrow_data_path) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a681113da5c6fc6a1cc216e2270171c4780898 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/__init__.py @@ -0,0 +1,49 @@ +from . import gaussian_diffusion as gd +from .respace import SpacedDiffusion, space_timesteps + + +def create_diffusion( + *, + steps=1000, + learn_sigma=True, + sigma_small=False, + noise_schedule="linear", + use_kl=False, + predict_type='epsilon', + rescale_timesteps=False, + rescale_learned_sigmas=False, + timestep_respacing="", + mse_loss_weight_type='constant', + beta_start=0.0001, + beta_end=0.02, + noise_offset=0.0, +): + betas = gd.get_named_beta_schedule(noise_schedule, steps, beta_start, beta_end) + if use_kl: + loss_type = gd.LossType.RESCALED_KL + elif rescale_learned_sigmas: + loss_type = gd.LossType.RESCALED_MSE + else: + loss_type = gd.LossType.MSE + if timestep_respacing is None or timestep_respacing == "": + timestep_respacing = [steps] + mean_type = gd.predict_type_dict[predict_type] + + return SpacedDiffusion( + use_timesteps=space_timesteps(steps, timestep_respacing), + betas=betas, + model_mean_type=mean_type, + model_var_type=( + ( + gd.ModelVarType.FIXED_LARGE + if not sigma_small + else gd.ModelVarType.FIXED_SMALL + ) + if not learn_sigma + else gd.ModelVarType.LEARNED_RANGE + ), + loss_type=loss_type, + rescale_timesteps=rescale_timesteps, + mse_loss_weight_type=mse_loss_weight_type, + noise_offset=noise_offset, + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/diffusion_utils.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/diffusion_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9c62d5b0f914eb8bef789d3612e3c48d68ec0cef --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/diffusion_utils.py @@ -0,0 +1,69 @@ +import torch as th +import numpy as np + + +def normal_kl(mean1, logvar1, mean2, logvar2): + """ + Compute the KL divergence between two gaussians. + Shapes are automatically broadcasted, so batches can be compared to + scalars, among other use cases. + """ + tensor = None + for obj in (mean1, logvar1, mean2, logvar2): + if isinstance(obj, th.Tensor): + tensor = obj + break + assert tensor is not None, "at least one argument must be a Tensor" + + # Force variances to be Tensors. Broadcasting helps convert scalars to + # Tensors, but it does not work for th.exp(). + logvar1, logvar2 = [ + x if isinstance(x, th.Tensor) else th.tensor(x).to(tensor) + for x in (logvar1, logvar2) + ] + + return 0.5 * ( + -1.0 + + logvar2 + - logvar1 + + th.exp(logvar1 - logvar2) + + ((mean1 - mean2) ** 2) * th.exp(-logvar2) + ) + + +def approx_standard_normal_cdf(x): + """ + A fast approximation of the cumulative distribution function of the + standard normal. + """ + return 0.5 * (1.0 + th.tanh(np.sqrt(2.0 / np.pi) * (x + 0.044715 * th.pow(x, 3)))) + + +def discretized_gaussian_log_likelihood(x, *, means, log_scales): + """ + Compute the log-likelihood of a Gaussian distribution discretizing to a + given image. + :param x: the target images. It is assumed that this was uint8 values, + rescaled to the range [-1, 1]. + :param means: the Gaussian mean Tensor. + :param log_scales: the Gaussian log stddev Tensor. + :return: a tensor like x of log probabilities (in nats). + """ + assert x.shape == means.shape == log_scales.shape + centered_x = x - means + inv_stdv = th.exp(-log_scales) + plus_in = inv_stdv * (centered_x + 1.0 / 255.0) + cdf_plus = approx_standard_normal_cdf(plus_in) + min_in = inv_stdv * (centered_x - 1.0 / 255.0) + cdf_min = approx_standard_normal_cdf(min_in) + log_cdf_plus = th.log(cdf_plus.clamp(min=1e-12)) + log_one_minus_cdf_min = th.log((1.0 - cdf_min).clamp(min=1e-12)) + cdf_delta = cdf_plus - cdf_min + log_probs = th.where( + x < -0.999, + log_cdf_plus, + th.where(x > 0.999, log_one_minus_cdf_min, th.log(cdf_delta.clamp(min=1e-12))), + ) + assert log_probs.shape == x.shape + return log_probs + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/gaussian_diffusion.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/gaussian_diffusion.py new file mode 100644 index 0000000000000000000000000000000000000000..636d0f678cc585832f1ad09873928b5be94f62fd --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/gaussian_diffusion.py @@ -0,0 +1,1384 @@ +import math + +import numpy as np +import torch as th +import enum + +from .diffusion_utils import discretized_gaussian_log_likelihood, normal_kl +from ..utils.tools import assert_shape + +from utils.npu_utils import is_npu_available + + +def mean_flat(tensor): + """ + Take the mean over all non-batch dimensions. + """ + return tensor.mean(dim=list(range(1, len(tensor.shape)))) + + +class ModelMeanType(enum.Enum): + """ + Which type of output the model predicts. + """ + + PREVIOUS_X = enum.auto() # the model predicts x_{t-1} + START_X = enum.auto() # the model predicts x_0 + EPSILON = enum.auto() # the model predicts epsilon + VELOCITY = enum.auto() # the model predicts v + + +predict_type_dict = { + 'epsilon': ModelMeanType.EPSILON, + 'sample': ModelMeanType.START_X, + 'v_prediction': ModelMeanType.VELOCITY, +} + + +class ModelVarType(enum.Enum): + """ + What is used as the model's output variance. + The LEARNED_RANGE option has been added to allow the model to predict + values between FIXED_SMALL and FIXED_LARGE, making its job easier. + """ + + LEARNED = enum.auto() + FIXED_SMALL = enum.auto() + FIXED_LARGE = enum.auto() + LEARNED_RANGE = enum.auto() + + +class LossType(enum.Enum): + MSE = enum.auto() # use raw MSE loss (and KL when learning variances) + RESCALED_MSE = ( + enum.auto() + ) # use raw MSE loss (with RESCALED_KL when learning variances) + KL = enum.auto() # use the variational lower-bound + RESCALED_KL = enum.auto() # like KL, but rescale to estimate the full VLB + + def is_vb(self): + return self == LossType.KL or self == LossType.RESCALED_KL + + +def _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, warmup_frac): + betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64) + warmup_time = int(num_diffusion_timesteps * warmup_frac) + betas[:warmup_time] = np.linspace(beta_start, beta_end, warmup_time, dtype=np.float64) + return betas + + +def get_beta_schedule(beta_schedule, *, beta_start, beta_end, num_diffusion_timesteps): + """ + This is the deprecated API for creating beta schedules. + See get_named_beta_schedule() for the new library of schedules. + """ + if beta_schedule == "quad": + betas = ( + np.linspace( + beta_start ** 0.5, + beta_end ** 0.5, + num_diffusion_timesteps, + dtype=np.float64, + ) + ** 2 + ) + elif beta_schedule == "linear": + betas = np.linspace(beta_start, beta_end, num_diffusion_timesteps, dtype=np.float64) + elif beta_schedule == "warmup10": + betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.1) + elif beta_schedule == "warmup50": + betas = _warmup_beta(beta_start, beta_end, num_diffusion_timesteps, 0.5) + elif beta_schedule == "const": + betas = beta_end * np.ones(num_diffusion_timesteps, dtype=np.float64) + elif beta_schedule == "jsd": # 1/T, 1/(T-1), 1/(T-2), ..., 1 + betas = 1.0 / np.linspace( + num_diffusion_timesteps, 1, num_diffusion_timesteps, dtype=np.float64 + ) + else: + raise NotImplementedError(beta_schedule) + assert_shape(betas, (num_diffusion_timesteps,)) + return betas + + +def get_named_beta_schedule(schedule_name, num_diffusion_timesteps, beta_start=0.0001, beta_end=0.02): + """ + Get a pre-defined beta schedule for the given name. + The beta schedule library consists of beta schedules which remain similar + in the limit of num_diffusion_timesteps. + Beta schedules may be added, but should not be removed or changed once + they are committed to maintain backwards compatibility. + """ + if schedule_name == "linear": + # Linear schedule from Ho et al, extended to work for any number of + # diffusion steps. + scale = 1000 / num_diffusion_timesteps + return get_beta_schedule( + "linear", + beta_start=scale * beta_start, # DDPM + beta_end=scale * beta_end, # DDPM + num_diffusion_timesteps=num_diffusion_timesteps, # DDPM + ) + elif schedule_name == "scaled_linear": + return get_beta_schedule( + "quad", + beta_start=beta_start, # StableDiffusion, should be 0.00085 + beta_end=beta_end, # StableDiffusion, should be 0.012 + num_diffusion_timesteps=num_diffusion_timesteps, # StableDiffusion + ) + elif schedule_name == "squaredcos_cap_v2": + return betas_for_alpha_bar( + num_diffusion_timesteps, + lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2, + ) + else: + raise NotImplementedError(f"unknown beta schedule: {schedule_name}") + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, + which defines the cumulative product of (1-beta) over time from t = [0,1]. + + :param num_diffusion_timesteps: the number of betas to produce. + :param alpha_bar: a lambda that takes an argument t from 0 to 1 and + produces the cumulative product of (1-beta) up to that + part of the diffusion process. + :param max_beta: the maximum beta to use; use values lower than 1 to + prevent singularities. + """ + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return np.array(betas) + + +class GaussianDiffusion: + """ + Utilities for training and sampling diffusion models. + + Ported directly from here, and then adapted over time to further experimentation. + https://github.com/hojonathanho/diffusion/blob/1e0dceb3b3495bbe19116a5e1b3596cd0706c543/diffusion_tf/diffusion_utils_2.py#L42 + + :param betas: a 1-D numpy array of betas for each diffusion timestep, + starting at T and going to 1. + :param model_mean_type: a ModelMeanType determining what the model outputs. + :param model_var_type: a ModelVarType determining how variance is output. + :param loss_type: a LossType determining the loss function to use. + :param rescale_timesteps: if True, pass floating point timesteps into the + model so that they are always scaled like in the + original paper (0 to 1000). + """ + + def __init__( + self, + *, + betas, + model_mean_type, + model_var_type, + loss_type, + rescale_timesteps=False, + mse_loss_weight_type='constant', + noise_offset=0.0, + ): + self.model_mean_type = model_mean_type + self.model_var_type = model_var_type + self.loss_type = loss_type + self.rescale_timesteps = rescale_timesteps + + self.mse_loss_weight_type = mse_loss_weight_type + self.noise_offset = noise_offset + + # Use float64 for accuracy. + betas = np.array(betas, dtype=np.float64) + alphas = 1.0 - betas + alphas_cumprod = np.cumprod(alphas, axis=0) + + self.betas = betas + assert len(betas.shape) == 1, "betas must be 1-D" + assert (betas > 0).all() and (betas <= 1).all() + + self.num_timesteps = int(betas.shape[0]) + + self.alphas_cumprod = alphas_cumprod + self.alphas_cumprod_prev = np.append(1.0, self.alphas_cumprod[:-1]) + self.alphas_cumprod_next = np.append(self.alphas_cumprod[1:], 0.0) + assert_shape(self.alphas_cumprod_prev, (self.num_timesteps,)) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.sqrt_alphas_cumprod = np.sqrt(self.alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = np.sqrt(1.0 - self.alphas_cumprod) + self.log_one_minus_alphas_cumprod = np.log(1.0 - self.alphas_cumprod) + self.sqrt_recip_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod) + self.sqrt_recipm1_alphas_cumprod = np.sqrt(1.0 / self.alphas_cumprod - 1) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + self.posterior_variance = ( + betas * (1.0 - self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + ) + # log calculation clipped because the posterior variance is 0 at the + # beginning of the diffusion chain. + self.posterior_log_variance_clipped = np.log( + np.append(self.posterior_variance[1], self.posterior_variance[1:]) + ) if len(self.posterior_variance) > 1 else np.array([]) + + self.posterior_mean_coef1 = ( + betas * np.sqrt(self.alphas_cumprod_prev) / (1.0 - self.alphas_cumprod) + ) + self.posterior_mean_coef2 = ( + (1.0 - self.alphas_cumprod_prev) + * np.sqrt(alphas) + / (1.0 - self.alphas_cumprod) + ) + + self.sampler = { + "ddpm": self.p_sample_loop, + "ddim": self.ddim_sample_loop, + "plms": self.plms_sample_loop, + } + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + ) + variance = _extract_into_tensor(1.0 - self.alphas_cumprod, t, x_start.shape) + log_variance = _extract_into_tensor( + self.log_one_minus_alphas_cumprod, t, x_start.shape + ) + return mean, variance, log_variance + + def q_sample(self, x_start, t, noise=None): + """ + Diffuse the data for a given number of diffusion steps. + + In other words, sample from q(x_t | x_0). + + :param x_start: the initial data batch. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :param noise: if specified, the split-out normal noise. + :return: A noisy version of x_start. + """ + if noise is None: + noise = th.randn_like(x_start) + assert_shape(noise, x_start) + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * x_start + + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) + * noise + ) + + def q_posterior_mean_variance(self, x_start, x_t, t): + """ + Compute the mean and variance of the diffusion posterior: + + q(x_{t-1} | x_t, x_0) + + """ + assert_shape(x_start, x_t) + posterior_mean = ( + _extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) * x_start + + _extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) * x_t + ) + posterior_variance = _extract_into_tensor(self.posterior_variance, t, x_t.shape) + posterior_log_variance_clipped = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape + ) + assert_shape( + posterior_mean.shape[:1], + posterior_variance.shape[:1], + posterior_log_variance_clipped.shape[:1], + x_start.shape[:1], + ) + return posterior_mean, posterior_variance, posterior_log_variance_clipped + + def p_mean_variance( + self, model, x, t, clip_denoised=True, denoised_fn=None, model_kwargs=None, + model_var_type=None, + ): + """ + Apply the model to get p(x_{t-1} | x_t), as well as a prediction of + the initial x, x_0. + + :param model: the model, which takes a signal and a batch of timesteps + as input. + :param x: the [N x C x ...] tensor at time t. + :param t: a 1-D Tensor of timesteps. + :param clip_denoised: if True, clip the denoised signal into [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. Applies before + clip_denoised. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :return: a dict with the following keys: + - 'mean': the model mean output. + - 'variance': the model variance output. + - 'log_variance': the log of 'variance'. + - 'pred_xstart': the prediction for x_0. + """ + if model_kwargs is None: + model_kwargs = {} + + if model_var_type is None: + model_var_type = self.model_var_type + + B, C = x.shape[:2] + assert_shape(t, (B,)) + out_dict = model(x, t, **model_kwargs) + model_output = out_dict['x'] + + if len(out_dict) > 1: + extra = {k: v for k, v in out_dict.items() if k != 'x'} + else: + extra = None + + # self.model_var_type corresponds to model output + if self.model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]: + assert_shape(model_output, (B, C * 2, *x.shape[2:])) + model_output, model_var_values = th.split(model_output, C, dim=1) + + # model_var_type corresponds to reverse diffusion process + if model_var_type in [ModelVarType.LEARNED, ModelVarType.LEARNED_RANGE]: + if model_var_type == ModelVarType.LEARNED: + model_log_variance = model_var_values + model_variance = th.exp(model_log_variance) + else: + min_log = _extract_into_tensor( + self.posterior_log_variance_clipped, t, x.shape + ) + max_log = _extract_into_tensor(np.log(self.betas), t, x.shape) + # The model_var_values is [-1, 1] for [min_var, max_var]. + frac = (model_var_values + 1) / 2 + model_log_variance = frac * max_log + (1 - frac) * min_log + model_variance = th.exp(model_log_variance) + else: + model_variance, model_log_variance = { + # for fixedlarge, we set the initial (log-)variance like so + # to get a better decoder log likelihood. + ModelVarType.FIXED_LARGE: ( + np.append(self.posterior_variance[1], self.betas[1:]), + np.log(np.append(self.posterior_variance[1], self.betas[1:])), + ), + ModelVarType.FIXED_SMALL: ( + self.posterior_variance, + self.posterior_log_variance_clipped, + ), + }[model_var_type] + model_variance = _extract_into_tensor(model_variance, t, x.shape) + model_log_variance = _extract_into_tensor(model_log_variance, t, x.shape) + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + if self.model_mean_type == ModelMeanType.PREVIOUS_X: + pred_xstart = process_xstart( + self._predict_xstart_from_xprev(x_t=x, t=t, xprev=model_output) + ) + model_mean = model_output + elif self.model_mean_type in [ + ModelMeanType.START_X, + ModelMeanType.EPSILON, + ModelMeanType.VELOCITY + ]: + if self.model_mean_type == ModelMeanType.START_X: + pred_xstart = process_xstart(model_output) + elif self.model_mean_type == ModelMeanType.EPSILON: + pred_xstart = process_xstart( + self._predict_xstart_from_eps(x_t=x, t=t, eps=model_output) + ) + else: + pred_xstart = process_xstart( + self._predict_xstart_from_v(x_t=x, t=t, v=model_output) + ) + model_mean, _, _ = self.q_posterior_mean_variance( + x_start=pred_xstart, x_t=x, t=t + ) + else: + raise NotImplementedError(self.model_mean_type) + + assert_shape(model_mean, model_log_variance, pred_xstart, x) + return { + "mean": model_mean, + "variance": model_variance, + "log_variance": model_log_variance, + "pred_xstart": pred_xstart, + "extra": extra, + } + + def _predict_xstart_from_eps(self, x_t, t, eps): + assert_shape(x_t, eps) + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t + - _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) * eps + ) + + def _predict_xstart_from_v(self, x_t, t, v): + assert_shape(x_t, v) + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_t.shape) * x_t + - _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_t.shape) * v + ) + + def _predict_xstart_from_xprev(self, x_t, t, xprev): + assert_shape(x_t, xprev) + return ( # (xprev - coef2*x_t) / coef1 + _extract_into_tensor(1.0 / self.posterior_mean_coef1, t, x_t.shape) * xprev + - _extract_into_tensor( + self.posterior_mean_coef2 / self.posterior_mean_coef1, t, x_t.shape + ) + * x_t + ) + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + return ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t + - pred_xstart + ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) + + def _velocity_from_xstart_and_noise(self, x_start, t, noise): + return ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) * noise + - _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, x_start.shape) + * x_start + ) + + def _vb_terms_bpd( + self, model, x_start, x_t, t, clip_denoised=True, model_kwargs=None + ): + """ + Get a term for the variational lower-bound. + + The resulting units are bits (rather than nats, as one might expect). + This allows for comparison to other papers. + + :return: a dict with the following keys: + - 'output': a shape [N] tensor of NLLs or KLs. + - 'pred_xstart': the x_0 predictions. + """ + true_mean, _, true_log_variance_clipped = self.q_posterior_mean_variance( + x_start=x_start, x_t=x_t, t=t + ) + out = self.p_mean_variance( + model, x_t, t, clip_denoised=clip_denoised, model_kwargs=model_kwargs + ) + kl = normal_kl( + true_mean, true_log_variance_clipped, out["mean"], out["log_variance"] + ) + kl = mean_flat(kl) / np.log(2.0) + + decoder_nll = -discretized_gaussian_log_likelihood( + x_start, means=out["mean"], log_scales=0.5 * out["log_variance"] + ) + assert_shape(decoder_nll, x_start) + decoder_nll = mean_flat(decoder_nll) / np.log(2.0) + + # At the first timestep return the decoder NLL, + # otherwise return KL(q(x_{t-1}|x_t,x_0) || p(x_{t-1}|x_t)) + output = th.where((t == 0), decoder_nll, kl) + return {"output": output, "pred_xstart": out["pred_xstart"], "extra": out["extra"]} + + def training_losses(self, model, x_start, step, model_kwargs=None, controlnet=None, noise=None): + """ + Compute training losses for a single timestep. + + :param model: the model to evaluate loss on. + :param x_start: the [N x C x ...] tensor of inputs. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param noise: if specified, the specific Gaussian noise to try to remove. + :return: a dict with the key "loss" containing a tensor of shape [N]. + Some mean or variance settings may also have other keys. + """ + # load_rand and dir are used for debug + load_rand = False + dir = './noise/0814' + + local_rank = th.distributed.get_rank() + dev = x_start.device + if model_kwargs is None: + model_kwargs = {} + # Time steps + t = th.randint(0, self.num_timesteps, (x_start.shape[0],), device=x_start.device) + + if load_rand: + t = th.load(f'{dir}/t_{step}_{local_rank}.pt').to(device=dev) + # Noise + if noise is None: + noise = th.randn_like(x_start) + if load_rand: + noise = th.load(f'{dir}/noise_{step}_{local_rank}.pt').to( + device=dev) + if self.noise_offset > 0: + # Add channel wise noise offset + # https://www.crosslabs.org/blog/diffusion-with-offset-noise + noise = noise + self.noise_offset * th.randn(*x_start.shape[:2], 1, 1, device=x_start.device) + x_t = self.q_sample(x_start, t, noise=noise) + + terms = {} + + if self.mse_loss_weight_type == 'constant': + mse_loss_weight = th.ones_like(t) + elif self.mse_loss_weight_type.startswith("min_snr_"): + alpha = _extract_into_tensor(self.sqrt_alphas_cumprod, t, t.shape) + sigma = _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t, t.shape) + snr = (alpha / sigma) ** 2 + + k = float(self.mse_loss_weight_type.split('min_snr_')[-1]) + # min{snr, k} + mse_loss_weight = th.stack([snr, k * th.ones_like(t)], dim=1).min(dim=1)[0] / snr + else: + raise ValueError(self.mse_loss_weight_type) + + if self.loss_type == LossType.KL or self.loss_type == LossType.RESCALED_KL: + out_dict = self._vb_terms_bpd( + model=model, + x_start=x_start, + x_t=x_t, + t=t, + clip_denoised=False, + model_kwargs=model_kwargs, + ) + terms["loss"] = out_dict["output"] + if self.loss_type == LossType.RESCALED_KL: + terms["loss"] *= self.num_timesteps + extra = out_dict["extra"] + elif self.loss_type == LossType.MSE or self.loss_type == LossType.RESCALED_MSE: + if controlnet != None: + controls = controlnet(x_t, t, **model_kwargs) + model_kwargs.pop('condition') + model_kwargs.update(controls) + out_dict = model(x_t, t, **model_kwargs) + model_output = out_dict['x'] + extra = {k: v for k, v in out_dict.items() if k != 'x'} + + if self.model_var_type in [ + ModelVarType.LEARNED, + ModelVarType.LEARNED_RANGE, + ]: + B, C = x_t.shape[:2] + assert_shape(model_output, (B, C * 2, *x_t.shape[2:])) + model_output, model_var_values = th.split(model_output, C, dim=1) + # Learn the variance using the variational bound, but don't let + # it affect our mean prediction. + frozen_out = th.cat([model_output.detach(), model_var_values], dim=1) + terms["vb"] = self._vb_terms_bpd( + model=lambda *args, r=frozen_out: dict(x=r), + x_start=x_start, + x_t=x_t, + t=t, + clip_denoised=False, + )["output"] + if self.loss_type == LossType.RESCALED_MSE: + # Divide by 1000 for equivalence with initial implementation. + # Without a factor of 1/1000, the VB term hurts the MSE term. + terms["vb"] *= self.num_timesteps / 1000.0 + + if self.model_mean_type == ModelMeanType.VELOCITY: + target = self._velocity_from_xstart_and_noise(x_start, t, noise) + else: + target = { + # ModelMeanType.PREVIOUS_X: self.q_posterior_mean_variance( + # x_start=x_start, x_t=x_t, t=t + # )[0], + ModelMeanType.START_X: x_start, + ModelMeanType.EPSILON: noise, + }[self.model_mean_type] + assert_shape(model_output, target, x_start) + raw_mse = mean_flat((target - model_output) ** 2).detach() + terms["mse"] = mse_loss_weight * mean_flat((target - model_output) ** 2) + if "vb" in terms: + terms["loss"] = terms["mse"] + terms["vb"] + terms["raw_loss"] = raw_mse + terms["vb"].detach() + else: + terms["loss"] = terms["mse"] + terms["raw_loss"] = raw_mse + else: + raise NotImplementedError(self.loss_type) + + terms.update(extra) + return terms + + def condition_mean(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute the mean for the previous step, given a function cond_fn that + computes the gradient of a conditional log probability with respect to + x. In particular, cond_fn computes grad(log(p(y|x))), and we want to + condition on y. + + This uses the conditioning strategy from Sohl-Dickstein et al. (2015). + """ + gradient = cond_fn(x, t, **model_kwargs) + new_mean = ( + p_mean_var["mean"].float() + p_mean_var["variance"] * gradient.float() + ) + return new_mean + + def condition_score(self, cond_fn, p_mean_var, x, t, model_kwargs=None): + """ + Compute what the p_mean_variance output would have been, should the + model's score function be conditioned by cond_fn. + See condition_mean() for details on cond_fn. + Unlike condition_mean(), this instead uses the conditioning strategy + from Song et al (2020). + """ + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + + eps = self._predict_eps_from_xstart(x, t, p_mean_var["pred_xstart"]) + eps = eps - (1 - alpha_bar).sqrt() * cond_fn( + x, t, **model_kwargs + ) + + out = p_mean_var.copy() + out["pred_xstart"] = self._predict_xstart_from_eps(x, t, eps) + out["mean"], _, _ = self.q_posterior_mean_variance( + x_start=out["pred_xstart"], x_t=x, t=t + ) + return out + + def p_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + **kwargs, + ): + """ + Sample x_{t-1} from the model at the given timestep. + + :param model: the model to sample from. + :param x: the current tensor at x_{t-1}. + :param t: the value of t, starting at 0 for the first diffusion step. + :param clip_denoised: if True, clip the x_start prediction to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :return: a dict containing the following keys: + - 'sample': a random sample from the model. + - 'pred_xstart': a prediction of x_0. + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + ) + noise = th.randn_like(x) + nonzero_mask = ( + (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + if cond_fn is not None: + out["mean"] = self.condition_mean( + cond_fn, out, x, t, model_kwargs=model_kwargs + ) + + sample = out["mean"] + nonzero_mask * th.exp(0.5 * out["log_variance"]) * noise + + return {"sample": sample, "pred_xstart": out["pred_xstart"]} + + def p_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + device=None, + progress=False, + progress_leave=True, + **kwargs, + ): + """ + Generate samples from the model. + + :param model: the model module. + :param shape: the shape of the samples, (N, C, H, W). + :param noise: if specified, the noise from the encoder to sample. + Should be of the same shape as `shape`. + :param clip_denoised: if True, clip x_start predictions to [-1, 1]. + :param denoised_fn: if not None, a function which applies to the + x_start prediction before it is used to sample. + :param cond_fn: if not None, this is a gradient function that acts + similarly to the model. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + :param model_var_type: if not None, overlap the default self.model_var_type. + It is useful when training with learned var but sampling with fixed var. + :param device: if specified, the device to create the samples on. + If not specified, use a model parameter's device. + :param progress: if True, show a tqdm progress bar. + :return: a non-differentiable batch of samples. + """ + final = None + for sample in self.p_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + device=device, + progress=progress, + progress_leave=progress_leave, + **kwargs, + ): + final = sample + return final["sample"] + + def p_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + model_var_type=None, + device=None, + progress=False, + progress_leave=True, + **kwargs, + ): + """ + Generate samples from the model and yield intermediate samples from + each timestep of diffusion. + + Arguments are the same as p_sample_loop(). + Returns a generator over dicts, where each dict is the return value of + p_sample(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.p_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + model_var_type=model_var_type, + **kwargs, + ) + yield out + img = out["sample"] + + def ddim_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + ): + """ + Sample x_{t-1} from the model using DDIM. + + Same usage as p_sample(). + """ + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score(cond_fn, out, x, t, model_kwargs=model_kwargs) + + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = self._predict_eps_from_xstart(x, t, out["pred_xstart"]) + + alpha_bar = _extract_into_tensor(self.alphas_cumprod, t, x.shape) + alpha_bar_prev = _extract_into_tensor(self.alphas_cumprod_prev, t, x.shape) + sigma = ( + eta + * th.sqrt((1 - alpha_bar_prev) / (1 - alpha_bar)) + * th.sqrt(1 - alpha_bar / alpha_bar_prev) + ) + # Equation 12. + noise = th.randn_like(x) + mean_pred = ( + out["pred_xstart"] * th.sqrt(alpha_bar_prev) + + th.sqrt(1 - alpha_bar_prev - sigma ** 2) * eps + ) + nonzero_mask = ( + (t != 0).float().view(-1, *([1] * (len(x.shape) - 1))) + ) # no noise when t == 0 + sample = mean_pred + nonzero_mask * sigma * noise + return {"sample": sample, "pred_xstart": out["pred_xstart"]} + + def ddim_reverse_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + eta=0.0, + ): + """ + Sample x_{t+1} from the model using DDIM reverse ODE. + """ + assert eta == 0.0, "Reverse ODE only for deterministic path" + out = self.p_mean_variance( + model, + x, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + model_kwargs=model_kwargs, + ) + if cond_fn is not None: + out = self.condition_score(cond_fn, out, x, t, model_kwargs=model_kwargs) + # Usually our model outputs epsilon, but we re-derive it + # in case we used x_start or x_prev prediction. + eps = ( + _extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x.shape) * x + - out["pred_xstart"] + ) / _extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x.shape) + alpha_bar_next = _extract_into_tensor(self.alphas_cumprod_next, t, x.shape) + + # Equation 12. reversed + mean_pred = ( + out["pred_xstart"] * th.sqrt(alpha_bar_next) + + th.sqrt(1 - alpha_bar_next) * eps + ) + + return {"sample": mean_pred, "pred_xstart": out["pred_xstart"]} + + def ddim_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + eta=0.0, + ): + """ + Generate samples from the model using DDIM. + + Same usage as p_sample_loop(). + """ + final = None + for sample in self.ddim_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + progress_leave=progress_leave, + eta=eta, + ): + final = sample + return final["sample"] + + def ddim_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + eta=0.0, + ): + """ + Use DDIM to sample from the model and yield intermediate samples from + each timestep of DDIM. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.ddim_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + eta=eta, + ) + yield out + img = out["sample"] + + def _prior_bpd(self, x_start): + """ + Get the prior KL term for the variational lower-bound, measured in + bits-per-dim. + + This term can't be optimized, as it only depends on the encoder. + + :param x_start: the [N x C x ...] tensor of inputs. + :return: a batch of [N] KL values (in bits), one per batch element. + """ + batch_size = x_start.shape[0] + t = th.tensor([self.num_timesteps - 1] * batch_size, device=x_start.device) + qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t) + kl_prior = normal_kl( + mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0 + ) + return mean_flat(kl_prior) / np.log(2.0) + + def calc_bpd_loop(self, model, x_start, clip_denoised=True, model_kwargs=None): + """ + Compute the entire variational lower-bound, measured in bits-per-dim, + as well as other related quantities. + + :param model: the model to evaluate loss on. + :param x_start: the [N x C x ...] tensor of inputs. + :param clip_denoised: if True, clip denoised samples. + :param model_kwargs: if not None, a dict of extra keyword arguments to + pass to the model. This can be used for conditioning. + + :return: a dict containing the following keys: + - total_bpd: the total variational lower-bound, per batch element. + - prior_bpd: the prior term in the lower-bound. + - vb: an [N x T] tensor of terms in the lower-bound. + - xstart_mse: an [N x T] tensor of x_0 MSEs for each timestep. + - mse: an [N x T] tensor of epsilon MSEs for each timestep. + """ + device = x_start.device + batch_size = x_start.shape[0] + + vb = [] + xstart_mse = [] + mse = [] + for t in list(range(self.num_timesteps))[::-1]: + t_batch = th.tensor([t] * batch_size, device=device) + noise = th.randn_like(x_start) + x_t = self.q_sample(x_start=x_start, t=t_batch, noise=noise) + # Calculate VLB term at the current timestep + with th.no_grad(): + out = self._vb_terms_bpd( + model, + x_start=x_start, + x_t=x_t, + t=t_batch, + clip_denoised=clip_denoised, + model_kwargs=model_kwargs, + ) + vb.append(out["output"]) + xstart_mse.append(mean_flat((out["pred_xstart"] - x_start) ** 2)) + eps = self._predict_eps_from_xstart(x_t, t_batch, out["pred_xstart"]) + mse.append(mean_flat((eps - noise) ** 2)) + + vb = th.stack(vb, dim=1) + xstart_mse = th.stack(xstart_mse, dim=1) + mse = th.stack(mse, dim=1) + + prior_bpd = self._prior_bpd(x_start) + total_bpd = vb.sum(dim=1) + prior_bpd + return { + "total_bpd": total_bpd, + "prior_bpd": prior_bpd, + "vb": vb, + "xstart_mse": xstart_mse, + "mse": mse, + } + + def get_eps( + self, + model, + x, + t, + model_kwargs, + cond_fn=None, + ): + model_output = model(x, t, **model_kwargs)['x'] + if isinstance(model_output, tuple): + model_output, _ = model_output + eps = model_output[:, :4] + if cond_fn is not None: + alpha_bar = _extract_into_tensor_lerp(self.alphas_cumprod, t, x.shape) + eps = eps - th.sqrt(1 - alpha_bar) * cond_fn(x, t, **model_kwargs) + return eps + + def eps_to_pred_xstart( + self, + x, + eps, + t, + ): + alpha_bar = _extract_into_tensor_lerp(self.alphas_cumprod, t, x.shape) + return (x - eps * th.sqrt(1 - alpha_bar)) / th.sqrt(alpha_bar) + + def pndm_transfer( + self, + x, + eps, + t_1, + t_2, + ): + pred_xstart = self.eps_to_pred_xstart(x, eps, t_1) + alpha_bar_prev = _extract_into_tensor_lerp(self.alphas_cumprod, t_2, x.shape) + return pred_xstart * th.sqrt(alpha_bar_prev) + th.sqrt(1 - alpha_bar_prev) * eps + + def prk_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + ): + """ + Generate samples from the model using PRK. + + Same usage as p_sample_loop(). + """ + final = None + for sample in self.prk_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + ): + final = sample + return final["sample"] + + def prk_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + ): + """ + Use PRK to sample from the model and yield intermediate samples from + each timestep of PRK. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1][1:-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=False) + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + out = self.prk_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + yield out + img = out["sample"] + + def prk_sample( + self, + model, + x, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model using fourth-order Pseudo Runge-Kutta + (https://openreview.net/forum?id=PlKWVd2yBkY). + + Same usage as p_sample(). + """ + if model_kwargs is None: + model_kwargs = {} + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + eps_1 = self.get_eps(model, x, t, model_kwargs, cond_fn) + x_1 = self.pndm_transfer(x, eps_1, t, t - 0.5) + eps_2 = self.get_eps(model, x_1, t - 0.5, model_kwargs, cond_fn) + x_2 = self.pndm_transfer(x, eps_2, t, t - 0.5) + eps_3 = self.get_eps(model, x_2, t - 0.5, model_kwargs, cond_fn) + x_3 = self.pndm_transfer(x, eps_3, t, t - 1) + eps_4 = self.get_eps(model, x_3, t - 1, model_kwargs, cond_fn) + eps_prime = (eps_1 + 2 * eps_2 + 2 * eps_3 + eps_4) / 6 + + sample = self.pndm_transfer(x, eps_prime, t, t - 1) + pred_xstart = self.eps_to_pred_xstart(x, eps_prime, t) + pred_xstart = process_xstart(pred_xstart) + return {"sample": sample, "pred_xstart": pred_xstart, "eps": eps_prime} + + def plms_sample_loop( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + ): + """ + Generate samples from the model using PLMS. + + Same usage as p_sample_loop(). + """ + assert self.model_mean_type == ModelMeanType.EPSILON, \ + 'plms_sample only support model_mean_type == ModelMeanType.EPSILON' + final = None + for sample in self.plms_sample_loop_progressive( + model, + shape, + noise=noise, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + device=device, + progress=progress, + progress_leave=progress_leave, + ): + final = sample + return final["sample"] + + def plms_sample_loop_progressive( + self, + model, + shape, + noise=None, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + device=None, + progress=False, + progress_leave=True, + ): + """ + Use PLMS to sample from the model and yield intermediate samples from + each timestep of PLMS. + + Same usage as p_sample_loop_progressive(). + """ + if device is None: + device = next(model.parameters()).device + assert isinstance(shape, (tuple, list)) + if noise is not None: + img = noise + else: + img = th.randn(*shape, device=device) + indices = list(range(self.num_timesteps))[::-1][1:-1] + + if progress: + # Lazy import so that we don't depend on tqdm. + from tqdm.auto import tqdm + + indices = tqdm(indices, leave=progress_leave) + + old_eps = [] + + for i in indices: + t = th.tensor([i] * shape[0], device=device) + with th.no_grad(): + if len(old_eps) < 3: + out = self.prk_sample( + model, + img, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + else: + out = self.plms_sample( + model, + img, + old_eps, + t, + clip_denoised=clip_denoised, + denoised_fn=denoised_fn, + cond_fn=cond_fn, + model_kwargs=model_kwargs, + ) + old_eps.pop(0) + old_eps.append(out["eps"]) + yield out + img = out["sample"] + + def plms_sample( + self, + model, + x, + old_eps, + t, + clip_denoised=True, + denoised_fn=None, + cond_fn=None, + model_kwargs=None, + ): + """ + Sample x_{t-1} from the model using fourth-order Pseudo Linear Multistep + (https://openreview.net/forum?id=PlKWVd2yBkY). + """ + if model_kwargs is None: + model_kwargs = {} + + def process_xstart(x): + if denoised_fn is not None: + x = denoised_fn(x) + if clip_denoised: + return x.clamp(-1, 1) + return x + + eps = self.get_eps(model, x, t, model_kwargs, cond_fn) + eps_prime = (55 * eps - 59 * old_eps[-1] + 37 * old_eps[-2] - 9 * old_eps[-3]) / 24 + + sample = self.pndm_transfer(x, eps_prime, t, t - 1) + pred_xstart = self.eps_to_pred_xstart(x, eps, t) + pred_xstart = process_xstart(pred_xstart) + return {"sample": sample, "pred_xstart": pred_xstart, "eps": eps} + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + """ + Extract values from a 1-D numpy array for a batch of indices. + + :param arr: the 1-D numpy array. + :param timesteps: a tensor of indices into the array to extract. + :param broadcast_shape: a larger shape of K dimensions with the batch + dimension equal to the length of timesteps. + :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims. + """ + res = th.from_numpy(arr).to(device=timesteps.device)[timesteps].float() + while len(res.shape) < len(broadcast_shape): + res = res[..., None] + return res.expand(broadcast_shape) + + +def _extract_into_tensor_lerp(arr, timesteps, broadcast_shape): + """ + Extract values from arr with fractional time steps + """ + timesteps = timesteps.float() + frac = timesteps.frac() + while len(frac.shape) < len(broadcast_shape): + frac = frac[..., None] + res_1 = _extract_into_tensor(arr, timesteps.floor().long(), broadcast_shape) + res_2 = _extract_into_tensor(arr, timesteps.ceil().long(), broadcast_shape) + return th.lerp(res_1, res_2, frac) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..e765b7052328628b541ef65a0183f797d1233a57 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline.py @@ -0,0 +1,794 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import ( + deprecate, + logging, + replace_example_docstring, +) +from diffusers.utils.torch_utils import randn_tensor +from transformers import BertModel, BertTokenizer +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ..modules.models import HunYuanDiT + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]): + A `BertTokenizer` or `CLIPTokenizer` to tokenize text. + unet (Optional[`HunYuanDiT`, `UNet2DConditionModel`]): + A `HunYuanDiT` or `UNet2DConditionModel` to denoise the encoded image latents. + Notice: Here we still keep the word `unet` for compatibility with the previous version of the pipeline. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: Union[BertModel, CLIPTextModel], + tokenizer: Union[BertTokenizer, CLIPTokenizer], + unet: Union[HunYuanDiT, UNet2DConditionModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + progress_bar_config: Dict[str, Any] = None, + embedder_t5=None, + infer_mode='torch', + ): + super().__init__() + + # ======================================================== + self.embedder_t5 = embedder_t5 + self.infer_mode = infer_mode + + # ======================================================== + if progress_bar_config is None: + progress_bar_config = {} + if not hasattr(self, '_progress_bar_config'): + self._progress_bar_config = {} + self._progress_bar_config.update(progress_bar_config) + # ======================================================== + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + embedder=None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + embedder: + T5 embedder (including text encoder and tokenizer) + """ + if embedder is None: + text_encoder = self.text_encoder + tokenizer = self.tokenizer + max_length = self.tokenizer.model_max_length + else: + text_encoder = embedder.model + tokenizer = embedder.tokenizer + max_length = embedder.max_length + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode( + untruncated_ids[:, tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + else: + attention_mask = None + + if text_encoder is not None: + prompt_embeds_dtype = text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_attention_mask = uncond_input.attention_mask.to(device) + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + uncond_attention_mask = uncond_attention_mask.repeat(num_images_per_prompt, 1) + else: + uncond_attention_mask = None + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + height: int, + width: int, + prompt: Union[str, List[str]] = None, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_t5: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_t5: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + image_meta_size: Optional[torch.LongTensor] = None, + style: Optional[torch.LongTensor] = None, + progress: bool = True, + use_fp16: bool = False, + freqs_cis_img: Optional[tuple] = None, + learn_sigma: bool = True, + ): + r""" + The call function to the pipeline for generation. + + Args: + height (`int`): + The height in pixels of the generated image. + width (`int`): + The width in pixels of the generated image. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor, + pred_x0: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + prompt_embeds_t5, negative_prompt_embeds_t5, attention_mask_t5, uncond_attention_mask_t5 = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds_t5, + negative_prompt_embeds=negative_prompt_embeds_t5, + lora_scale=text_encoder_lora_scale, + embedder=self.embedder_t5, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([uncond_attention_mask, attention_mask]) + prompt_embeds_t5 = torch.cat([negative_prompt_embeds_t5, prompt_embeds_t5]) + attention_mask_t5 = torch.cat([uncond_attention_mask_t5, attention_mask_t5]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents(batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=latent_model_input.device) + + if use_fp16: + latent_model_input = latent_model_input.half() + t_expand = t_expand.half() + prompt_embeds = prompt_embeds.half() + ims = image_meta_size.half() if image_meta_size is not None else None + else: + ims = image_meta_size if image_meta_size is not None else None + + # predict the noise residual + if self.infer_mode in ["fa", "torch"]: + noise_pred = self.unet( + latent_model_input, + t_expand, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + ) + elif self.infer_mode == "trt": + noise_pred = self.unet( + x=latent_model_input.contiguous(), + t_emb=t_expand.contiguous(), + context=prompt_embeds.contiguous(), + image_meta_size=ims.contiguous(), + style=style.contiguous(), + freqs_cis_img0=freqs_cis_img[0].to(device).contiguous(), + freqs_cis_img1=freqs_cis_img[1].to(device).contiguous(), + text_embedding_mask=attention_mask.contiguous(), + encoder_hidden_states_t5=prompt_embeds_t5.contiguous(), + text_embedding_mask_t5=attention_mask_t5.contiguous(), + ) + else: + raise ValueError("Unknown infer_mode: {self.infer_mode}") + if learn_sigma: + noise_pred, _ = noise_pred.chunk(2, dim=1) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + results = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=True) + latents = results.prev_sample + pred_x0 = results.pred_original_sample if hasattr(results, 'pred_original_sample') else None + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents, pred_x0) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5d0beb8e5dfda19fc9391a856c15468209d85a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/pipeline_controlnet.py @@ -0,0 +1,821 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from diffusers.configuration_utils import FrozenDict +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from diffusers.models import AutoencoderKL, UNet2DConditionModel +from diffusers.models.lora import adjust_lora_scale_text_encoder +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput +from diffusers.pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import ( + deprecate, + logging, + replace_example_docstring, +) +from diffusers.utils.torch_utils import randn_tensor +from transformers import BertModel, BertTokenizer +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ..modules.models import HunYuanDiT + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionControlNetPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]): + A `BertTokenizer` or `CLIPTokenizer` to tokenize text. + unet (Optional[`HunYuanDiT`, `UNet2DConditionModel`]): + A `HunYuanDiT` or `UNet2DConditionModel` to denoise the encoded image latents. + Notice: Here we still keep the word `unet` for compatibility with the previous version of the pipeline. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: Union[BertModel, CLIPTextModel], + tokenizer: Union[BertTokenizer, CLIPTokenizer], + unet: Union[HunYuanDiT, UNet2DConditionModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + progress_bar_config: Dict[str, Any] = None, + embedder_t5=None, + infer_mode='torch', + controlnet=None, + ): + super().__init__() + + # ======================================================== + self.embedder_t5 = embedder_t5 + self.infer_mode = infer_mode + + # ======================================================== + if progress_bar_config is None: + progress_bar_config = {} + if not hasattr(self, '_progress_bar_config'): + self._progress_bar_config = {} + self._progress_bar_config.update(progress_bar_config) + # ======================================================== + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + controlnet=controlnet, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + embedder=None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + embedder: + T5 embedder (including text encoder and tokenizer) + """ + if embedder is None: + text_encoder = self.text_encoder + tokenizer = self.tokenizer + max_length = self.tokenizer.model_max_length + else: + text_encoder = embedder.model + tokenizer = embedder.tokenizer + max_length = embedder.max_length + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode( + untruncated_ids[:, tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + else: + attention_mask = None + + if text_encoder is not None: + prompt_embeds_dtype = text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_attention_mask = uncond_input.attention_mask.to(device) + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + uncond_attention_mask = uncond_attention_mask.repeat(num_images_per_prompt, 1) + else: + uncond_attention_mask = None + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + height: int, + width: int, + prompt: Union[str, List[str]] = None, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_t5: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_t5: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + image_meta_size: Optional[torch.LongTensor] = None, + style: Optional[torch.LongTensor] = None, + progress: bool = True, + use_fp16: bool = False, + freqs_cis_img: Optional[tuple] = None, + learn_sigma: bool = True, + image=None, + control_weight=1.0 + ): + r""" + The call function to the pipeline for generation. + + Args: + height (`int`): + The height in pixels of the generated image. + width (`int`): + The width in pixels of the generated image. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor, + pred_x0: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds, attention_mask, uncond_attention_mask = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + prompt_embeds_t5, negative_prompt_embeds_t5, attention_mask_t5, uncond_attention_mask_t5 = \ + self.encode_prompt(prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds_t5, + negative_prompt_embeds=negative_prompt_embeds_t5, + lora_scale=text_encoder_lora_scale, + embedder=self.embedder_t5, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([uncond_attention_mask, attention_mask]) + prompt_embeds_t5 = torch.cat([negative_prompt_embeds_t5, prompt_embeds_t5]) + attention_mask_t5 = torch.cat([uncond_attention_mask_t5, attention_mask_t5]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents(batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + condition = self.vae.encode(image.float()).latent_dist.sample(generator).mul_(self.vae.config.scaling_factor).half() + condition = torch.cat([condition] * 2) if do_classifier_free_guidance else condition + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=latent_model_input.device) + + if use_fp16: + latent_model_input = latent_model_input.half() + t_expand = t_expand.half() + prompt_embeds = prompt_embeds.half() + ims = image_meta_size.half() if image_meta_size is not None else None + else: + ims = image_meta_size if image_meta_size is not None else None + + # predict the noise residual + if self.infer_mode in ["fa", "torch"]: + controls = self.controlnet( + latent_model_input, + t_expand, + condition, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + ) + if isinstance(control_weight, list): + assert len(control_weight) == len(controls) + controls = [control * weight for control, weight in zip(controls, control_weight)] + else: + controls = [control * control_weight for control in controls] + noise_pred = self.unet( + latent_model_input, + t_expand, + encoder_hidden_states=prompt_embeds, + text_embedding_mask=attention_mask, + encoder_hidden_states_t5=prompt_embeds_t5, + text_embedding_mask_t5=attention_mask_t5, + image_meta_size=ims, + style=style, + cos_cis_img=freqs_cis_img[0], + sin_cis_img=freqs_cis_img[1], + return_dict=False, + controls=controls + ) + elif self.infer_mode == "trt": + noise_pred = self.unet( + x=latent_model_input.contiguous(), + t_emb=t_expand.contiguous(), + context=prompt_embeds.contiguous(), + image_meta_size=ims.contiguous(), + style=style.contiguous(), + freqs_cis_img0=freqs_cis_img[0].to(device).contiguous(), + freqs_cis_img1=freqs_cis_img[1].to(device).contiguous(), + text_embedding_mask=attention_mask.contiguous(), + encoder_hidden_states_t5=prompt_embeds_t5.contiguous(), + text_embedding_mask_t5=attention_mask_t5.contiguous(), + ) + else: + raise ValueError("Unknown infer_mode: {self.infer_mode}") + if learn_sigma: + noise_pred, _ = noise_pred.chunk(2, dim=1) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + results = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=True) + latents = results.prev_sample + pred_x0 = results.pred_original_sample if hasattr(results, 'pred_original_sample') else None + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents, pred_x0) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/respace.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/respace.py new file mode 100644 index 0000000000000000000000000000000000000000..1e2bf7c0a6bfd78a0295efba8e9f99ef95ca8471 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/diffusion/respace.py @@ -0,0 +1,144 @@ +import numpy as np +import torch as th + +from .gaussian_diffusion import GaussianDiffusion + + +def space_timesteps(num_timesteps, section_counts): + """ + Create a list of timesteps to use from an original diffusion process, + given the number of timesteps we want to take from equally-sized portions + of the original process. + For example, if there's 300 timesteps and the section counts are [10,15,20] + then the first 100 timesteps are strided to be 10 timesteps, the second 100 + are strided to be 15 timesteps, and the final 100 are strided to be 20. + If the stride is a string starting with "ddim", then the fixed striding + from the DDIM paper is used, and only one section is allowed. + :param num_timesteps: the number of diffusion steps in the original + process to divide up. + :param section_counts: either a list of numbers, or a string containing + comma-separated numbers, indicating the step count + per section. As a special case, use "ddimN" where N + is a number of steps to use the striding from the + DDIM paper. + :return: a set of diffusion steps from the original process to use. + """ + if isinstance(section_counts, str): + if section_counts.startswith("ddim"): + desired_count = int(section_counts[len("ddim") :]) + for i in range(1, num_timesteps): + if len(range(0, num_timesteps, i)) == desired_count: + return set(range(0, num_timesteps, i)) + raise ValueError( + f"cannot create exactly {num_timesteps} steps with an integer stride" + ) + section_counts = [int(x) for x in section_counts.split(",")] + size_per = num_timesteps // len(section_counts) + extra = num_timesteps % len(section_counts) + start_idx = 0 + all_steps = [] + for i, section_count in enumerate(section_counts): + size = size_per + (1 if i < extra else 0) + if size < section_count: + raise ValueError( + f"cannot divide section of {size} steps into {section_count}" + ) + if section_count <= 1: + frac_stride = 1 + else: + frac_stride = (size - 1) / (section_count - 1) + cur_idx = 0.0 + taken_steps = [] + for _ in range(section_count): + taken_steps.append(start_idx + round(cur_idx)) + cur_idx += frac_stride + all_steps += taken_steps + start_idx += size + return set(all_steps) + + +class SpacedDiffusion(GaussianDiffusion): + """ + Improved DDPM + + A diffusion process which can skip steps in a base diffusion process. + :param use_timesteps: a collection (sequence or set) of timesteps from the + original diffusion process to retain. + :param kwargs: the kwargs to create the base diffusion process. + """ + + def __init__(self, use_timesteps, **kwargs): + self.use_timesteps = set(use_timesteps) + self.timestep_map = [] + self.original_num_steps = len(kwargs["betas"]) + + base_diffusion = GaussianDiffusion(**kwargs) # pylint: disable=missing-kwoa + last_alpha_cumprod = 1.0 + new_betas = [] + for i, alpha_cumprod in enumerate(base_diffusion.alphas_cumprod): + if i in self.use_timesteps: + new_betas.append(1 - alpha_cumprod / last_alpha_cumprod) + last_alpha_cumprod = alpha_cumprod + self.timestep_map.append(i) + kwargs["betas"] = np.array(new_betas) + super().__init__(**kwargs) + + def p_mean_variance( + self, model, *args, **kwargs + ): # pylint: disable=signature-differs + return super().p_mean_variance(self._wrap_model(model), *args, **kwargs) + + def training_losses( + self, model, controlnet=None, *args, **kwargs + ): # pylint: disable=signature-differs + if controlnet != None: + return super().training_losses(self._wrap_model(model), controlnet=self._wrap_model(controlnet), *args, **kwargs) + else: + return super().training_losses(self._wrap_model(model), *args, **kwargs) + + def condition_mean(self, cond_fn, *args, **kwargs): + return super().condition_mean(self._wrap_model(cond_fn), *args, **kwargs) + + def condition_score(self, cond_fn, *args, **kwargs): + return super().condition_score(self._wrap_model(cond_fn), *args, **kwargs) + + def get_eps(self, model, *args, **kwargs): + return super().get_eps(self._wrap_model(model), *args, **kwargs) + + def _wrap_model(self, model): + if isinstance(model, _WrappedModel): + return model + return _WrappedModel( + model, self.timestep_map, self.original_num_steps + ) + + def _scale_timesteps(self, t): + # Scaling is done by the wrapped model. + return t + + +class _WrappedModel: + """ + Improved DDPM + + When using a subsequent timesteps (e.g., 250), we must wrap the model + for mapping the timesteps 1-250 with step 1 to 1-1000 with step 4 + """ + def __init__(self, model, timestep_map, original_num_steps): + self.model = model + self.timestep_map = timestep_map + # self.rescale_timesteps = rescale_timesteps + self.original_num_steps = original_num_steps + + def __call__(self, x, ts, **kwargs): + """ + Here we must make a interpolation because `ts` maybe a float (e.g., 4.5) + in the PLMS/PNDM sampler. + """ + ts = ts.float() + frac = ts.frac() + map_tensor = th.tensor(self.timestep_map, device=ts.device, dtype=ts.dtype) + new_ts_1 = map_tensor[ts.floor().long()] + new_ts_2 = map_tensor[ts.ceil().long()] + new_ts = th.lerp(new_ts_1, new_ts_2, frac) + return self.model(x, new_ts, **kwargs) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/ds_config.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/ds_config.py new file mode 100644 index 0000000000000000000000000000000000000000..d108a6de496de852dd21ed755e04e7e7515a89ca --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/ds_config.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +import os + +def deepspeed_config_from_args(args, global_batch_size): + if args.use_zero_stage == 2: + deepspeed_config = { + "train_batch_size": global_batch_size, + "train_micro_batch_size_per_gpu": args.batch_size, + "gradient_accumulation_steps": args.grad_accu_steps, + "steps_per_print": args.log_every, + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": args.weight_decay + } + }, + + "zero_optimization": { + "stage": 2, + "reduce_scatter": False, + "reduce_bucket_size": 1e9, + }, + + "gradient_clipping": 1.0, + "prescale_gradients": True, + + "fp16": { + "enabled": args.use_fp16, + "loss_scale": 0, + "loss_scale_window": 500, + "hysteresis": 2, + "min_loss_scale": 1e-3, + "initial_scale_power": 15 + }, + + "bf16": { + "enabled": False + }, + + "wall_clock_breakdown": False + } + + elif args.use_zero_stage == 3: + deepspeed_config = { + "train_batch_size": args.global_batch_size, + # "train_micro_batch_size_per_gpu": args.batch_size, + "gradient_accumulation_steps": args.grad_accu_steps, + "steps_per_print": args.log_every, + + "optimizer": { + "type": "AdamW", + "params": { + "lr": args.lr, + "betas": [ + 0.9, + 0.999 + ], + "eps": 1e-08, + "weight_decay": args.weight_decay + } + }, + + "zero_optimization": { + "stage": 3, + "allgather_partitions": True, + "overlap_comm": True, + "reduce_scatter": True, + "contiguous_gradients": True, + "stage3_prefetch_bucket_size": 5e8, + "stage3_max_live_parameters" : 6e8, + "reduce_bucket_size": 1.2e9, + "sub_group_size": 1e9, + "sub_group_buffer_num": 10, + "pipeline_optimizer": True, + "max_contigous_event_size": 0, + "cache_sub_group_rate": 0.0, + "prefetch_cache_sub_group_rate": 1.0, + "max_contigous_params_size": -1, + "max_param_reduce_events": 0, + "stage3_param_persistence_threshold": 9e9, + "is_communication_time_profiling": False, + "save_large_model_multi_slice": True, + "use_fused_op_with_grad_norm_overflow": False, + "offload_optimizer": { + "device": "cpu", + "pin_memory": True + }, + "offload_param": { + "device": "cpu", + "pin_memory": True + } + }, + + "gradient_clipping": 1.0, + "prescale_gradients": False, + + "fp16": { + "enabled": True, + "loss_scale": 0, + "loss_scale_window": 500, + "hysteresis": 2, + "min_loss_scale": 1, + "initial_scale_power": 15 + }, + + "bf16": { + "enabled": False + }, + + "wall_clock_breakdown": False, + "mem_chunk": { + "default_chunk_size": 536870911, + "use_fake_dist": False, + "client": { + "mem_tracer": { + "use_async_mem_monitor": True, + "warmup_gpu_chunk_mem_ratio": 0.8, + "overall_gpu_mem_ratio": 0.8, + "overall_cpu_mem_ratio": 1.0, + "margin_use_ratio": 0.8, + "use_fake_dist": False + }, + "opts": { + "with_mem_cache": True, + "with_async_move": True + } + } + } + } + else: + raise ValueError + return deepspeed_config + + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..682f29d7faa2d17a567e9f4d623f6eb8e1361181 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference.py @@ -0,0 +1,413 @@ +import random +import time +from pathlib import Path + +import numpy as np +import torch + +# For reproducibility +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +from diffusers import schedulers +from diffusers.models import AutoencoderKL +from loguru import logger +from transformers import BertModel, BertTokenizer +from transformers.modeling_utils import logger as tf_logger + +from .constants import SAMPLER_FACTORY, NEGATIVE_PROMPT, TRT_MAX_WIDTH, TRT_MAX_HEIGHT, TRT_MAX_BATCH_SIZE +from .diffusion.pipeline import StableDiffusionPipeline +from .modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG +from .modules.posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from .modules.text_encoder import MT5Embedder +from .utils.tools import set_seeds +from peft import LoraConfig + + +class Resolution: + def __init__(self, width, height): + self.width = width + self.height = height + + def __str__(self): + return f'{self.height}x{self.width}' + + +class ResolutionGroup: + def __init__(self): + self.data = [ + Resolution(1024, 1024), # 1:1 + Resolution(1280, 1280), # 1:1 + Resolution(1024, 768), # 4:3 + Resolution(1152, 864), # 4:3 + Resolution(1280, 960), # 4:3 + Resolution(768, 1024), # 3:4 + Resolution(864, 1152), # 3:4 + Resolution(960, 1280), # 3:4 + Resolution(1280, 768), # 16:9 + Resolution(768, 1280), # 9:16 + ] + self.supported_sizes = set([(r.width, r.height) for r in self.data]) + + def is_valid(self, width, height): + return (width, height) in self.supported_sizes + + +STANDARD_RATIO = np.array([ + 1.0, # 1:1 + 4.0 / 3.0, # 4:3 + 3.0 / 4.0, # 3:4 + 16.0 / 9.0, # 16:9 + 9.0 / 16.0, # 9:16 +]) +STANDARD_SHAPE = [ + [(1024, 1024), (1280, 1280)], # 1:1 + [(1280, 960)], # 4:3 + [(960, 1280)], # 3:4 + [(1280, 768)], # 16:9 + [(768, 1280)], # 9:16 +] +STANDARD_AREA = [ + np.array([w * h for w, h in shapes]) + for shapes in STANDARD_SHAPE +] + + +def get_standard_shape(target_width, target_height): + """ + Map image size to standard size. + """ + target_ratio = target_width / target_height + closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio)) + closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height)) + width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx] + return width, height + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +def get_pipeline(args, vae, text_encoder, tokenizer, model, device, rank, + embedder_t5, infer_mode, sampler=None): + """ + Get scheduler and pipeline for sampling. The sampler and pipeline are both + based on diffusers and make some modifications. + + Returns + ------- + pipeline: StableDiffusionPipeline + sampler_name: str + """ + sampler = sampler or args.sampler + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args.noise_schedule + kwargs['beta_start'] = args.beta_start + kwargs['beta_end'] = args.beta_end + kwargs['prediction_type'] = args.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(args.infer_steps, device) + + # Only enable progress bar for rank 0 + progress_bar_config = {} if rank == 0 else {'disable': True} + + pipeline = StableDiffusionPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=embedder_t5, + infer_mode=infer_mode, + ) + + pipeline = pipeline.to(device) + + return pipeline, sampler + + +class End2End(object): + def __init__(self, args, models_root_path): + self.args = args + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + # Disable BertModel logging checkpoint info + tf_logger.setLevel('ERROR') + + # ======================================================================== + logger.info(f"Loading CLIP Text Encoder...") + text_encoder_path = self.root / "clip_text_encoder" + self.clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + logger.info(f"Loading CLIP Text Encoder finished") + + # ======================================================================== + logger.info(f"Loading CLIP Tokenizer...") + tokenizer_path = self.root / "tokenizer" + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + logger.info(f"Loading CLIP Tokenizer finished") + + # ======================================================================== + logger.info(f"Loading T5 Text Encoder and T5 Tokenizer...") + t5_text_encoder_path = self.root / 'mt5' + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.embedder_t5 = embedder_t5 + logger.info(f"Loading t5_text_encoder and t5_tokenizer finished") + + # ======================================================================== + logger.info(f"Loading VAE...") + vae_path = self.root / "sdxl-vae-fp16-fix" + self.vae = AutoencoderKL.from_pretrained(str(vae_path)).to(self.device) + logger.info(f"Loading VAE finished") + + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + self.patch_size = model_config['patch_size'] + self.head_size = model_config['hidden_size'] // model_config['num_heads'] + self.resolutions, self.freqs_cis_img = self.standard_shapes() # Used for TensorRT models + self.image_size = _to_tuple(self.args.image_size) + latent_size = (self.image_size[0] // 8, self.image_size[1] // 8) + + self.infer_mode = self.args.infer_mode + if self.infer_mode in ['fa', 'torch']: + model_dir = self.root / "model" + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + # Build model structure + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + # Load model checkpoint + logger.info(f"Loading torch model {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + + lora_ckpt = args.lora_ckpt + if lora_ckpt is not None and lora_ckpt != "": + logger.info(f"Loading Lora checkpoint {lora_ckpt}...") + + self.model.load_adapter(lora_ckpt) + self.model.merge_and_unload() + + + self.model.eval() + logger.info(f"Loading torch model finished") + elif self.infer_mode == 'trt': + from .modules.trt.hcf_model import TRTModel + + trt_dir = self.root / "model_trt" + engine_dir = trt_dir / "engine" + plugin_path = trt_dir / "fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so" + model_name = "model_onnx" + + logger.info(f"Loading TensorRT model {engine_dir}/{model_name}...") + self.model = TRTModel(model_name=model_name, + engine_dir=str(engine_dir), + image_height=TRT_MAX_HEIGHT, + image_width=TRT_MAX_WIDTH, + text_maxlen=args.text_len, + embedding_dim=args.text_states_dim, + plugin_path=str(plugin_path), + max_batch_size=TRT_MAX_BATCH_SIZE, + ) + logger.info(f"Loading TensorRT model finished") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Build inference pipeline. We use a customized StableDiffusionPipeline. + logger.info(f"Loading inference pipeline...") + self.pipeline, self.sampler = self.load_sampler() + logger.info(f'Loading pipeline finished') + + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_sampler(self, sampler=None): + pipeline, sampler = get_pipeline(self.args, + self.vae, + self.clip_text_encoder, + self.tokenizer, + self.model, + device=self.device, + rank=0, + embedder_t5=self.embedder_t5, + infer_mode=self.infer_mode, + sampler=sampler, + ) + return pipeline, sampler + + def calc_rope(self, height, width): + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(self.head_size, *sub_args) + return rope + + def standard_shapes(self): + resolutions = ResolutionGroup() + freqs_cis_img = {} + for reso in resolutions.data: + freqs_cis_img[str(reso)] = self.calc_rope(reso.height, reso.width) + return resolutions, freqs_cis_img + + def predict(self, + user_prompt, + height=1024, + width=1024, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=100, + guidance_scale=6, + batch_size=1, + src_size_cond=(1024, 1024), + sampler=None, + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = set_seeds(seed, device=self.device) + # ======================================================================== + # Arguments: target_width, target_height + # ======================================================================== + if width <= 0 or height <= 0: + raise ValueError(f"`height` and `width` must be positive integers, got height={height}, width={width}") + logger.info(f"Input (height, width) = ({height}, {width})") + if self.infer_mode in ['fa', 'torch']: + # We must force height and width to align to 16 and to be an integer. + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + logger.info(f"Align to 16: (height, width) = ({target_height}, {target_width})") + elif self.infer_mode == 'trt': + target_width, target_height = get_standard_shape(width, height) + logger.info(f"Align to standard shape: (height, width) = ({target_height}, {target_width})") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is None or negative_prompt == '': + negative_prompt = self.default_negative_prompt + if not isinstance(negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + # ======================================================================== + # Arguments: style. (A fixed argument. Don't Change it.) + # ======================================================================== + style = torch.as_tensor([0, 0] * batch_size, device=self.device) + + # ======================================================================== + # Inner arguments: image_meta_size (Please refer to SDXL.) + # ======================================================================== + if isinstance(src_size_cond, int): + src_size_cond = [src_size_cond, src_size_cond] + if not isinstance(src_size_cond, (list, tuple)): + raise TypeError(f"`src_size_cond` must be a list or tuple, but got {type(src_size_cond)}") + if len(src_size_cond) != 2: + raise ValueError(f"`src_size_cond` must be a tuple of 2 integers, but got {len(src_size_cond)}") + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * 2 * batch_size, device=self.device) + + # ======================================================================== + start_time = time.time() + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + (height, width): {(target_height, target_width)} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + image_meta_size: {size_cond} + """) + reso = f'{target_height}x{target_width}' + if reso in self.freqs_cis_img: + freqs_cis_img = self.freqs_cis_img[reso] + else: + freqs_cis_img = self.calc_rope(target_height, target_width) + + if sampler is not None and sampler != self.sampler: + self.pipeline, self.sampler = self.load_sampler(sampler) + + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + )[0] + gen_time = time.time() - start_time + logger.debug(f"Success, time: {gen_time}") + + return { + 'images': samples, + 'seed': seed, + } diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..0aa161f19d0a4e4e8f1e8c664c75df930baf9280 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/inference_controlnet.py @@ -0,0 +1,434 @@ +import random +import time +from pathlib import Path + +import numpy as np +import torch + +# For reproducibility +# torch.backends.cudnn.benchmark = False +# torch.backends.cudnn.deterministic = True + +from diffusers import schedulers +from diffusers.models import AutoencoderKL +from loguru import logger +from transformers import BertModel, BertTokenizer +from transformers.modeling_utils import logger as tf_logger + +from .constants import SAMPLER_FACTORY, NEGATIVE_PROMPT, TRT_MAX_WIDTH, TRT_MAX_HEIGHT, TRT_MAX_BATCH_SIZE +from .diffusion.pipeline_controlnet import StableDiffusionControlNetPipeline +from .modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG +from .modules.controlnet import HunYuanControlNet +from .modules.posemb_layers import get_2d_rotary_pos_embed, get_fill_resize_and_crop +from .modules.text_encoder import MT5Embedder +from .utils.tools import set_seeds +from peft import LoraConfig + + +class Resolution: + def __init__(self, width, height): + self.width = width + self.height = height + + def __str__(self): + return f'{self.height}x{self.width}' + + +class ResolutionGroup: + def __init__(self): + self.data = [ + Resolution(1024, 1024), # 1:1 + Resolution(1280, 1280), # 1:1 + Resolution(1024, 768), # 4:3 + Resolution(1152, 864), # 4:3 + Resolution(1280, 960), # 4:3 + Resolution(768, 1024), # 3:4 + Resolution(864, 1152), # 3:4 + Resolution(960, 1280), # 3:4 + Resolution(1280, 768), # 16:9 + Resolution(768, 1280), # 9:16 + ] + self.supported_sizes = set([(r.width, r.height) for r in self.data]) + + def is_valid(self, width, height): + return (width, height) in self.supported_sizes + + +STANDARD_RATIO = np.array([ + 1.0, # 1:1 + 4.0 / 3.0, # 4:3 + 3.0 / 4.0, # 3:4 + 16.0 / 9.0, # 16:9 + 9.0 / 16.0, # 9:16 +]) +STANDARD_SHAPE = [ + [(1024, 1024), (1280, 1280)], # 1:1 + [(1280, 960)], # 4:3 + [(960, 1280)], # 3:4 + [(1280, 768)], # 16:9 + [(768, 1280)], # 9:16 +] +STANDARD_AREA = [ + np.array([w * h for w, h in shapes]) + for shapes in STANDARD_SHAPE +] + + +def get_standard_shape(target_width, target_height): + """ + Map image size to standard size. + """ + target_ratio = target_width / target_height + closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio)) + closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height)) + width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx] + return width, height + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +def get_pipeline(args, vae, text_encoder, tokenizer, model, device, rank, + embedder_t5, infer_mode, controlnet, sampler=None): + """ + Get scheduler and pipeline for sampling. The sampler and pipeline are both + based on diffusers and make some modifications. + + Returns + ------- + pipeline: StableDiffusionControlNetPipeline + sampler_name: str + """ + sampler = sampler or args.sampler + + # Load sampler from factory + kwargs = SAMPLER_FACTORY[sampler]['kwargs'] + scheduler = SAMPLER_FACTORY[sampler]['scheduler'] + + # Update sampler according to the arguments + kwargs['beta_schedule'] = args.noise_schedule + kwargs['beta_start'] = args.beta_start + kwargs['beta_end'] = args.beta_end + kwargs['prediction_type'] = args.predict_type + + # Build scheduler according to the sampler. + scheduler_class = getattr(schedulers, scheduler) + scheduler = scheduler_class(**kwargs) + + # Set timesteps for inference steps. + scheduler.set_timesteps(args.infer_steps, device) + + # Only enable progress bar for rank 0 + progress_bar_config = {} if rank == 0 else {'disable': True} + + pipeline = StableDiffusionControlNetPipeline(vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=model, + scheduler=scheduler, + feature_extractor=None, + safety_checker=None, + requires_safety_checker=False, + progress_bar_config=progress_bar_config, + embedder_t5=embedder_t5, + infer_mode=infer_mode, + controlnet=controlnet + ) + + pipeline = pipeline.to(device) + + return pipeline, sampler + + +class End2End(object): + def __init__(self, args, models_root_path): + self.args = args + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + # Disable BertModel logging checkpoint info + tf_logger.setLevel('ERROR') + + # ======================================================================== + logger.info(f"Loading CLIP Text Encoder...") + text_encoder_path = self.root / "clip_text_encoder" + self.clip_text_encoder = BertModel.from_pretrained(str(text_encoder_path), False, revision=None).to(self.device) + logger.info(f"Loading CLIP Text Encoder finished") + + # ======================================================================== + logger.info(f"Loading CLIP Tokenizer...") + tokenizer_path = self.root / "tokenizer" + self.tokenizer = BertTokenizer.from_pretrained(str(tokenizer_path)) + logger.info(f"Loading CLIP Tokenizer finished") + + # ======================================================================== + logger.info(f"Loading T5 Text Encoder and T5 Tokenizer...") + t5_text_encoder_path = self.root / 'mt5' + embedder_t5 = MT5Embedder(t5_text_encoder_path, torch_dtype=torch.float16, max_length=256) + self.embedder_t5 = embedder_t5 + logger.info(f"Loading t5_text_encoder and t5_tokenizer finished") + + # ======================================================================== + logger.info(f"Loading VAE...") + vae_path = self.root / "sdxl-vae-fp16-fix" + self.vae = AutoencoderKL.from_pretrained(str(vae_path)).to(self.device) + logger.info(f"Loading VAE finished") + + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + self.patch_size = model_config['patch_size'] + self.head_size = model_config['hidden_size'] // model_config['num_heads'] + self.resolutions, self.freqs_cis_img = self.standard_shapes() # Used for TensorRT models + self.image_size = _to_tuple(self.args.image_size) + latent_size = (self.image_size[0] // 8, self.image_size[1] // 8) + + self.infer_mode = self.args.infer_mode + if self.infer_mode in ['fa', 'torch']: + model_dir = self.root / "model" + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + # Build model structure + controlnet_dir = self.root / "controlnet" + controlnet_path = controlnet_dir / f"pytorch_model_{self.args.control_type}_{self.args.load_key}.pt" + if not controlnet_path.exists(): + raise ValueError(f"controlnet_path not exists: {controlnet_path}") + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + self.controlnet = HunYuanControlNet(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) + controlnet_state_dict = torch.load(controlnet_path) + if 'module' in controlnet_state_dict: + controlnet_state_dict = controlnet_state_dict['module'] + self.controlnet.load_state_dict(controlnet_state_dict) + logger.info(f"Loading controlnet finished") + # Load model checkpoint + logger.info(f"Loading torch model {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + + lora_ckpt = args.lora_ckpt + if lora_ckpt is not None and lora_ckpt != "": + logger.info(f"Loading Lora checkpoint {lora_ckpt}...") + + self.model.load_adapter(lora_ckpt) + self.model.merge_and_unload() + + + self.model.eval() + self.controlnet.eval() + logger.info(f"Loading torch model finished") + elif self.infer_mode == 'trt': + from .modules.trt.hcf_model import TRTModel + + trt_dir = self.root / "model_trt" + engine_dir = trt_dir / "engine" + plugin_path = trt_dir / "fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so" + model_name = "model_onnx" + + logger.info(f"Loading TensorRT model {engine_dir}/{model_name}...") + self.model = TRTModel(model_name=model_name, + engine_dir=str(engine_dir), + image_height=TRT_MAX_HEIGHT, + image_width=TRT_MAX_WIDTH, + text_maxlen=args.text_len, + embedding_dim=args.text_states_dim, + plugin_path=str(plugin_path), + max_batch_size=TRT_MAX_BATCH_SIZE, + ) + logger.info(f"Loading TensorRT model finished") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Build inference pipeline. We use a customized StableDiffusionControlNetPipeline. + logger.info(f"Loading inference pipeline...") + self.pipeline, self.sampler = self.load_sampler() + logger.info(f'Loading pipeline finished') + + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_sampler(self, sampler=None): + pipeline, sampler = get_pipeline(self.args, + self.vae, + self.clip_text_encoder, + self.tokenizer, + self.model, + device=self.device, + rank=0, + embedder_t5=self.embedder_t5, + infer_mode=self.infer_mode, + sampler=sampler, + controlnet=self.controlnet + ) + return pipeline, sampler + + def calc_rope(self, height, width): + th = height // 8 // self.patch_size + tw = width // 8 // self.patch_size + base_size = 512 // 8 // self.patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + rope = get_2d_rotary_pos_embed(self.head_size, *sub_args) + return rope + + def standard_shapes(self): + resolutions = ResolutionGroup() + freqs_cis_img = {} + for reso in resolutions.data: + freqs_cis_img[str(reso)] = self.calc_rope(reso.height, reso.width) + return resolutions, freqs_cis_img + + def predict(self, + user_prompt, + image, + height=1024, + width=1024, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=100, + guidance_scale=6, + batch_size=1, + src_size_cond=(1024, 1024), + sampler=None, + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = set_seeds(seed, device=self.device) + # ======================================================================== + # Arguments: target_width, target_height + # ======================================================================== + if width <= 0 or height <= 0: + raise ValueError(f"`height` and `width` must be positive integers, got height={height}, width={width}") + logger.info(f"Input (height, width) = ({height}, {width})") + if self.infer_mode in ['fa', 'torch']: + # We must force height and width to align to 16 and to be an integer. + target_height = int((height // 16) * 16) + target_width = int((width // 16) * 16) + logger.info(f"Align to 16: (height, width) = ({target_height}, {target_width})") + elif self.infer_mode == 'trt': + target_width, target_height = get_standard_shape(width, height) + logger.info(f"Align to standard shape: (height, width) = ({target_height}, {target_width})") + else: + raise ValueError(f"Unknown infer_mode: {self.infer_mode}") + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is None or negative_prompt == '': + negative_prompt = self.default_negative_prompt + if not isinstance(negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + # ======================================================================== + # Arguments: style. (A fixed argument. Don't Change it.) + # ======================================================================== + style = torch.as_tensor([0, 0] * batch_size, device=self.device) + + # ======================================================================== + # Inner arguments: image_meta_size (Please refer to SDXL.) + # ======================================================================== + if isinstance(src_size_cond, int): + src_size_cond = [src_size_cond, src_size_cond] + if not isinstance(src_size_cond, (list, tuple)): + raise TypeError(f"`src_size_cond` must be a list or tuple, but got {type(src_size_cond)}") + if len(src_size_cond) != 2: + raise ValueError(f"`src_size_cond` must be a tuple of 2 integers, but got {len(src_size_cond)}") + size_cond = list(src_size_cond) + [target_width, target_height, 0, 0] + image_meta_size = torch.as_tensor([size_cond] * 2 * batch_size, device=self.device) + + # ======================================================================== + start_time = time.time() + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + (height, width): {(target_height, target_width)} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + image_meta_size: {size_cond} + """) + reso = f'{target_height}x{target_width}' + if reso in self.freqs_cis_img: + freqs_cis_img = self.freqs_cis_img[reso] + else: + freqs_cis_img = self.calc_rope(target_height, target_width) + + if sampler is not None and sampler != self.sampler: + self.pipeline, self.sampler = self.load_sampler(sampler) + + samples = self.pipeline( + height=target_height, + width=target_width, + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + image_meta_size=image_meta_size, + style=style, + return_dict=False, + generator=generator, + freqs_cis_img=freqs_cis_img, + use_fp16=self.args.use_fp16, + learn_sigma=self.args.learn_sigma, + image=image, + control_weight=eval(self.args.control_weight), + )[0] + gen_time = time.time() - start_time + logger.debug(f"Success, time: {gen_time}") + + return { + 'images': samples, + 'seed': seed, + } diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/lr_scheduler.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..027c7e6b42eaa8bfeac48a2405d479869bdda683 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/lr_scheduler.py @@ -0,0 +1,761 @@ +""" +Implementation of learning rate schedules. + +Taken and modified from PyTorch v1.0.1 source +https://github.com/pytorch/pytorch/blob/v1.1.0/torch/optim/lr_scheduler.py +""" + +import argparse +from torch.optim import Optimizer +import math + +LR_SCHEDULE = 'lr_schedule' +LR_RANGE_TEST = 'LRRangeTest' +ONE_CYCLE = 'OneCycle' +WARMUP_LR = 'WarmupLR' +WARMUP_DECAY_LR = 'WarmupDecayLR' +VALID_LR_SCHEDULES = [LR_RANGE_TEST, ONE_CYCLE, WARMUP_LR, WARMUP_DECAY_LR] + +LR_RANGE_TEST_MIN_LR = 'lr_range_test_min_lr' +LR_RANGE_TEST_STEP_RATE = 'lr_range_test_step_rate' +LR_RANGE_TEST_STEP_SIZE = 'lr_range_test_step_size' +LR_RANGE_TEST_STAIRCASE = 'lr_range_test_staircase' + +EDGE_VALUE = 'edge_value' +MID_VALUE = 'mid_value' + +CYCLE_FIRST_STEP_SIZE = 'cycle_first_step_size' +CYCLE_FIRST_STAIR_COUNT = 'cycle_first_stair_count' +CYCLE_SECOND_STEP_SIZE = 'cycle_second_step_size' +CYCLE_SECOND_STAIR_COUNT = 'cycle_second_stair_count' +DECAY_STEP_SIZE = 'decay_step_size' + +CYCLE_MIN_LR = 'cycle_min_lr' +CYCLE_MAX_LR = 'cycle_max_lr' +DECAY_LR_RATE = 'decay_lr_rate' + +CYCLE_MIN_MOM = 'cycle_min_mom' +CYCLE_MAX_MOM = 'cycle_max_mom' +DECAY_MOM_RATE = 'decay_mom_rate' + +WARMUP_MIN_LR = 'warmup_min_lr' +WARMUP_MAX_LR = 'warmup_max_lr' +WARMUP_NUM_STEPS = 'warmup_num_steps' +WARMUP_TYPE = 'warmup_type' +WARMUP_LOG_RATE = 'log' +WARMUP_LINEAR_RATE = 'linear' + +TOTAL_NUM_STEPS = 'total_num_steps' + + +def add_tuning_arguments(parser): + group = parser.add_argument_group('Convergence Tuning', 'Convergence tuning configurations') + + # LR scheduler + group.add_argument('--lr_schedule', type=str, default=None, help='LR schedule for training.') + + # Learning rate range test + group.add_argument("--lr_range_test_min_lr", type=float, default=0.001, help='Starting lr value.') + group.add_argument("--lr_range_test_step_rate", type=float, default=1.0, help='scaling rate for LR range test.') + group.add_argument("--lr_range_test_step_size", type=int, default=1000, help='training steps per LR change.') + group.add_argument("--lr_range_test_staircase", + type=bool, + default=False, + help='use staircase scaling for LR range test.') + + # OneCycle schedule + group.add_argument("--cycle_first_step_size", + type=int, + default=1000, + help='size of first step of 1Cycle schedule (training steps).') + group.add_argument("--cycle_first_stair_count", + type=int, + default=-1, + help='first stair count for 1Cycle schedule.') + group.add_argument("--cycle_second_step_size", + type=int, + default=-1, + help='size of second step of 1Cycle schedule (default first_step_size).') + group.add_argument("--cycle_second_stair_count", + type=int, + default=-1, + help='second stair count for 1Cycle schedule.') + group.add_argument("--decay_step_size", + type=int, + default=1000, + help='size of intervals for applying post cycle decay (training steps).') + + # 1Cycle LR + group.add_argument("--cycle_min_lr", type=float, default=0.01, help='1Cycle LR lower bound.') + group.add_argument("--cycle_max_lr", type=float, default=0.1, help='1Cycle LR upper bound.') + group.add_argument("--decay_lr_rate", type=float, default=0.0, help='post cycle LR decay rate.') + + # 1Cycle Momentum + group.add_argument('--cycle_momentum', default=False, action='store_true', help='Enable 1Cycle momentum schedule.') + group.add_argument("--cycle_min_mom", type=float, default=0.8, help='1Cycle momentum lower bound.') + group.add_argument("--cycle_max_mom", type=float, default=0.9, help='1Cycle momentum upper bound.') + group.add_argument("--decay_mom_rate", type=float, default=0.0, help='post cycle momentum decay rate.') + + # Warmup LR + group.add_argument('--warmup_min_lr', type=float, default=0, help='WarmupLR minimum/initial LR value') + group.add_argument('--warmup_max_lr', type=float, default=0.001, help='WarmupLR maximum LR value.') + group.add_argument('--warmup_num_steps', type=int, default=1000, help='WarmupLR step count for LR warmup.') + group.add_argument('--warmup_type', + type=str, + default=WARMUP_LOG_RATE, + help='WarmupLR increasing function during warmup') + return parser + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser = add_tuning_arguments(parser) + + lr_sched_args, unknown_args = parser.parse_known_args() + return lr_sched_args, unknown_args + + +def override_lr_range_test_params(args, params): + if hasattr(args, LR_RANGE_TEST_MIN_LR) and args.lr_range_test_min_lr is not None: + params[LR_RANGE_TEST_MIN_LR] = args.lr_range_test_min_lr + + if hasattr(args, LR_RANGE_TEST_STEP_RATE) and args.lr_range_test_step_rate is not None: + params[LR_RANGE_TEST_STEP_RATE] = args.lr_range_test_step_rate + + if hasattr(args, LR_RANGE_TEST_STEP_SIZE) and args.lr_range_test_step_size is not None: + params[LR_RANGE_TEST_STEP_SIZE] = args.lr_range_test_step_size + + if hasattr(args, LR_RANGE_TEST_STAIRCASE) and args.lr_range_test_staircase is not None: + params[LR_RANGE_TEST_STAIRCASE] = args.lr_range_test_staircase + + +def override_1cycle_params(args, params): + if hasattr(args, CYCLE_FIRST_STEP_SIZE) and args.cycle_first_step_size is not None: + params[CYCLE_FIRST_STEP_SIZE] = args.cycle_first_step_size + + if hasattr(args, CYCLE_FIRST_STAIR_COUNT) and args.cycle_first_stair_count is not None: + params[CYCLE_FIRST_STAIR_COUNT] = args.cycle_first_stair_count + + if hasattr(args, CYCLE_SECOND_STEP_SIZE) and args.cycle_second_step_size is not None: + params[CYCLE_SECOND_STEP_SIZE] = args.cycle_second_step_size + + if hasattr(args, CYCLE_SECOND_STAIR_COUNT) and args.cycle_second_stair_count is not None: + params[CYCLE_SECOND_STAIR_COUNT] = args.cycle_second_stair_count + + if hasattr(args, DECAY_STEP_SIZE) and args.decay_step_size is not None: + params[DECAY_STEP_SIZE] = args.decay_step_size + + # 1Cycle LR params + if hasattr(args, CYCLE_MIN_LR) and args.cycle_min_lr is not None: + params[CYCLE_MIN_LR] = args.cycle_min_lr + + if hasattr(args, CYCLE_MAX_LR) and args.cycle_max_lr is not None: + params[CYCLE_MAX_LR] = args.cycle_max_lr + + if hasattr(args, DECAY_LR_RATE) and args.decay_lr_rate is not None: + params[DECAY_LR_RATE] = args.decay_lr_rate + + # 1Cycle MOM params + if hasattr(args, CYCLE_MIN_MOM) and args.cycle_min_mom is not None: + params[CYCLE_MIN_MOM] = args.cycle_min_mom + + if hasattr(args, CYCLE_MAX_MOM) and args.cycle_max_mom is not None: + params[CYCLE_MAX_MOM] = args.cycle_max_mom + + if hasattr(args, DECAY_MOM_RATE) and args.decay_mom_rate is not None: + params[DECAY_MOM_RATE] = args.decay_mom_rate + + +def override_warmupLR_params(args, params): + if hasattr(args, WARMUP_MIN_LR) and args.warmup_min_lr is not None: + params[WARMUP_MIN_LR] = args.warmup_min_lr + + if hasattr(args, WARMUP_MAX_LR) and args.warmup_max_lr is not None: + params[WARMUP_MAX_LR] = args.warmup_max_lr + + if hasattr(args, WARMUP_NUM_STEPS) and args.warmup_num_steps is not None: + params[WARMUP_NUM_STEPS] = args.warmup_num_steps + + if hasattr(args, WARMUP_TYPE) and args.warmup_type is not None: + params[WARMUP_TYPE] = args.warmup_type + + +def override_params(args, params): + # LR range test params + override_lr_range_test_params(args, params) + + # 1Cycle params + override_1cycle_params(args, params) + + # WarmupLR params + override_warmupLR_params(args, params) + + +def get_config_from_args(args): + if not hasattr(args, LR_SCHEDULE) or args.lr_schedule is None: + return None, '--{} not specified on command line'.format(LR_SCHEDULE) + + if not args.lr_schedule in VALID_LR_SCHEDULES: + return None, '{} is not supported LR schedule'.format(args.lr_schedule) + + config = {} + config['type'] = args.lr_schedule + config['params'] = {} + + if args.lr_schedule == LR_RANGE_TEST: + override_lr_range_test_params(args, config['params']) + elif args.lr_schedule == ONE_CYCLE: + override_1cycle_params(args, config['params']) + else: + override_warmupLR_params(args, config['params']) + + return config, None + + +def get_lr_from_config(config): + if not 'type' in config: + return None, 'LR schedule type not defined in config' + + if not 'params' in config: + return None, 'LR schedule params not defined in config' + + lr_schedule = config['type'] + lr_params = config['params'] + + if not lr_schedule in VALID_LR_SCHEDULES: + return None, '{} is not a valid LR schedule'.format(lr_schedule) + + if lr_schedule == LR_RANGE_TEST: + return lr_params[LR_RANGE_TEST_MIN_LR], '' + if lr_schedule == ONE_CYCLE: + return lr_params[CYCLE_MAX_LR], '' + # Warmup LR + return lr_params[WARMUP_MAX_LR], '' + + +""" +Only optimizers that are subclass of torch.optim.Optimizer are supported. So check the passed optimizer and wrapped +optimizer to see if requirement is satisfied. +TODO: Looking under the hood to examine the wrapped optimizer is a hack that requires a better long-term fix. +""" + + +def get_torch_optimizer(optimizer): + if isinstance(optimizer, Optimizer): + return optimizer + + if hasattr(optimizer, 'optimizer') and isinstance(optimizer.optimizer, Optimizer): + return optimizer.optimizer + + raise TypeError('{} is not a subclass of torch.optim.Optimizer'.format(type(optimizer).__name__)) + + +class LRRangeTest(object): + """Sets the learning rate of each parameter group according to + learning rate range test (LRRT) policy. The policy increases learning + rate starting from a base value with a constant frequency, as detailed in + the paper `A disciplined approach to neural network hyper-parameters: Part1`_. + + LRRT policy is used for finding maximum LR that trains a model without divergence, and can be used to + configure the LR boundaries for Cyclic LR schedules. + + LRRT changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_range_test_min_lr (float or list): Initial learning rate which is the + lower boundary in the range test for each parameter group. + lr_range_test_step_size (int): Interval of training steps to increase learning rate. Default: 2000 + lr_range_test_step_rate (float): Scaling rate for range test. Default: 1.0 + lr_range_test_staircase (bool): Scale in staircase fashion, rather than continuous. Default: False. + last_batch_iteration (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_batch_iteration=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = LRRangeTest(optimizer) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + _A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay: + https://arxiv.org/abs/1803.09820 +""" + + def __init__(self, + optimizer: Optimizer, + lr_range_test_min_lr: float = 1e-3, + lr_range_test_step_size: int = 2000, + lr_range_test_step_rate: float = 1.0, + lr_range_test_staircase: bool = False, + last_batch_iteration: int = -1): + + self.optimizer = get_torch_optimizer(optimizer) + + if isinstance(lr_range_test_min_lr, list) or isinstance(lr_range_test_min_lr, tuple): + if len(lr_range_test_min_lr) != len(self.optimizer.param_groups): + raise ValueError("expected {} lr_range_test_min_lr, got {}".format(len(self.optimizer.param_groups), + len(lr_range_test_min_lr))) + self.min_lr = list(lr_range_test_min_lr) + else: + self.min_lr = [lr_range_test_min_lr] * len(self.optimizer.param_groups) + + self.step_size = lr_range_test_step_size + self.step_rate = lr_range_test_step_rate + self.last_batch_iteration = last_batch_iteration + self.staircase = lr_range_test_staircase + self.interval_fn = self._staircase_interval if lr_range_test_staircase else self._continuous_interval + + if last_batch_iteration == -1: + self._update_optimizer(self.min_lr) + + def _staircase_interval(self): + return math.floor(float(self.last_batch_iteration + 1) / self.step_size) + + def _continuous_interval(self): + return float(self.last_batch_iteration + 1) / self.step_size + + def _get_increase(self): + return (1 + self.step_rate * self.interval_fn()) + + def get_lr(self): + lr_increase = self._get_increase() + return [lr_range_test_min_lr * lr_increase for lr_range_test_min_lr in self.min_lr] + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def _update_optimizer(self, group_lrs): + for param_group, lr in zip(self.optimizer.param_groups, group_lrs): + param_group['lr'] = lr + + def step(self, batch_iteration=None): + if batch_iteration is None: + batch_iteration = self.last_batch_iteration + 1 + self.last_batch_iteration = batch_iteration + self._update_optimizer(self.get_lr()) + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + + +class OneCycle(object): + """Sets the learning rate of each parameter group according to + 1Cycle learning rate policy (1CLR). 1CLR is a variation of the + Cyclical Learning Rate (CLR) policy that involves one cycle followed by + decay. The policy simultaneously cycles the learning rate (and momentum) + between two boundaries with a constant frequency, as detailed in + the paper `A disciplined approach to neural network hyper-parameters`_. + + 1CLR policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This implementation was adapted from the github repo: `pytorch/pytorch`_ + + Args: + optimizer (Optimizer): Wrapped optimizer. + cycle_min_lr (float or list): Initial learning rate which is the + lower boundary in the cycle for each parameter group. + cycle_max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (cycle_max_lr - cycle_min_lr). + The lr at any cycle is the sum of cycle_min_lr + and some scaling of the amplitude; therefore + cycle_max_lr may not actually be reached depending on + scaling function. + decay_lr_rate(float): Decay rate for learning rate. Default: 0. + cycle_first_step_size (int): Number of training iterations in the + increasing half of a cycle. Default: 2000 + cycle_second_step_size (int): Number of training iterations in the + decreasing half of a cycle. If cycle_second_step_size is None, + it is set to cycle_first_step_size. Default: None + cycle_first_stair_count(int): Number of stairs in first half of cycle phase. This means + lr/mom are changed in staircase fashion. Default 0, means staircase disabled. + cycle_second_stair_count(int): Number of stairs in second half of cycle phase. This means + lr/mom are changed in staircase fashion. Default 0, means staircase disabled. + decay_step_size (int): Intervals for applying decay in decay phase. Default: 0, means no decay. + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'cycle_min_mom' and 'cycle_max_mom'. + Default: True + cycle_min_mom (float or list): Initial momentum which is the + lower boundary in the cycle for each parameter group. + Default: 0.8 + cycle_max_mom (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (cycle_max_mom - cycle_min_mom). + The momentum at any cycle is the difference of cycle_max_mom + and some scaling of the amplitude; therefore + cycle_min_mom may not actually be reached depending on + scaling function. Default: 0.9 + decay_mom_rate (float): Decay rate for momentum. Default: 0. + last_batch_iteration (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_batch_iteration=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = OneCycle(optimizer, 0.0001, 0.0010) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + + .. _A disciplined approach to neural network hyper-parameters: Part 1 -- learning rate, batch size, momentum, and weight decay: https://arxiv.org/abs/1803.09820 + """ + + def __init__(self, + optimizer, + cycle_min_lr, + cycle_max_lr, + decay_lr_rate=0., + cycle_first_step_size=2000, + cycle_second_step_size=None, + cycle_first_stair_count=0, + cycle_second_stair_count=None, + decay_step_size=0, + cycle_momentum=True, + cycle_min_mom=0.8, + cycle_max_mom=0.9, + decay_mom_rate=0., + last_batch_iteration=-1): + + self.optimizer = get_torch_optimizer(optimizer) + + # Initialize cycle shape + self._initialize_cycle(cycle_first_step_size, cycle_second_step_size, cycle_first_stair_count, + cycle_second_stair_count, decay_step_size) + + # Initialize cycle lr + self._initialize_lr(self.optimizer, cycle_min_lr, cycle_max_lr, decay_lr_rate, last_batch_iteration) + + # Initialize cyclic momentum + self.cycle_momentum = cycle_momentum + if cycle_momentum: + self._initialize_momentum(self.optimizer, cycle_min_mom, cycle_max_mom, decay_mom_rate, + last_batch_iteration) + + # Initialize batch iteration tracker + self.last_batch_iteration = last_batch_iteration + + # Configure cycle shape + + def _initialize_cycle(self, cycle_first_step_size, cycle_second_step_size, cycle_first_stair_count, + cycle_second_stair_count, decay_step_size): + cycle_first_step_size = float(cycle_first_step_size) + cycle_second_step_size = float( + cycle_second_step_size) if cycle_second_step_size is not None else cycle_first_step_size + + self.total_size = cycle_first_step_size + cycle_second_step_size + self.step_ratio = cycle_first_step_size / self.total_size + self.first_stair_count = cycle_first_stair_count + self.second_stair_count = cycle_first_stair_count if cycle_second_stair_count is None else cycle_second_stair_count + self.decay_step_size = decay_step_size + + if math.isclose(self.decay_step_size, 0): + self.skip_lr_decay = True + self.skip_mom_decay = True + else: + self.skip_lr_decay = False + self.skip_mom_decay = False + + # Configure lr schedule + def _initialize_lr(self, optimizer, cycle_min_lr, cycle_max_lr, decay_lr_rate, last_batch_iteration): + self.min_lrs = [cycle_min_lr] * len(optimizer.param_groups) + if last_batch_iteration == -1: + for lr, group in zip(self.min_lrs, optimizer.param_groups): + group['lr'] = lr + + self.max_lrs = [cycle_max_lr] * len(optimizer.param_groups) + self.decay_lr_rate = decay_lr_rate + + if math.isclose(self.decay_lr_rate, 0): + self.skip_lr_decay = True + + # Configure momentum schedule + def _initialize_momentum(self, optimizer, cycle_min_mom, cycle_max_mom, decay_mom_rate, last_batch_iteration): + if 'betas' not in optimizer.defaults: + optimizer_name = type(optimizer).__name__ + print( + f"cycle_momentum is disabled because optimizer {optimizer_name} does not support momentum, no betas attribute in defaults" + ) + self.cycle_momentum = False + return + + self.decay_mom_rate = decay_mom_rate + self.min_moms = [(cycle_min_mom, 0.99)] * len(optimizer.param_groups) + self.max_moms = [(cycle_max_mom, 0.99)] * len(optimizer.param_groups) + + if last_batch_iteration == -1: + for momentum, group in zip(self.min_moms, optimizer.param_groups): + group['betas'] = momentum + + if math.isclose(self.decay_mom_rate, 0): + self.skip_mom_decay = True + + def _get_scale_factor(self): + batch_iteration = (self.last_batch_iteration + 1) + cycle = math.floor(1 + batch_iteration / self.total_size) + x = 1. + batch_iteration / self.total_size - cycle + if x <= self.step_ratio: + scale_factor = x / self.step_ratio + else: + scale_factor = (x - 1) / (self.step_ratio - 1) + + return scale_factor + + def _get_cycle_mom(self): + scale_factor = self._get_scale_factor() + momentums = [] + for base_betas, max_betas in zip(self.min_moms, self.max_moms): + cycle_min_mom = base_betas[0] + cycle_max_mom = max_betas[0] + base_height = (cycle_max_mom - cycle_min_mom) * scale_factor + momentum = cycle_max_mom - base_height + momentums.append((momentum, base_betas[1])) + return momentums + + def _get_cycle_lr(self): + scale_factor = self._get_scale_factor() + lrs = [] + for cycle_min_lr, cycle_max_lr in zip(self.min_lrs, self.max_lrs): + base_height = (cycle_max_lr - cycle_min_lr) * scale_factor + lr = cycle_min_lr + base_height + lrs.append(lr) + + return lrs + + def _get_decay_mom(self, decay_batch_iteration): + if self.skip_mom_decay: + return self.max_moms + + decay_interval = decay_batch_iteration / self.decay_step_size + mom_decay_factor = (1 + self.decay_mom_rate * decay_interval) + momentums = [(beta0 * mom_decay_factor, beta1) for beta0, beta1 in self.max_moms] + + return momentums + + def _get_decay_lr(self, decay_batch_iteration): + """Calculates the learning rate at batch index. This function is used + after the cycle completes and post cycle decaying of lr/mom is enabled. + This function treats `self.last_batch_iteration` as the last batch index. + """ + if self.skip_lr_decay: + return self.min_lrs + + decay_interval = decay_batch_iteration / self.decay_step_size + lr_decay_factor = (1 + self.decay_lr_rate * decay_interval) + lrs = [cycle_min_lr / lr_decay_factor for cycle_min_lr in self.min_lrs] + + return lrs + + def get_lr(self): + """Calculates the learning rate at batch index. This function treats + `self.last_batch_iteration` as the last batch index. + """ + if self.last_batch_iteration < self.total_size: + return self._get_cycle_lr() + return self._get_decay_lr(self.last_batch_iteration - self.total_size + 1) + + def get_mom(self): + """Calculates the momentum at batch index. This function treats + `self.last_batch_iteration` as the last batch index. + """ + if not self.cycle_momentum: + return None + + if self.last_batch_iteration < self.total_size: + return self._get_cycle_mom() + return self._get_decay_mom(self.last_batch_iteration - self.total_size + 1) + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def step(self, batch_iteration=None): + """ Updates the optimizer with the learning rate for the last batch index. + `self.last_batch_iteration` is treated as the last batch index. + + If self.cycle_momentum is true, also updates optimizer momentum. + """ + if batch_iteration is None: + batch_iteration = self.last_batch_iteration + 1 + + self.last_batch_iteration = batch_iteration + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + if self.cycle_momentum: + momentums = self.get_mom() + for param_group, momentum in zip(self.optimizer.param_groups, momentums): + param_group['betas'] = momentum + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + + +class WarmupLR(object): + """Increase the learning rate of each parameter group from min lr to max lr + over warmup_num_steps steps, and then fix at max lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + warmup_min_lr (float or list): minimum learning rate. Default: 0 + warmup_max_lr (float or list): maximum learning rate. Default: 0.001 + warmup_num_steps (int): number of steps to warm up from min_lr to max_lr. Default: 1000 + warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log + last_batch_iteration (int): The index of the last batch. Default: -1. + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = WarmupLR(optimizer) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + """ + + def __init__(self, + optimizer: Optimizer, + warmup_min_lr: float = 0.0, + warmup_max_lr: float = 0.001, + warmup_num_steps: int = 1000, + warmup_type: str = WARMUP_LOG_RATE, + last_batch_iteration: int = -1): + + self.optimizer = get_torch_optimizer(optimizer) + + self.min_lrs = self._format_param(self.optimizer, warmup_min_lr, "min_lr") + self.max_lrs = self._format_param(self.optimizer, warmup_max_lr, "max_lr") + self.delta_lrs = [big - small for big, small in zip(self.max_lrs, self.min_lrs)] + self.warmup_num_steps = max(2, warmup_num_steps) + # Currently only support linear and log function + if warmup_type not in {WARMUP_LOG_RATE, WARMUP_LINEAR_RATE}: + print(f"Using unknown warmup_type: {warmup_type}. The increasing function " + f"is set to default (log)") + warmup_type = WARMUP_LOG_RATE + self.warmup_type = warmup_type + self.inverse_log_warm_up = 1.0 / math.log(self.warmup_num_steps) + self.last_batch_iteration = last_batch_iteration + + def get_lr(self): + if self.last_batch_iteration < 0: + print("Attempting to get learning rate from scheduler before it has started") + return [0.0] + gamma = self._get_gamma() + return [min_lr + (delta_lr * gamma) for min_lr, delta_lr in zip(self.min_lrs, self.delta_lrs)] + + def get_last_lr(self): + """ Return last computed learning rate by current scheduler. + """ + assert getattr(self, '_last_lr', None) is not None, "need to call step() first" + return self._last_lr + + def step(self, last_batch_iteration=None): + if last_batch_iteration is None: + last_batch_iteration = self.last_batch_iteration + 1 + self.last_batch_iteration = last_batch_iteration + for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): + param_group['lr'] = lr + self._last_lr = [group['lr'] for group in self.optimizer.param_groups] + + def state_dict(self): + return {'last_batch_iteration': self.last_batch_iteration} + + def load_state_dict(self, sd): + self.last_batch_iteration = sd['last_batch_iteration'] + + def _get_gamma(self): + if self.last_batch_iteration < self.warmup_num_steps: + if self.warmup_type == WARMUP_LOG_RATE: + return self.inverse_log_warm_up * math.log(self.last_batch_iteration + 1) + elif self.warmup_type == WARMUP_LINEAR_RATE: + return self.last_batch_iteration / self.warmup_num_steps + return 1.0 + + def _format_param(self, optimizer, param_value, param_name): + if isinstance(param_value, list) or isinstance(param_value, tuple): + if len(param_value) != len(optimizer.param_groups): + raise ValueError("expected {} value for {}, got {}".format(len(optimizer.param_groups), param_name, + FileNotFoundError(param_value))) + return list(param_value) + return [param_value] * len(optimizer.param_groups) + + + +class WarmupDecayLR(WarmupLR): + """Increase the learning rate of each parameter group from min lr to max lr + over warmup_num_steps steps, and then decay at linear rate over the remaining training steps. + + Args: + optimizer (Optimizer): Wrapped optimizer. + total_num_steps (int): total number of training steps + warmup_min_lr (float or list): minimum learning rate. Default: 0 + warmup_max_lr (float or list): maximum learning rate. Default: 0.001 + warmup_num_steps (int): number of steps to warm up from min_lr to max_lr. Default: 1000 + warmup_type {‘log’, ‘linear’}: increasing function from min_lr to max_lr during warmup. Default: log + last_batch_iteration (int): The index of the last batch. Default: -1. + Example: + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = WarmupDecayLR(optimizer, 1000000) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + """ + + def __init__(self, + optimizer: Optimizer, + total_num_steps: int, + warmup_min_lr: float = 0.0, + warmup_max_lr: float = 0.001, + warmup_num_steps: int = 1000, + warmup_type: str = WARMUP_LOG_RATE, + last_batch_iteration: int = -1): + + self.total_num_steps = total_num_steps + super(WarmupDecayLR, self).__init__(optimizer, warmup_min_lr, warmup_max_lr, warmup_num_steps, warmup_type, + last_batch_iteration) + if self.total_num_steps < self.warmup_num_steps: + print('total_num_steps {} is less than warmup_num_steps {}'.format( + total_num_steps, warmup_num_steps)) + + def _get_gamma(self): + if self.last_batch_iteration < self.warmup_num_steps: + if self.warmup_type == WARMUP_LOG_RATE: + return self.inverse_log_warm_up * math.log(self.last_batch_iteration + 1) + elif self.warmup_type == WARMUP_LINEAR_RATE: + return self.last_batch_iteration / self.warmup_num_steps + return max( + 0.0, + float(self.total_num_steps - self.last_batch_iteration) / + float(max(1.0, self.total_num_steps - self.warmup_num_steps))) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/attn_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/attn_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..ba666e74d93e0d905f03b74b56aacc452877a272 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/attn_layers.py @@ -0,0 +1,447 @@ +import torch +import torch.nn as nn +from typing import Tuple, Union, Optional + +from utils.npu_utils import is_npu_available +if is_npu_available(): + import torch_npu + +try: + import flash_attn + if hasattr(flash_attn, '__version__') and int(flash_attn.__version__[0]) == 2: + from flash_attn.flash_attn_interface import flash_attn_kvpacked_func + from flash_attn.modules.mha import FlashSelfAttention, FlashCrossAttention + else: + from flash_attn.flash_attn_interface import flash_attn_unpadded_kvpacked_func + from flash_attn.modules.mha import FlashSelfAttention, FlashCrossAttention +except Exception as e: + print(f'flash_attn import failed: {e}') + + +class NpuFlashAttention(torch.nn.Module): + def __init__(self, attention_dropout=0): + super().__init__() + self.attention_dropout = attention_dropout + + def forward(self, query, key, value): + heads = query.shape[2] + attention_mask = None + output = torch_npu.npu_fusion_attention( + query, key, value, heads, input_layout='BSND', + pse=None, + atten_mask=attention_mask, + scale=1.0 / math.sqrt(query.shape[-1]), + pre_tockens=65536, + next_tockens=65536, + keep_prob=1. - self.attention_dropout, + sync=False, + inner_precise=0, + )[0] + return output + + +def reshape_for_broadcast(freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], x: torch.Tensor, head_first=False): + """ + Reshape frequency tensor for broadcasting it with another tensor. + + This function reshapes the frequency tensor to have the same shape as the target tensor 'x' + for the purpose of broadcasting the frequency tensor during element-wise operations. + + Args: + freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Frequency tensor to be reshaped. + x (torch.Tensor): Target tensor for broadcasting compatibility. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + torch.Tensor: Reshaped frequency tensor. + + Raises: + AssertionError: If the frequency tensor doesn't match the expected shape. + AssertionError: If the target tensor 'x' doesn't have the expected number of dimensions. + """ + ndim = x.ndim + assert 0 <= 1 < ndim + + if isinstance(freqs_cis, tuple): + # freqs_cis: (cos, sin) in real space + if head_first: + assert freqs_cis[0].shape == ( + x.shape[-2], x.shape[-1]), f'freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}' + shape = [d if i == ndim - 2 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + else: + assert freqs_cis[0].shape == ( + x.shape[1], x.shape[-1]), f'freqs_cis shape {freqs_cis[0].shape} does not match x shape {x.shape}' + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis[0].view(*shape), freqs_cis[1].view(*shape) + else: + # freqs_cis: values in complex space + if head_first: + assert freqs_cis.shape == ( + x.shape[-2], x.shape[-1]), f'freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}' + shape = [d if i == ndim - 2 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + else: + assert freqs_cis.shape == ( + x.shape[1], x.shape[-1]), f'freqs_cis shape {freqs_cis.shape} does not match x shape {x.shape}' + shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] + return freqs_cis.view(*shape) + + +def rotate_half(x): + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + return torch.stack([-x_imag, x_real], dim=-1).flatten(3) + + +def apply_rotary_emb( + xq: torch.Tensor, + xk: Optional[torch.Tensor], + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + head_first: bool = False, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. + + This function applies rotary embeddings to the given query 'xq' and key 'xk' tensors using the provided + frequency tensor 'freqs_cis'. The input tensors are reshaped as complex numbers, and the frequency tensor + is reshaped for broadcasting compatibility. The resulting tensors contain rotary embeddings and are + returned as real tensors. + + Args: + xq (torch.Tensor): Query tensor to apply rotary embeddings. [B, S, H, D] + xk (torch.Tensor): Key tensor to apply rotary embeddings. [B, S, H, D] + freqs_cis (Union[torch.Tensor, Tuple[torch.Tensor]]): Precomputed frequency tensor for complex exponentials. + head_first (bool): head dimension first (except batch dim) or not. + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + + """ + xk_out = None + if isinstance(freqs_cis, tuple): + cos, sin = reshape_for_broadcast(freqs_cis, xq, head_first) # [S, D] + cos, sin = cos.to(xq.device), sin.to(xq.device) + xq_out = xq * cos + rotate_half(xq) * sin + if xk is not None: + xk_out = xk * cos + rotate_half(xk) * sin + else: + xq_ = torch.view_as_complex(xq.reshape(*xq.shape[:-1], -1, 2)) # [B, S, H, D//2] + freqs_cis = reshape_for_broadcast(freqs_cis, xq_, head_first).to(xq.device) # [S, D//2] --> [1, S, 1, D//2] + xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) + if xk is not None: + xk_ = torch.view_as_complex(xk.reshape(*xk.shape[:-1], -1, 2)) # [B, S, H, D//2] + xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) + + return xq_out, xk_out + + +class FlashSelfMHAModified(nn.Module): + """ + Use QK Normalization. + """ + + def __init__(self, + dim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + FAG_deterministic=False, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.dim = dim + self.num_heads = num_heads + assert self.dim % num_heads == 0, "self.kdim must be divisible by num_heads" + self.head_dim = self.dim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + + self.Wqkv = nn.Linear(dim, 3 * dim, bias=qkv_bias, **factory_kwargs) + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6, + dtype=torch.float32) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6, + dtype=torch.float32) if qk_norm else nn.Identity() + if is_npu_available(): + self.inner_attn = NpuFlashAttention(attention_dropout=attn_drop) + else: + self.inner_attn = FlashSelfAttention(attention_dropout=attn_drop, deterministic=FAG_deterministic) + self.out_proj = nn.Linear(dim, dim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen, hidden_dim) (where hidden_dim = num heads * head dim) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // 2), RoPE for image + """ + b, s, d = x.shape + + qkv = self.Wqkv(x) + qkv = qkv.view(b, s, 3, self.num_heads, self.head_dim) # [b, s, 3, h, d] + q, k, v = qkv.unbind(dim=2) # [b, s, h, d] + input_dtype = qkv.dtype + q = self.q_norm(q) # [b, s, h, d] + k = self.k_norm(k) + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, kk = apply_rotary_emb(q, k, freqs_cis_img) + assert qq.shape == q.shape and kk.shape == k.shape, f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}' + q, k = qq, kk + + if is_npu_available(): + q = q.to(input_dtype) + k = k.to(input_dtype) + if q.dtype not in [torch.float16, torch.bfloat16] or k.dtype not in [torch.float16, torch.bfloat16] or v.dtype not in [torch.float16, torch.bfloat16]: + raise ValueError("The dtype of q/k/v must be torch.float16 or torch.bfloat16.") + context = self.inner_attn(q, k, v) + else: + qkv = torch.stack([q, k, v], dim=2) # [b, s, 3, h, d] + qkv = qkv.to(input_dtype) + context = self.inner_attn(qkv) + + out = self.out_proj(context.view(b, s, d)) + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class FlashCrossMHAModified(nn.Module): + """ + Use QK Normalization. + """ + + def __init__(self, + qdim, + kdim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + FAG_deterministic=False, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.qdim = qdim + self.kdim = kdim + self.num_heads = num_heads + assert self.qdim % num_heads == 0, "self.qdim must be divisible by num_heads" + self.head_dim = self.qdim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + + self.scale = self.head_dim ** -0.5 + + self.q_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.kv_proj = nn.Linear(kdim, 2 * qdim, bias=qkv_bias, **factory_kwargs) + + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6, + dtype=torch.float32) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6, + dtype=torch.float32) if qk_norm else nn.Identity() + + if is_npu_available(): + self.inner_attn = NpuFlashAttention(attention_dropout=attn_drop) + else: + self.inner_attn = FlashCrossAttention(attention_dropout=attn_drop, deterministic=FAG_deterministic) + + self.out_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, y, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen1, hidden_dim) (where hidden_dim = num_heads * head_dim) + y: torch.Tensor + (batch, seqlen2, hidden_dim2) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // num_heads), RoPE for image + """ + b, s1, _ = x.shape # [b, s1, D] + _, s2, _ = y.shape # [b, s2, 1024] + + q = self.q_proj(x).view(b, s1, self.num_heads, self.head_dim) # [b, s1, h, d] + kv = self.kv_proj(y).view(b, s2, 2, self.num_heads, self.head_dim) # [b, s2, 2, h, d] + input_dtype = kv.dtype + k, v = kv.unbind(dim=2) # [b, s2, h, d] + + q = self.q_norm(q) # [b, s1, h, d] + k = self.k_norm(k) # [b, s2, h, d] + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, _ = apply_rotary_emb(q, None, freqs_cis_img) + assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}' + q = qq # [b, s1, h, d] + + if is_npu_available(): + q = q.to(input_dtype) + k = k.to(input_dtype) + if q.dtype not in [torch.float16, torch.bfloat16] or k.dtype not in [torch.float16, torch.bfloat16] or v.dtype not in [torch.float16, torch.bfloat16]: + raise ValueError("The dtype of q/k/v must be torch.float16 or torch.bfloat16.") + context = self.inner_attn(q, k, v) + else: + kv = torch.stack([k, v], dim=2) # [b, s1, 2, h, d] + q = q.to(input_dtype) + kv = kv.to(input_dtype) + context = self.inner_attn(q, kv) # [b, s1, h, d] + + context = context.view(b, s1, -1) # [b, s1, D] + + out = self.out_proj(context) + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class CrossAttention(nn.Module): + """ + Use QK Normalization. + """ + + def __init__(self, + qdim, + kdim, + num_heads, + qkv_bias=True, + qk_norm=False, + attn_drop=0.0, + proj_drop=0.0, + device=None, + dtype=None, + norm_layer=nn.LayerNorm, + ): + factory_kwargs = {'device': device, 'dtype': dtype} + super().__init__() + self.qdim = qdim + self.kdim = kdim + self.num_heads = num_heads + assert self.qdim % num_heads == 0, "self.qdim must be divisible by num_heads" + self.head_dim = self.qdim // num_heads + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + self.scale = self.head_dim ** -0.5 + + self.q_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.kv_proj = nn.Linear(kdim, 2 * qdim, bias=qkv_bias, **factory_kwargs) + + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.out_proj = nn.Linear(qdim, qdim, bias=qkv_bias, **factory_kwargs) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, y, freqs_cis_img=None): + """ + Parameters + ---------- + x: torch.Tensor + (batch, seqlen1, hidden_dim) (where hidden_dim = num heads * head dim) + y: torch.Tensor + (batch, seqlen2, hidden_dim2) + freqs_cis_img: torch.Tensor + (batch, hidden_dim // 2), RoPE for image + """ + b, s1, c = x.shape # [b, s1, D] + _, s2, c = y.shape # [b, s2, 1024] + + q = self.q_proj(x).view(b, s1, self.num_heads, self.head_dim) # [b, s1, h, d] + kv = self.kv_proj(y).view(b, s2, 2, self.num_heads, self.head_dim) # [b, s2, 2, h, d] + k, v = kv.unbind(dim=2) # [b, s, h, d] + q = self.q_norm(q) + k = self.k_norm(k) + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, _ = apply_rotary_emb(q, None, freqs_cis_img) + assert qq.shape == q.shape, f'qq: {qq.shape}, q: {q.shape}' + q = qq + + q = q * self.scale + q = q.transpose(-2, -3).contiguous() # q -> B, L1, H, C - B, H, L1, C + k = k.permute(0, 2, 3, 1).contiguous() # k -> B, L2, H, C - B, H, C, L2 + attn = q @ k # attn -> B, H, L1, L2 + attn = attn.softmax(dim=-1) # attn -> B, H, L1, L2 + attn = self.attn_drop(attn) + x = attn @ v.transpose(-2, -3) # v -> B, L2, H, C - B, H, L2, C x-> B, H, L1, C + context = x.transpose(1, 2) # context -> B, H, L1, C - B, L1, H, C + + context = context.contiguous().view(b, s1, -1) + + out = self.out_proj(context) # context.reshape - B, L1, -1 + out = self.proj_drop(out) + + out_tuple = (out,) + + return out_tuple + + +class Attention(nn.Module): + """ + We rename some layer names to align with flash attention + """ + + def __init__(self, dim, num_heads, qkv_bias=True, qk_norm=False, attn_drop=0., proj_drop=0., + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + assert self.dim % num_heads == 0, 'dim should be divisible by num_heads' + self.head_dim = self.dim // num_heads + # This assertion is aligned with flash attention + assert self.head_dim % 8 == 0 and self.head_dim <= 128, "Only support head_dim <= 128 and divisible by 8" + self.scale = self.head_dim ** -0.5 + + # qkv --> Wqkv + self.Wqkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + # TODO: eps should be 1 / 65530 if using fp16 + self.q_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = norm_layer(self.head_dim, elementwise_affine=True, eps=1e-6) if qk_norm else nn.Identity() + self.attn_drop = nn.Dropout(attn_drop) + self.out_proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x, freqs_cis_img=None): + B, N, C = x.shape + qkv = self.Wqkv(x).reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4) # [3, b, h, s, d] + q, k, v = qkv.unbind(0) # [b, h, s, d] + q = self.q_norm(q) # [b, h, s, d] + k = self.k_norm(k) # [b, h, s, d] + + # Apply RoPE if needed + if freqs_cis_img is not None: + qq, kk = apply_rotary_emb(q, k, freqs_cis_img, head_first=True) + assert qq.shape == q.shape and kk.shape == k.shape, \ + f'qq: {qq.shape}, q: {q.shape}, kk: {kk.shape}, k: {k.shape}' + q, k = qq, kk + + q = q * self.scale + attn = q @ k.transpose(-2, -1) # [b, h, s, d] @ [b, h, d, s] + attn = attn.softmax(dim=-1) # [b, h, s, s] + attn = self.attn_drop(attn) + x = attn @ v # [b, h, s, d] + + x = x.transpose(1, 2).reshape(B, N, C) # [b, s, h, d] + x = self.out_proj(x) + x = self.proj_drop(x) + + out_tuple = (x,) + + return out_tuple diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c051f2b66ca7df022e0be2c66552d8b02d3e521f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/controlnet.py @@ -0,0 +1,273 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from timm.models.vision_transformer import Mlp + +from .attn_layers import Attention, FlashCrossMHAModified, FlashSelfMHAModified, CrossAttention +from .embedders import TimestepEmbedder, PatchEmbed, timestep_embedding +from .norm_layers import RMSNorm +from .poolers import AttentionPool + +from .models import FP32_Layernorm, FP32_SiLU, HunYuanDiTBlock + +def zero_module(module): + for p in module.parameters(): + nn.init.zeros_(p) + return module + +class HunYuanControlNet(ModelMixin, ConfigMixin): + """ + HunYuanDiT: Diffusion model with a Transformer backbone. + + Inherit ModelMixin and ConfigMixin to be compatible with the sampler StableDiffusionPipeline of diffusers. + + Parameters + ---------- + args: argparse.Namespace + The arguments parsed by argparse. + input_size: tuple + The size of the input image. + patch_size: int + The size of the patch. + in_channels: int + The number of input channels. + hidden_size: int + The hidden size of the transformer backbone. + depth: int + The number of transformer blocks. + num_heads: int + The number of attention heads. + mlp_ratio: float + The ratio of the hidden size of the MLP in the transformer block. + log_fn: callable + The logging function. + """ + @register_to_config + def __init__( + self, args, + input_size=(32, 32), + patch_size=2, + in_channels=4, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + log_fn=print, + ): + super().__init__() + self.args = args + self.log_fn = log_fn + self.depth = depth + self.learn_sigma = args.learn_sigma + self.in_channels = in_channels + self.patch_size = patch_size + self.num_heads = num_heads + self.hidden_size = hidden_size + self.text_states_dim = args.text_states_dim + self.text_states_dim_t5 = args.text_states_dim_t5 + self.text_len = args.text_len + self.text_len_t5 = args.text_len_t5 + self.norm = args.norm + + use_flash_attn = args.infer_mode == 'fa' or args.use_flash_attn + if use_flash_attn: + log_fn(f" Enable Flash Attention.") + qk_norm = args.qk_norm # See http://arxiv.org/abs/2302.05442 for details. + + self.mlp_t5 = nn.Sequential( + nn.Linear(self.text_states_dim_t5, self.text_states_dim_t5 * 4, bias=True), + FP32_SiLU(), + nn.Linear(self.text_states_dim_t5 * 4, self.text_states_dim, bias=True), + ) + # learnable replace + self.text_embedding_padding = nn.Parameter( + torch.randn(self.text_len + self.text_len_t5, self.text_states_dim, dtype=torch.float32)) + + # Attention pooling + self.pooler = AttentionPool(self.text_len_t5, self.text_states_dim_t5, num_heads=8, output_dim=1024) + + # Here we use a default learned embedder layer for future extension. + self.style_embedder = nn.Embedding(1, hidden_size) + + # Image size and crop size conditions + self.extra_in_dim = 256 * 6 + hidden_size + + # Text embedding for `add` + self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size) + self.t_embedder = TimestepEmbedder(hidden_size) + self.extra_in_dim += 1024 + self.extra_embedder = nn.Sequential( + nn.Linear(self.extra_in_dim, hidden_size * 4), + FP32_SiLU(), + nn.Linear(hidden_size * 4, hidden_size, bias=True), + ) + + # Image embedding + num_patches = self.x_embedder.num_patches + log_fn(f" Number of tokens: {num_patches}") + + # HUnYuanDiT Blocks + self.blocks = nn.ModuleList([ + HunYuanDiTBlock(hidden_size=hidden_size, + c_emb_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + text_states_dim=self.text_states_dim, + use_flash_attn=use_flash_attn, + qk_norm=qk_norm, + norm_type=self.norm, + skip=False, + ) + for _ in range(19) + ]) + + # Input zero linear for the first block + self.before_proj = zero_module(nn.Linear(self.hidden_size, self.hidden_size)) + + # Output zero linear for the every block + self.after_proj_list = nn.ModuleList( + [zero_module(nn.Linear(self.hidden_size, self.hidden_size)) for _ in range(len(self.blocks))] + ) + + self.fix_weight_modules = ['mlp_t5', 'text_embedding_padding', 'pooler', 'style_embedder', 'x_embedder', 't_embedder', 'extra_embedder'] + + + def from_dit(self, dit): + """ + Load the parameters from a pre-trained HunYuanDiT model. + + Parameters + ---------- + dit: HunYuanDiT + The pre-trained HunYuanDiT model. + """ + + + self.mlp_t5.load_state_dict(dit.mlp_t5.state_dict()) + + self.text_embedding_padding.data = dit.text_embedding_padding.data + self.pooler.load_state_dict(dit.pooler.state_dict()) + self.style_embedder.load_state_dict(dit.style_embedder.state_dict()) + self.x_embedder.load_state_dict(dit.x_embedder.state_dict()) + self.t_embedder.load_state_dict(dit.t_embedder.state_dict()) + self.extra_embedder.load_state_dict(dit.extra_embedder.state_dict()) + + for i, block in enumerate(self.blocks): + block.load_state_dict(dit.blocks[i].state_dict()) + + def set_trainable(self): + + self.mlp_t5.requires_grad_(False) + self.text_embedding_padding.requires_grad_(False) + self.pooler.requires_grad_(False) + self.style_embedder.requires_grad_(False) + self.x_embedder.requires_grad_(False) + self.t_embedder.requires_grad_(False) + self.extra_embedder.requires_grad_(False) + + self.blocks.requires_grad_(True) + self.before_proj.requires_grad_(True) + self.after_proj_list.requires_grad_(True) + + self.blocks.train() + self.before_proj.train() + self.after_proj_list.train() + + + + def forward(self, + x, + t, + condition, + encoder_hidden_states=None, + text_embedding_mask=None, + encoder_hidden_states_t5=None, + text_embedding_mask_t5=None, + image_meta_size=None, + style=None, + cos_cis_img=None, + sin_cis_img=None, + return_dict=True, + ): + """ + Forward pass of the encoder. + + Parameters + ---------- + x: torch.Tensor + (B, D, H, W) + t: torch.Tensor + (B) + encoder_hidden_states: torch.Tensor + CLIP text embedding, (B, L_clip, D) + text_embedding_mask: torch.Tensor + CLIP text embedding mask, (B, L_clip) + encoder_hidden_states_t5: torch.Tensor + T5 text embedding, (B, L_t5, D) + text_embedding_mask_t5: torch.Tensor + T5 text embedding mask, (B, L_t5) + image_meta_size: torch.Tensor + (B, 6) + style: torch.Tensor + (B) + cos_cis_img: torch.Tensor + sin_cis_img: torch.Tensor + return_dict: bool + Whether to return a dictionary. + """ + text_states = encoder_hidden_states # 2,77,1024 + text_states_t5 = encoder_hidden_states_t5 # 2,256,2048 + text_states_mask = text_embedding_mask.bool() # 2,77 + text_states_t5_mask = text_embedding_mask_t5.bool() # 2,256 + b_t5, l_t5, c_t5 = text_states_t5.shape + text_states_t5 = self.mlp_t5(text_states_t5.view(-1, c_t5)) + text_states = torch.cat([text_states, text_states_t5.view(b_t5, l_t5, -1)], dim=1) # 2,205，1024 + clip_t5_mask = torch.cat([text_states_mask, text_states_t5_mask], dim=-1) + + clip_t5_mask = clip_t5_mask + text_states = torch.where(clip_t5_mask.unsqueeze(2), text_states, self.text_embedding_padding.to(text_states)) + + _, _, oh, ow = x.shape + th, tw = oh // self.patch_size, ow // self.patch_size + + # ========================= Build time and image embedding ========================= + t = self.t_embedder(t) + x = self.x_embedder(x) + + # Get image RoPE embedding according to `reso`lution. + freqs_cis_img = (cos_cis_img, sin_cis_img) + + # ========================= Concatenate all extra vectors ========================= + # Build text tokens with pooling + extra_vec = self.pooler(encoder_hidden_states_t5) + + # Build image meta size tokens + image_meta_size = timestep_embedding(image_meta_size.view(-1), 256) # [B * 6, 256] + if self.args.use_fp16: + image_meta_size = image_meta_size.half() + image_meta_size = image_meta_size.view(-1, 6 * 256) + extra_vec = torch.cat([extra_vec, image_meta_size], dim=1) # [B, D + 6 * 256] + + # Build style tokens + style_embedding = self.style_embedder(style) + extra_vec = torch.cat([extra_vec, style_embedding], dim=1) + + # Concatenate all extra vectors + c = t + self.extra_embedder(extra_vec) # [B, D] + + # ========================= Deal with Condition ========================= + condition = self.x_embedder(condition) + + # ========================= Forward pass through HunYuanDiT blocks ========================= + controls = [] + x = x + self.before_proj(condition) # add condition + for layer, block in enumerate(self.blocks): + x = block(x, c, text_states, freqs_cis_img) + controls.append(self.after_proj_list[layer](x)) # zero linear for output + + + if return_dict: + return {'controls': controls} + return controls \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/ema.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..acc31ee296f7f30bb37f101fdd911539df54e20a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/ema.py @@ -0,0 +1,127 @@ +from collections import OrderedDict +from copy import deepcopy + +import torch +from deepspeed.utils import instrument_w_nvtx +from pathlib import Path + +def requires_grad(model, flag=True): + """ + Set requires_grad flag for all parameters in a model. + """ + for p in model.parameters(): + p.requires_grad = flag + + +class EMA(object): + def __init__(self, args, model, device, logger): + if args.ema_dtype == 'fp32': + self.warmup = args.ema_warmup + self.update_after_step = 0 + self.max_value = args.ema_decay if args.ema_decay is not None else 0.9999 + self.inv_gamma = 1.0 + self.power = args.ema_warmup_power if args.ema_warmup_power is not None else 2 / 3 + self.min_value = 0.0 + else: + self.warmup = args.ema_warmup + self.update_after_step = 0 + self.max_value = args.ema_decay if args.ema_decay is not None else 0.992 + self.inv_gamma = 1.0 + self.power = args.ema_warmup_power if args.ema_warmup_power is not None else 0.446249 + # 0.446249 == math.log(1 - 0.992) / math.log(50000) + self.min_value = 0.0 + + self.ema_reset_decay = args.ema_reset_decay + self.decay_steps = 0 + + if args.ema_dtype == 'none': + ema_dtype = 'fp16' if args.use_fp16 else 'fp32' + else: + ema_dtype = args.ema_dtype + + # 由于module.half()和module.float()会发生inplace类型修改，因此需要先copy后修改类型 + self.ema_model = deepcopy(model) + if ema_dtype == 'fp16': + self.ema_model = self.ema_model.half().to(device) + elif ema_dtype == 'fp32': + self.ema_model = self.ema_model.float().to(device) + else: + raise ValueError(f"Unknown EMA dtype {ema_dtype}.") + + requires_grad(self.ema_model, False) + + logger.info(f" Using EMA with date type {args.ema_dtype} " + f"(decay={args.ema_decay}, warmup={args.ema_warmup}, warmup_power={args.ema_warmup_power}, " + f"reset_decay={args.ema_reset_decay}).") + + def get_decay(self): + """ + @crowsonkb's notes on EMA Warmup: + If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan + to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), + gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 + at 215.4k steps). + + @jarvizhang's notes on EMA max_value when enabling FP16: + If using FP16 for EMA, max_value=0.995 is better (Don't larger than 0.999, unless you know + what you are doing). This is because FP16 has less precision than FP32, so the EMA value can + be pushed out of the range of FP16. + + gamma=1, power=0.446249 are good values for models (reaches decay factor 0.99 at 30K steps, + 0.992 at 50K steps). + """ + if self.warmup: + step = max(0, self.decay_steps - self.update_after_step - 1) + value = 1 - (1 + step / self.inv_gamma) ** -self.power + + if step <= 0: + return 0.0 + + return max(self.min_value, min(value, self.max_value)) + else: + return self.max_value + + @torch.no_grad() + @instrument_w_nvtx + def update(self, model, step, decay=None): + """ + Step the EMA model towards the current model. + + Parameters + ---------- + model: nn.Module + The current model + step: int + The current training step. This is used to determine the decay factor. If you want to control + the decay, you can pass in a custom step instead. + For example, if you want to restart the EMA decay, you can pass in step=0 at start and increase + step by step. + decay: float + The decay factor. If None, will be determined by the current step. + """ + if decay is None: + if self.ema_reset_decay: + self.decay_steps += 1 + else: + self.decay_steps = step + decay = self.get_decay() + + ema_params = OrderedDict(self.ema_model.named_parameters()) + model_params = OrderedDict(model.named_parameters()) + for name, param in model_params.items(): + # TODO: Consider applying only to params that require_grad to avoid small numerical changes of pos_embed + ema_params[name].mul_(decay).add_(param.data, alpha=1 - decay) + + return None + + def state_dict(self, *args, **kwargs): + return self.ema_model.state_dict(*args, **kwargs) + + def load_state_dict(self, *args, **kwargs): + return self.ema_model.load_state_dict(*args, **kwargs) + + def train(self): + self.ema_model.train() + + def eval(self): + self.ema_model.eval() diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/embedders.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/embedders.py new file mode 100644 index 0000000000000000000000000000000000000000..9fe08cba22eef41ca9fd9f70fe6f062a4dd606c8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/embedders.py @@ -0,0 +1,111 @@ +import math +import torch +import torch.nn as nn +from einops import repeat + +from timm.models.layers import to_2tuple + + +class PatchEmbed(nn.Module): + """ 2D Image to Patch Embedding + + Image to Patch Embedding using Conv2d + + A convolution based approach to patchifying a 2D image w/ embedding projection. + + Based on the impl in https://github.com/google-research/vision_transformer + + Hacked together by / Copyright 2020 Ross Wightman + + Remove the _assert function in forward function to be compatible with multi-resolution images. + """ + def __init__( + self, + img_size=224, + patch_size=16, + in_chans=3, + embed_dim=768, + norm_layer=None, + flatten=True, + bias=True, + ): + super().__init__() + if isinstance(img_size, int): + img_size = to_2tuple(img_size) + elif isinstance(img_size, (tuple, list)) and len(img_size) == 2: + img_size = tuple(img_size) + else: + raise ValueError(f"img_size must be int or tuple/list of length 2. Got {img_size}") + patch_size = to_2tuple(patch_size) + self.img_size = img_size + self.patch_size = patch_size + self.grid_size = (img_size[0] // patch_size[0], img_size[1] // patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + self.flatten = flatten + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size, bias=bias) + self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity() + + def update_image_size(self, img_size): + self.img_size = img_size + self.grid_size = (img_size[0] // self.patch_size[0], img_size[1] // self.patch_size[1]) + self.num_patches = self.grid_size[0] * self.grid_size[1] + + def forward(self, x): + # B, C, H, W = x.shape + # _assert(H == self.img_size[0], f"Input image height ({H}) doesn't match model ({self.img_size[0]}).") + # _assert(W == self.img_size[1], f"Input image width ({W}) doesn't match model ({self.img_size[1]}).") + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) # BCHW -> BNC + x = self.norm(x) + return x + + +def timestep_embedding(t, dim, max_period=10000, repeat_only=False): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + # https://github.com/openai/glide-text2im/blob/main/glide_text2im/nn.py + if not repeat_only: + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) + * torch.arange(start=0, end=half, dtype=torch.float32) + / half + ).to(device=t.device) # size: [dim/2], 一个指数衰减的曲线 + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat( + [embedding, torch.zeros_like(embedding[:, :1])], dim=-1 + ) + else: + embedding = repeat(t, "b -> b d", d=dim) + return embedding + + +class TimestepEmbedder(nn.Module): + """ + Embeds scalar timesteps into vector representations. + """ + def __init__(self, hidden_size, frequency_embedding_size=256, out_size=None): + super().__init__() + if out_size is None: + out_size = hidden_size + self.mlp = nn.Sequential( + nn.Linear(frequency_embedding_size, hidden_size, bias=True), + nn.SiLU(), + nn.Linear(hidden_size, out_size, bias=True), + ) + self.frequency_embedding_size = frequency_embedding_size + + def forward(self, t): + t_freq = timestep_embedding(t, self.frequency_embedding_size).type(self.mlp[0].weight.dtype) + t_emb = self.mlp(t_freq) + return t_emb diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/fp16_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/fp16_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..6812811045eccbacde3a60fc3d230cd3fb9da0ad --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/fp16_layers.py @@ -0,0 +1,86 @@ +import torch +from torch.autograd import Variable +from torch.nn.parameter import Parameter + + +_FLOAT_TYPES = (torch.FloatTensor, torch.cuda.FloatTensor) +_HALF_TYPES = (torch.HalfTensor, torch.cuda.HalfTensor) + + +def conversion_helper(val, conversion): + """Apply conversion to val. Recursively apply conversion if `val` + #is a nested tuple/list structure.""" + if isinstance(val, dict): + res_dict = {} + for k, v in val.items(): + if k!= 'cos_cis_img' and k != 'sin_cis_img': + res_dict[k] = conversion_helper(v, conversion) + else: + res_dict[k] = v + return res_dict + if not isinstance(val, (tuple, list)): + return conversion(val) + rtn = [conversion_helper(v, conversion) for v in val] + if isinstance(val, tuple): + rtn = tuple(rtn) + return rtn + + +def fp32_to_float16(val, float16_convertor): + """Convert fp32 `val` to fp16/bf16""" + def half_conversion(val): + val_typecheck = val + if isinstance(val_typecheck, (Parameter, Variable)): + val_typecheck = val.data + if isinstance(val_typecheck, _FLOAT_TYPES): + val = float16_convertor(val) + return val + return conversion_helper(val, half_conversion) + + +def float16_to_fp32(val): + """Convert fp16/bf16 `val` to fp32""" + def float_conversion(val): + val_typecheck = val + if isinstance(val_typecheck, (Parameter, Variable)): + val_typecheck = val.data + if isinstance(val_typecheck, (_HALF_TYPES,)): + val = val.float() + return val + return conversion_helper(val, float_conversion) + + +class Float16Module(torch.nn.Module): + + def __init__(self, module, args): + super(Float16Module, self).__init__() + + self.add_module('module', module.half()) + + def float16_convertor(val): + return val.half() + + self.float16_convertor = float16_convertor + + self.config = self.module.config + self.dtype = torch.float16 + + def forward(self, *inputs, **kwargs): + inputs = fp32_to_float16(inputs, self.float16_convertor) + kwargs = fp32_to_float16(kwargs, self.float16_convertor) + outputs = self.module(*inputs, **kwargs) + outputs = float16_to_fp32(outputs) + return outputs + + def state_dict(self, destination=None, prefix='', keep_vars=False): + return self.module.state_dict(destination, prefix, keep_vars) + + + def state_dict_for_save_checkpoint(self, destination=None, prefix='', + keep_vars=False): + return self.module.state_dict_for_save_checkpoint(destination, prefix, + keep_vars) + + def load_state_dict(self, state_dict, strict=True): + self.module.load_state_dict(state_dict, strict=strict) + diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/models.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/models.py new file mode 100644 index 0000000000000000000000000000000000000000..69dfc4384f1ecf579e8ac32a91c264d21ac04573 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/models.py @@ -0,0 +1,508 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from timm.models.vision_transformer import Mlp + +from .attn_layers import Attention, FlashCrossMHAModified, FlashSelfMHAModified, CrossAttention +from .embedders import TimestepEmbedder, PatchEmbed, timestep_embedding +from .norm_layers import RMSNorm +from .poolers import AttentionPool + +from transformers.integrations import PeftAdapterMixin +from typing import Any, Optional, Union +from tqdm import tqdm +from peft.utils import ( + ModulesToSaveWrapper, + _get_submodules, +) + + +def modulate(x, shift, scale): + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1) + + +class FP32_Layernorm(nn.LayerNorm): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + origin_dtype = inputs.dtype + return F.layer_norm(inputs.float(), self.normalized_shape, self.weight.float(), self.bias.float(), + self.eps).to(origin_dtype) + + +class FP32_SiLU(nn.SiLU): + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + return torch.nn.functional.silu(inputs.float(), inplace=False).to(inputs.dtype) + + +class HunYuanDiTBlock(nn.Module): + """ + A HunYuanDiT block with `add` conditioning. + """ + def __init__(self, + hidden_size, + c_emb_size, + num_heads, + mlp_ratio=4.0, + text_states_dim=1024, + use_flash_attn=False, + qk_norm=False, + norm_type="layer", + skip=False, + ): + super().__init__() + self.use_flash_attn = use_flash_attn + use_ele_affine = True + + if norm_type == "layer": + norm_layer = FP32_Layernorm + elif norm_type == "rms": + norm_layer = RMSNorm + else: + raise ValueError(f"Unknown norm_type: {norm_type}") + + # ========================= Self-Attention ========================= + self.norm1 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + if use_flash_attn: + self.attn1 = FlashSelfMHAModified(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + else: + self.attn1 = Attention(hidden_size, num_heads=num_heads, qkv_bias=True, qk_norm=qk_norm) + + # ========================= FFN ========================= + self.norm2 = norm_layer(hidden_size, elementwise_affine=use_ele_affine, eps=1e-6) + mlp_hidden_dim = int(hidden_size * mlp_ratio) + approx_gelu = lambda: nn.GELU(approximate="tanh") + self.mlp = Mlp(in_features=hidden_size, hidden_features=mlp_hidden_dim, act_layer=approx_gelu, drop=0) + + # ========================= Add ========================= + # Simply use add like SDXL. + self.default_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, hidden_size, bias=True) + ) + + # ========================= Cross-Attention ========================= + if use_flash_attn: + self.attn2 = FlashCrossMHAModified(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + else: + self.attn2 = CrossAttention(hidden_size, text_states_dim, num_heads=num_heads, qkv_bias=True, + qk_norm=qk_norm) + self.norm3 = norm_layer(hidden_size, elementwise_affine=True, eps=1e-6) + + # ========================= Skip Connection ========================= + if skip: + self.skip_norm = norm_layer(2 * hidden_size, elementwise_affine=True, eps=1e-6) + self.skip_linear = nn.Linear(2 * hidden_size, hidden_size) + else: + self.skip_linear = None + + def forward(self, x, c=None, text_states=None, freq_cis_img=None, skip=None): + # Long Skip Connection + if self.skip_linear is not None: + cat = torch.cat([x, skip], dim=-1) + cat = self.skip_norm(cat) + x = self.skip_linear(cat) + + # Self-Attention + shift_msa = self.default_modulation(c).unsqueeze(dim=1) + attn_inputs = ( + self.norm1(x) + shift_msa, freq_cis_img, + ) + x = x + self.attn1(*attn_inputs)[0] + + # Cross-Attention + cross_inputs = ( + self.norm3(x), text_states, freq_cis_img + ) + x = x + self.attn2(*cross_inputs)[0] + + # FFN Layer + mlp_inputs = self.norm2(x) + x = x + self.mlp(mlp_inputs) + + return x + + +class FinalLayer(nn.Module): + """ + The final layer of HunYuanDiT. + """ + def __init__(self, final_hidden_size, c_emb_size, patch_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(final_hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(final_hidden_size, patch_size * patch_size * out_channels, bias=True) + self.adaLN_modulation = nn.Sequential( + FP32_SiLU(), + nn.Linear(c_emb_size, 2 * final_hidden_size, bias=True) + ) + + def forward(self, x, c): + shift, scale = self.adaLN_modulation(c).chunk(2, dim=1) + x = modulate(self.norm_final(x), shift, scale) + x = self.linear(x) + return x + + +class HunYuanDiT(ModelMixin, ConfigMixin, PeftAdapterMixin): + """ + HunYuanDiT: Diffusion model with a Transformer backbone. + + Inherit ModelMixin and ConfigMixin to be compatible with the sampler StableDiffusionPipeline of diffusers. + + Inherit PeftAdapterMixin to be compatible with the PEFT training pipeline. + + Parameters + ---------- + args: argparse.Namespace + The arguments parsed by argparse. + input_size: tuple + The size of the input image. + patch_size: int + The size of the patch. + in_channels: int + The number of input channels. + hidden_size: int + The hidden size of the transformer backbone. + depth: int + The number of transformer blocks. + num_heads: int + The number of attention heads. + mlp_ratio: float + The ratio of the hidden size of the MLP in the transformer block. + log_fn: callable + The logging function. + """ + @register_to_config + def __init__( + self, args, + input_size=(32, 32), + patch_size=2, + in_channels=4, + hidden_size=1152, + depth=28, + num_heads=16, + mlp_ratio=4.0, + log_fn=print, + ): + super().__init__() + self.args = args + self.log_fn = log_fn + self.depth = depth + self.learn_sigma = args.learn_sigma + self.in_channels = in_channels + self.out_channels = in_channels * 2 if args.learn_sigma else in_channels + self.patch_size = patch_size + self.num_heads = num_heads + self.hidden_size = hidden_size + self.text_states_dim = args.text_states_dim + self.text_states_dim_t5 = args.text_states_dim_t5 + self.text_len = args.text_len + self.text_len_t5 = args.text_len_t5 + self.norm = args.norm + + use_flash_attn = args.infer_mode == 'fa' or args.use_flash_attn + if use_flash_attn: + log_fn(f" Enable Flash Attention.") + qk_norm = args.qk_norm # See http://arxiv.org/abs/2302.05442 for details. + + self.mlp_t5 = nn.Sequential( + nn.Linear(self.text_states_dim_t5, self.text_states_dim_t5 * 4, bias=True), + FP32_SiLU(), + nn.Linear(self.text_states_dim_t5 * 4, self.text_states_dim, bias=True), + ) + # learnable replace + self.text_embedding_padding = nn.Parameter( + torch.randn(self.text_len + self.text_len_t5, self.text_states_dim)) + + # Attention pooling + self.pooler = AttentionPool(self.text_len_t5, self.text_states_dim_t5, num_heads=8, output_dim=1024) + + # Here we use a default learned embedder layer for future extension. + self.style_embedder = nn.Embedding(1, hidden_size) + + # Image size and crop size conditions + self.extra_in_dim = 256 * 6 + hidden_size + + # Text embedding for `add` + self.x_embedder = PatchEmbed(input_size, patch_size, in_channels, hidden_size) + self.t_embedder = TimestepEmbedder(hidden_size) + self.extra_in_dim += 1024 + self.extra_embedder = nn.Sequential( + nn.Linear(self.extra_in_dim, hidden_size * 4), + FP32_SiLU(), + nn.Linear(hidden_size * 4, hidden_size, bias=True), + ) + + # Image embedding + num_patches = self.x_embedder.num_patches + log_fn(f" Number of tokens: {num_patches}") + + # HUnYuanDiT Blocks + self.blocks = nn.ModuleList([ + HunYuanDiTBlock(hidden_size=hidden_size, + c_emb_size=hidden_size, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + text_states_dim=self.text_states_dim, + use_flash_attn=use_flash_attn, + qk_norm=qk_norm, + norm_type=self.norm, + skip=layer > depth // 2, + ) + for layer in range(depth) + ]) + + self.final_layer = FinalLayer(hidden_size, hidden_size, patch_size, self.out_channels) + self.unpatchify_channels = self.out_channels + + self.initialize_weights() + + def forward(self, + x, + t, + encoder_hidden_states=None, + text_embedding_mask=None, + encoder_hidden_states_t5=None, + text_embedding_mask_t5=None, + image_meta_size=None, + style=None, + cos_cis_img=None, + sin_cis_img=None, + return_dict=True, + controls=None, + ): + """ + Forward pass of the encoder. + + Parameters + ---------- + x: torch.Tensor + (B, D, H, W) + t: torch.Tensor + (B) + encoder_hidden_states: torch.Tensor + CLIP text embedding, (B, L_clip, D) + text_embedding_mask: torch.Tensor + CLIP text embedding mask, (B, L_clip) + encoder_hidden_states_t5: torch.Tensor + T5 text embedding, (B, L_t5, D) + text_embedding_mask_t5: torch.Tensor + T5 text embedding mask, (B, L_t5) + image_meta_size: torch.Tensor + (B, 6) + style: torch.Tensor + (B) + cos_cis_img: torch.Tensor + sin_cis_img: torch.Tensor + return_dict: bool + Whether to return a dictionary. + """ + text_states = encoder_hidden_states # 2,77,1024 + text_states_t5 = encoder_hidden_states_t5 # 2,256,2048 + text_states_mask = text_embedding_mask.bool() # 2,77 + text_states_t5_mask = text_embedding_mask_t5.bool() # 2,256 + b_t5, l_t5, c_t5 = text_states_t5.shape + text_states_t5 = self.mlp_t5(text_states_t5.view(-1, c_t5)) + text_states = torch.cat([text_states, text_states_t5.view(b_t5, l_t5, -1)], dim=1) # 2,205，1024 + clip_t5_mask = torch.cat([text_states_mask, text_states_t5_mask], dim=-1) + + clip_t5_mask = clip_t5_mask + text_states = torch.where(clip_t5_mask.unsqueeze(2), text_states, self.text_embedding_padding.to(text_states)) + + _, _, oh, ow = x.shape + th, tw = oh // self.patch_size, ow // self.patch_size + + # ========================= Build time and image embedding ========================= + t = self.t_embedder(t) + x = self.x_embedder(x) + + # Get image RoPE embedding according to `reso`lution. + freqs_cis_img = (cos_cis_img, sin_cis_img) + + # ========================= Concatenate all extra vectors ========================= + # Build text tokens with pooling + extra_vec = self.pooler(encoder_hidden_states_t5) + + # Build image meta size tokens + image_meta_size = timestep_embedding(image_meta_size.view(-1), 256) # [B * 6, 256] + if self.args.use_fp16 and self.args.autocast_dtype == "fp16": + image_meta_size = image_meta_size.half() + image_meta_size = image_meta_size.view(-1, 6 * 256) + extra_vec = torch.cat([extra_vec, image_meta_size], dim=1) # [B, D + 6 * 256] + + # Build style tokens + style_embedding = self.style_embedder(style) + extra_vec = torch.cat([extra_vec, style_embedding], dim=1) + + # Concatenate all extra vectors + c = t + self.extra_embedder(extra_vec) # [B, D] + + # ========================= Forward pass through HunYuanDiT blocks ========================= + skips = [] + for layer, block in enumerate(self.blocks): + if layer > self.depth // 2: + if controls is not None: + skip = skips.pop() + controls.pop() + else: + skip = skips.pop() + x = block(x, c, text_states, freqs_cis_img, skip) # (N, L, D) + else: + x = block(x, c, text_states, freqs_cis_img) # (N, L, D) + + if layer < (self.depth // 2 - 1): + skips.append(x) + if controls is not None and len(controls) != 0: + raise ValueError("The number of controls is not equal to the number of skip connections.") + + # ========================= Final layer ========================= + x = self.final_layer(x, c) # (N, L, patch_size ** 2 * out_channels) + x = self.unpatchify(x, th, tw) # (N, out_channels, H, W) + + if return_dict: + return {'x': x} + return x + + def initialize_weights(self): + # Initialize transformer layers: + def _basic_init(module): + if isinstance(module, nn.Linear): + torch.nn.init.xavier_uniform_(module.weight) + if module.bias is not None: + nn.init.constant_(module.bias, 0) + self.apply(_basic_init) + + # Initialize patch_embed like nn.Linear (instead of nn.Conv2d): + w = self.x_embedder.proj.weight.data + nn.init.xavier_uniform_(w.view([w.shape[0], -1])) + nn.init.constant_(self.x_embedder.proj.bias, 0) + + # Initialize label embedding table: + nn.init.normal_(self.extra_embedder[0].weight, std=0.02) + nn.init.normal_(self.extra_embedder[2].weight, std=0.02) + + # Initialize timestep embedding MLP: + nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02) + nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02) + + # Zero-out adaLN modulation layers in HunYuanDiT blocks: + for block in self.blocks: + nn.init.constant_(block.default_modulation[-1].weight, 0) + nn.init.constant_(block.default_modulation[-1].bias, 0) + + # Zero-out output layers: + nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0) + nn.init.constant_(self.final_layer.adaLN_modulation[-1].bias, 0) + nn.init.constant_(self.final_layer.linear.weight, 0) + nn.init.constant_(self.final_layer.linear.bias, 0) + + def unpatchify(self, x, h, w): + """ + x: (N, T, patch_size**2 * C) + imgs: (N, H, W, C) + """ + c = self.unpatchify_channels + p = self.x_embedder.patch_size[0] + # h = w = int(x.shape[1] ** 0.5) + assert h * w == x.shape[1] + + x = x.reshape(shape=(x.shape[0], h, w, p, p, c)) + x = torch.einsum('nhwpqc->nchpwq', x) + imgs = x.reshape(shape=(x.shape[0], c, h * p, w * p)) + return imgs + + def _replace_module(self, parent, child_name, new_module, child) -> None: + setattr(parent, child_name, new_module) + # It's not necessary to set requires_grad here, as that is handled by + # _mark_only_adapters_as_trainable + + # child layer wraps the original module, unpack it + if hasattr(child, "base_layer"): + child = child.get_base_layer() + elif hasattr(child, "quant_linear_module"): + # TODO maybe not necessary to have special treatment? + child = child.quant_linear_module + + if not hasattr(new_module, "base_layer"): + new_module.weight = child.weight + if hasattr(child, "bias"): + new_module.bias = child.bias + + if getattr(child, "state", None) is not None: + if hasattr(new_module, "base_layer"): + new_module.base_layer.state = child.state + else: + new_module.state = child.state + new_module.to(child.weight.device) + + # dispatch to correct device + for name, module in new_module.named_modules(): + # if any(prefix in name for prefix in PREFIXES): + # module.to(child.weight.device) + if "ranknum" in name: + module.to(child.weight.device) + + def merge_and_unload(self, + merge=True, + progressbar: bool = False, + safe_merge: bool = False, + adapter_names = None,): + if merge: + if getattr(self, "quantization_method", None) == "gptq": + raise ValueError("Cannot merge layers when the model is gptq quantized") + + def merge_recursively(module): + # helper function to recursively merge the base_layer of the target + path = [] + layer = module + while hasattr(layer, "base_layer"): + path.append(layer) + layer = layer.base_layer + for layer_before, layer_after in zip(path[:-1], path[1:]): + layer_after.merge(safe_merge=safe_merge, adapter_names=adapter_names) + layer_before.base_layer = layer_after.base_layer + module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + + key_list = [key for key, _ in self.named_modules()] + desc = "Unloading " + ("and merging " if merge else "") + "model" + + for key in tqdm(key_list, disable=not progressbar, desc=desc): + try: + parent, target, target_name = _get_submodules(self, key) + except AttributeError: + continue + + if hasattr(target, "base_layer"): + if merge: + merge_recursively(target) + self._replace_module(parent, target_name, target.get_base_layer(), target) + elif isinstance(target, ModulesToSaveWrapper): + # save any additional trainable modules part of `modules_to_save` + new_module = target.modules_to_save[target.active_adapter] + if hasattr(new_module, "base_layer"): + # check if the module is itself a tuner layer + if merge: + new_module.merge(safe_merge=safe_merge, adapter_names=adapter_names) + new_module = new_module.get_base_layer() + setattr(parent, target_name, new_module) + + + +################################################################################# +# HunYuanDiT Configs # +################################################################################# + +HUNYUAN_DIT_CONFIG = { + 'DiT-g/2': {'depth': 40, 'hidden_size': 1408, 'patch_size': 2, 'num_heads': 16, 'mlp_ratio': 4.3637}, + 'DiT-XL/2': {'depth': 28, 'hidden_size': 1152, 'patch_size': 2, 'num_heads': 16}, +} + +def DiT_g_2(args, **kwargs): + return HunYuanDiT(args, depth=40, hidden_size=1408, patch_size=2, num_heads=16, mlp_ratio=4.3637, **kwargs) +def DiT_XL_2(args, **kwargs): + return HunYuanDiT(args, depth=28, hidden_size=1152, patch_size=2, num_heads=16, **kwargs) + +HUNYUAN_DIT_MODELS = { + 'DiT-g/2': DiT_g_2, + 'DiT-XL/2': DiT_XL_2, +} \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/norm_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/norm_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..3c25bc3f3b99047372982943027942cf7f2ded3a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/norm_layers.py @@ -0,0 +1,74 @@ +import torch +import torch.nn as nn +from utils.npu_utils import is_npu_available +if is_npu_available(): + import torch_npu + + +class RMSNorm(nn.Module): + def __init__(self, dim: int, elementwise_affine=True, eps: float = 1e-6): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + super().__init__() + self.eps = eps + if is_npu_available() or elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim)) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + if is_npu_available(): + output = torch_npu.npu_rms_norm(x, self.weight, epsilon=self.eps)[0] + else: + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +class GroupNorm32(nn.GroupNorm): + def __init__(self, num_groups, num_channels, eps=1e-5, dtype=None): + super().__init__(num_groups=num_groups, num_channels=num_channels, eps=eps, dtype=dtype) + + def forward(self, x): + y = super().forward(x).to(x.dtype) + return y + +def normalization(channels, dtype=None): + """ + Make a standard normalization layer. + :param channels: number of input channels. + :return: an nn.Module for normalization. + """ + return GroupNorm32(num_channels=channels, num_groups=32, dtype=dtype) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/poolers.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/poolers.py new file mode 100644 index 0000000000000000000000000000000000000000..a4adcaca51fded2268a644ca4c70d5b33dfcd3b0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/poolers.py @@ -0,0 +1,39 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class AttentionPool(nn.Module): + def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter(torch.randn(spacial_dim + 1, embed_dim) / embed_dim ** 0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.permute(1, 0, 2) # NLC -> LNC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (L+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (L+1)NC + x, _ = F.multi_head_attention_forward( + query=x[:1], key=x, value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False + ) + return x.squeeze(0) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/posemb_layers.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/posemb_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..dcb41a713cd94ea8472ff26e8865066887b1e486 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/posemb_layers.py @@ -0,0 +1,224 @@ +import torch +import numpy as np +from typing import Union + + +def _to_tuple(x): + if isinstance(x, int): + return x, x + else: + return x + + +def get_fill_resize_and_crop(src, tgt): + th, tw = _to_tuple(tgt) + h, w = _to_tuple(src) + + tr = th / tw # base resolution + r = h / w # target resolution + + # resize + if r > tr: + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) # resize the target resolution down based on the base resolution + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +def get_meshgrid(start, *args): + if len(args) == 0: + # start is grid_size + num = _to_tuple(start) + start = (0, 0) + stop = num + elif len(args) == 1: + # start is start, args[0] is stop, step is 1 + start = _to_tuple(start) + stop = _to_tuple(args[0]) + num = (stop[0] - start[0], stop[1] - start[1]) + elif len(args) == 2: + # start is start, args[0] is stop, args[1] is num + start = _to_tuple(start) + stop = _to_tuple(args[0]) + num = _to_tuple(args[1]) + else: + raise ValueError(f"len(args) should be 0, 1 or 2, but got {len(args)}") + + grid_h = np.linspace(start[0], stop[0], num[0], endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], num[1], endpoint=False, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) # [2, W, H] + return grid + +################################################################################# +# Sine/Cosine Positional Embedding Functions # +################################################################################# +# https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py + +def get_2d_sincos_pos_embed(embed_dim, start, *args, cls_token=False, extra_tokens=0): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid = get_meshgrid(start, *args) # [2, H, w] + # grid_h = np.arange(grid_size, dtype=np.float32) + # grid_w = np.arange(grid_size, dtype=np.float32) + # grid = np.meshgrid(grid_w, grid_h) # here w goes first + # grid = np.stack(grid, axis=0) # [2, W, H] + + grid = grid.reshape([2, 1, *grid.shape[1:]]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (W,H) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2. + omega = 1. / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +################################################################################# +# Rotary Positional Embedding Functions # +################################################################################# +# https://github.com/facebookresearch/llama/blob/main/llama/model.py#L443 + +def get_2d_rotary_pos_embed(embed_dim, start, *args, use_real=True): + """ + This is a 2d version of precompute_freqs_cis, which is a RoPE for image tokens with 2d structure. + + Parameters + ---------- + embed_dim: int + embedding dimension size + start: int or tuple of int + If len(args) == 0, start is num; If len(args) == 1, start is start, args[0] is stop, step is 1; + If len(args) == 2, start is start, args[0] is stop, args[1] is num. + use_real: bool + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + + Returns + ------- + pos_embed: torch.Tensor + [HW, D/2] + """ + grid = get_meshgrid(start, *args) # [2, H, w] + grid = grid.reshape([2, 1, *grid.shape[1:]]) # Returns a sampling matrix with the same resolution as the target resolution + pos_embed = get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=use_real) + return pos_embed + + +def get_2d_rotary_pos_embed_from_grid(embed_dim, grid, use_real=False): + assert embed_dim % 4 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_rotary_pos_embed(embed_dim // 2, grid[0].reshape(-1), use_real=use_real) # (H*W, D/4) + emb_w = get_1d_rotary_pos_embed(embed_dim // 2, grid[1].reshape(-1), use_real=use_real) # (H*W, D/4) + + if use_real: + cos = torch.cat([emb_h[0], emb_w[0]], dim=1) # (H*W, D/2) + sin = torch.cat([emb_h[1], emb_w[1]], dim=1) # (H*W, D/2) + return cos, sin + else: + emb = torch.cat([emb_h, emb_w], dim=1) # (H*W, D/2) + return emb + + +def get_1d_rotary_pos_embed(dim: int, pos: Union[np.ndarray, int], theta: float = 10000.0, use_real=False): + """ + Precompute the frequency tensor for complex exponentials (cis) with given dimensions. + + This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' + and the end index 'end'. The 'theta' parameter scales the frequencies. + The returned tensor contains complex values in complex64 data type. + + Args: + dim (int): Dimension of the frequency tensor. + pos (np.ndarray, int): Position indices for the frequency tensor. [S] or scalar + theta (float, optional): Scaling factor for frequency computation. Defaults to 10000.0. + use_real (bool, optional): If True, return real part and imaginary part separately. + Otherwise, return complex numbers. + + Returns: + torch.Tensor: Precomputed frequency tensor with complex exponentials. [S, D/2] + + """ + if isinstance(pos, int): + pos = np.arange(pos) + freqs = 1.0 / (theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)) # [D/2] + t = torch.from_numpy(pos).to(freqs.device) # type: ignore # [S] + freqs = torch.outer(t, freqs).float() # type: ignore # [S, D/2] + if use_real: + freqs_cos = freqs.cos().repeat_interleave(2, dim=1) # [S, D] + freqs_sin = freqs.sin().repeat_interleave(2, dim=1) # [S, D] + return freqs_cos, freqs_sin + else: + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 # [S, D/2] + return freqs_cis + + + +def calc_sizes(rope_img, patch_size, th, tw): + if rope_img == 'extend': + # Expansion mode + sub_args = [(th, tw)] + elif rope_img.startswith('base'): + # Based on the specified dimensions, other dimensions are obtained through interpolation. + base_size = int(rope_img[4:]) // 8 // patch_size + start, stop = get_fill_resize_and_crop((th, tw), base_size) + sub_args = [start, stop, (th, tw)] + else: + raise ValueError(f"Unknown rope_img: {rope_img}") + return sub_args + + +def init_image_posemb(rope_img, + resolutions, + patch_size, + hidden_size, + num_heads, + log_fn, + rope_real=True, + ): + freqs_cis_img = {} + for reso in resolutions: + th, tw = reso.height // 8 // patch_size, reso.width // 8 // patch_size + sub_args = calc_sizes(rope_img, patch_size, th, tw) + freqs_cis_img[str(reso)] = get_2d_rotary_pos_embed(hidden_size // num_heads, *sub_args, use_real=rope_real) + log_fn(f" Using image RoPE ({rope_img}) ({'real' if rope_real else 'complex'}): {sub_args} | ({reso}) " + f"{freqs_cis_img[str(reso)][0].shape if rope_real else freqs_cis_img[str(reso)].shape}") + return freqs_cis_img diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/text_encoder.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7a16b21f4ffbd9acc896c93e2833caf5aecfa0a2 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/text_encoder.py @@ -0,0 +1,95 @@ +import torch +import torch.nn as nn +from transformers import AutoTokenizer, T5EncoderModel, T5ForConditionalGeneration + + +class MT5Embedder(nn.Module): + available_models = ["t5-v1_1-xxl"] + + def __init__( + self, + model_dir="t5-v1_1-xxl", + model_kwargs=None, + torch_dtype=None, + use_tokenizer_only=False, + conditional_generation=False, + max_length=128, + ): + super().__init__() + self.device = "cuda" if torch.cuda.is_available() else "cpu" + self.torch_dtype = torch_dtype or torch.bfloat16 + self.max_length = max_length + if model_kwargs is None: + model_kwargs = { + # "low_cpu_mem_usage": True, + "torch_dtype": self.torch_dtype, + } + model_kwargs["device_map"] = {"shared": self.device, "encoder": self.device} + self.tokenizer = AutoTokenizer.from_pretrained(model_dir) + if use_tokenizer_only: + return + if conditional_generation: + self.model = None + self.generation_model = T5ForConditionalGeneration.from_pretrained( + model_dir + ) + return + self.model = T5EncoderModel.from_pretrained(model_dir, **model_kwargs).eval().to(self.torch_dtype) + + def get_tokens_and_mask(self, texts): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + tokens = text_tokens_and_mask["input_ids"][0] + mask = text_tokens_and_mask["attention_mask"][0] + # tokens = torch.tensor(tokens).clone().detach() + # mask = torch.tensor(mask, dtype=torch.bool).clone().detach() + return tokens, mask + + def get_text_embeddings(self, texts, attention_mask=True, layer_index=-1): + text_tokens_and_mask = self.tokenizer( + texts, + max_length=self.max_length, + padding="max_length", + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + with torch.no_grad(): + outputs = self.model( + input_ids=text_tokens_and_mask["input_ids"].to(self.device), + attention_mask=text_tokens_and_mask["attention_mask"].to(self.device) + if attention_mask + else None, + output_hidden_states=True, + ) + text_encoder_embs = outputs["hidden_states"][layer_index].detach() + + return text_encoder_embs, text_tokens_and_mask["attention_mask"].to(self.device) + + @torch.no_grad() + def __call__(self, tokens, attention_mask, layer_index=-1): + with torch.cuda.amp.autocast(): + outputs = self.model( + input_ids=tokens, + attention_mask=attention_mask, + output_hidden_states=True, + ) + + z = outputs.hidden_states[layer_index].detach() + return z + + def general(self, text: str): + # input_ids = input_ids = torch.tensor([list(text.encode("utf-8"))]) + num_special_tokens + input_ids = self.tokenizer(text, max_length=128).input_ids + print(input_ids) + outputs = self.generation_model(input_ids) + return outputs \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/engine.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/engine.py new file mode 100644 index 0000000000000000000000000000000000000000..bba110edf70dd4fd2c6df218a2d3aa1f915d6617 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/engine.py @@ -0,0 +1,152 @@ +# +# Copyright 2022 The HuggingFace Inc. team. +# SPDX-FileCopyrightText: Copyright (c) 1993-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +from collections import OrderedDict +from copy import copy + +import numpy as np +import tensorrt as trt +import torch +from polygraphy import cuda +from polygraphy.backend.common import bytes_from_path +from polygraphy.backend.trt import CreateConfig, Profile +from polygraphy.backend.trt import engine_from_bytes, engine_from_network, network_from_onnx_path, save_engine +from polygraphy.backend.trt import util as trt_util +import ctypes +from glob import glob +from cuda import cudart + +TRT_LOGGER = trt.Logger(trt.Logger.INFO) +trt_util.TRT_LOGGER = TRT_LOGGER + + +class Engine(): + def __init__( + self, + model_name, + engine_dir, + onnx_file=None, + ): + self.engine_path = os.path.join(engine_dir, model_name + '.plan') + self.engine = None + self.context = None + self.buffers = OrderedDict() + self.tensors = OrderedDict() + + self.weightNameList = None + self.refitter = None + self.onnx_initializers = None + self.onnx_file = onnx_file + self.trt_lora_weight = None + self.trt_lora_weight_mem = None + self.torch_weight = None + + def __del__(self): + del self.engine + del self.context + del self.buffers + del self.tensors + + def build(self, onnx_path, fp16, input_profile=None, enable_preview=False, sparse_weights=False): + print(f"Building TensorRT engine for {onnx_path}: {self.engine_path}") + p = Profile() + if input_profile: + for name, dims in input_profile.items(): + assert len(dims) == 3 + p.add(name, min=dims[0], opt=dims[1], max=dims[2]) + + preview_features = [] + if enable_preview: + trt_version = [int(i) for i in trt.__version__.split(".")] + # FASTER_DYNAMIC_SHAPES_0805 should only be used for TRT 8.5.1 or above. + if trt_version[0] > 8 or \ + (trt_version[0] == 8 and (trt_version[1] > 5 or (trt_version[1] == 5 and trt_version[2] >= 1))): + preview_features = [trt.PreviewFeature.FASTER_DYNAMIC_SHAPES_0805] + + engine = engine_from_network(network_from_onnx_path(onnx_path), config=CreateConfig(fp16=fp16, profiles=[p], + preview_features=preview_features, + sparse_weights=sparse_weights)) + save_engine(engine, path=self.engine_path) + + def activate(self, plugin_path=""): + ctypes.cdll.LoadLibrary(plugin_path) + self.engine = engine_from_bytes(bytes_from_path(self.engine_path)) + self.context = self.engine.create_execution_context() + + def get_shared_memory(self): + _, device_memory = cudart.cudaMalloc(self.engine.device_memory_size) + self.device_memory = device_memory + return self.device_memory + + def set_shared_memory(self, device_memory_size): + self.context.device_memory = device_memory_size + + def binding_input(self, name, shape): + idx = self.engine.get_binding_index(name) + result = self.context.set_binding_shape(idx, shape) + return result + + def allocate_buffers(self, shape_dict=None, device='cuda'): + print("Allocate buffers and bindings inputs:") + for idx in range(trt_util.get_bindings_per_profile(self.engine)): + binding = self.engine[idx] + print("binding: ", binding) + if shape_dict and binding in shape_dict: + shape = shape_dict[binding] + else: + shape = self.engine.get_binding_shape(binding) + nv_dtype = self.engine.get_binding_dtype(binding) + dtype_map = {trt.DataType.FLOAT: np.float32, + trt.DataType.HALF: np.float16, + trt.DataType.INT8: np.int8, + trt.DataType.INT64: np.int64, + trt.DataType.BOOL: bool} + if hasattr(trt.DataType, 'INT32'): + dtype_map[trt.DataType.INT32] = np.int32 + dtype = dtype_map[nv_dtype] + if self.engine.binding_is_input(binding): + self.context.set_binding_shape(idx, shape) + # Workaround to convert np dtype to torch + np_type_tensor = np.empty(shape=[], dtype=dtype) + torch_type_tensor = torch.from_numpy(np_type_tensor) + tensor = torch.empty(tuple(shape), dtype=torch_type_tensor.dtype).to(device=device) + + print(f" binding={binding}, shape={shape}, dtype={tensor.dtype}") + self.tensors[binding] = tensor + self.buffers[binding] = cuda.DeviceView(ptr=tensor.data_ptr(), shape=shape, dtype=dtype) + + def infer(self, feed_dict, stream): + start_binding, end_binding = trt_util.get_active_profile_bindings(self.context) + # shallow copy of ordered dict + device_buffers = copy(self.buffers) + for name, buf in feed_dict.items(): + assert isinstance(buf, cuda.DeviceView) + device_buffers[name] = buf + self.binding_input(name, buf.shape) + bindings = [0] * start_binding + [buf.ptr for buf in device_buffers.values()] + noerror = self.context.execute_async_v2(bindings=bindings, stream_handle=stream.ptr) + if not noerror: + raise ValueError(f"ERROR: inference failed.") + + for idx in range(trt_util.get_bindings_per_profile(self.engine)): + binding = self.engine[idx] + if not self.engine.binding_is_input(binding): + shape = self.context.get_binding_shape(idx) + self.tensors[binding].resize_(tuple(shape)) + return self.tensors diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/hcf_model.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/hcf_model.py new file mode 100644 index 0000000000000000000000000000000000000000..b0ded21c8192af6c03568f81eecf078eb56226a3 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/modules/trt/hcf_model.py @@ -0,0 +1,123 @@ +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models import ModelMixin +from polygraphy import cuda + +from .engine import Engine + + +class TRTModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + in_channels=4, + model_name="unet-dyn", + engine_dir="./unet", + device_id=0, + fp16=True, + image_width=1024, + image_height=1024, + text_maxlen=77, + embedding_dim=768, + max_batch_size=1, + plugin_path="./ckpts/trt_model/fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so", + ): + super().__init__() + # create engine + self.in_channels = in_channels # For pipeline compatibility + self.fp16 = fp16 + self.max_batch_size = max_batch_size + self.model_name = model_name + self.engine_dir = engine_dir + self.engine = Engine(self.model_name, self.engine_dir) + self.engine.activate(plugin_path) + # create cuda stream + self.stream = cuda.Stream() + # create inputs buffer + self.latent_width = image_width // 8 + self.latent_height = image_height // 8 + self.text_maxlen = text_maxlen + self.embedding_dim = embedding_dim + shape_dict = { + 'x': (2 * self.max_batch_size, 4, self.latent_height, self.latent_width), + 't': (2 * self.max_batch_size,), + 'encoder_hidden_states': (2 * self.max_batch_size, self.text_maxlen, self.embedding_dim), + 'text_embedding_mask': (2 * self.max_batch_size, self.text_maxlen), + 'encoder_hidden_states_t5': (2 * self.max_batch_size, 256, 2048), + 'text_embedding_mask_t5': (2 * self.max_batch_size, 256), + 'image_meta_size': (2 * self.max_batch_size, 6), + 'style': (2 * self.max_batch_size,), + 'cos_cis_img': (6400, 88), + 'sin_cis_img': (6400, 88), + 'output': (2 * self.max_batch_size, 8, self.latent_height, self.latent_width), + } + device = "cuda:{}".format(device_id) + self.engine_device = torch.device(device) + self.engine.allocate_buffers(shape_dict=shape_dict, device=device) + + print("[INFO] Create hcf nv controlled unet success") + + @property + def device(self): + return self.engine_device + + def __call__(self, x, t_emb, context, image_meta_size, style, freqs_cis_img0, + freqs_cis_img1, text_embedding_mask, encoder_hidden_states_t5, text_embedding_mask_t5): + return self.forward(x=x, t_emb=t_emb, context=context, image_meta_size=image_meta_size, style=style, + freqs_cis_img0=freqs_cis_img0, freqs_cis_img1=freqs_cis_img1, + text_embedding_mask=text_embedding_mask, encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5) + + def get_shared_memory(self): + return self.engine.get_shared_memory() + + def set_shared_memory(self, shared_memory): + self.engine.set_shared_memory(shared_memory) + + def forward(self, x, t_emb, context, image_meta_size, style, freqs_cis_img0, + freqs_cis_img1, text_embedding_mask, encoder_hidden_states_t5, text_embedding_mask_t5): + x_c = x.half() + t_emb_c = t_emb.half() + context_c = context.half() + image_meta_size_c = image_meta_size.half() + style_c = style.long() + freqs_cis_img0_c = freqs_cis_img0.float() + freqs_cis_img1_c = freqs_cis_img1.float() + text_embedding_mask_c = text_embedding_mask.long() + encoder_hidden_states_t5_c = encoder_hidden_states_t5.half() + text_embedding_mask_t5_c = text_embedding_mask_t5.long() + dtype = np.float16 + batch_size = x.shape[0] // 2 + if batch_size <= self.max_batch_size: + sample_inp = cuda.DeviceView(ptr=x_c.reshape(-1).data_ptr(), shape=x_c.shape, dtype=np.float16) + t_emb_inp = cuda.DeviceView(ptr=t_emb_c.reshape(-1).data_ptr(), shape=t_emb_c.shape, dtype=np.float16) + embeddings_inp = cuda.DeviceView(ptr=context_c.reshape(-1).data_ptr(), shape=context_c.shape, + dtype=np.float16) + image_meta_size_inp = cuda.DeviceView(ptr=image_meta_size_c.reshape(-1).data_ptr(), + shape=image_meta_size_c.shape, dtype=np.float16) + style_inp = cuda.DeviceView(ptr=style_c.reshape(-1).data_ptr(), shape=style_c.shape, dtype=np.int64) + freqs_cis_img0_inp = cuda.DeviceView(ptr=freqs_cis_img0_c.reshape(-1).data_ptr(), + shape=freqs_cis_img0_c.shape, dtype=np.float32) + freqs_cis_img1_inp = cuda.DeviceView(ptr=freqs_cis_img1_c.reshape(-1).data_ptr(), + shape=freqs_cis_img1_c.shape, dtype=np.float32) + text_embedding_mask_inp = cuda.DeviceView(ptr=text_embedding_mask_c.reshape(-1).data_ptr(), + shape=text_embedding_mask_c.shape, dtype=np.int64) + encoder_hidden_states_t5_inp = cuda.DeviceView(ptr=encoder_hidden_states_t5_c.reshape(-1).data_ptr(), + shape=encoder_hidden_states_t5_c.shape, dtype=np.float16) + text_embedding_mask_t5_inp = cuda.DeviceView(ptr=text_embedding_mask_t5_c.reshape(-1).data_ptr(), + shape=text_embedding_mask_t5_c.shape, dtype=np.int64) + feed_dict = {"x": sample_inp, + "t": t_emb_inp, + "encoder_hidden_states": embeddings_inp, + "image_meta_size": image_meta_size_inp, + "text_embedding_mask": text_embedding_mask_inp, + "encoder_hidden_states_t5": encoder_hidden_states_t5_inp, + "text_embedding_mask_t5": text_embedding_mask_t5_inp, + "style": style_inp, "cos_cis_img": freqs_cis_img0_inp, + "sin_cis_img": freqs_cis_img1_inp} + latent = self.engine.infer(feed_dict, self.stream) + return latent['output'] + else: + raise ValueError( + "[ERROR] Input batch_size={} execeed max_batch_size={}".format(batch_size, self.max_batch_size)) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g.sh b/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g.sh new file mode 100644 index 0000000000000000000000000000000000000000..63a9e30f9fe3f30f334d2f976f528be3d94628af --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g.sh @@ -0,0 +1,31 @@ +model='DiT-g/2' +params=" \ + --qk-norm \ + --model ${model} \ + --rope-img base512 \ + --rope-real \ + " +deepspeed --num_gpus 8 --num_nodes 1 --master_port=29000 hydit/train_deepspeed.py ${params} "$@" + +#HOSTFILE="/home/l50041210/HunyuanDiT_combine/hostfile" +#MASTER_ADDR=$(head -n1 $HOSTFILE | awk '{print $1;}') +#MASTER_PORT=6001 +#NODE_ADDR=`hostname -I | awk '{for(i=1;i<=NF;i++)print $i}' | grep ${MASTER_ADDR%.*}. | awk -F " "'{print$1}'` +#NODE_RANK=$(awk '{ranks[$1]=(FNR-1);}END{print ranks["'$NODE_ADDR'"];}' $HOSTFILE) +#NNODES=$(cat $HOSTFILE | wc -l) +#NPUS_PER_NODE=8 +#WORLD_SIZE=$(($NPUS_PER_NODE*$NNODES)) +#echo $MASTER_ADDR +#echo $NODE_ADDR +#echo $NODE_RANK +#echo $NNODES +# +#DISTRIBUTED_ARGS=" +# --nproc_per_node $NPUS_PER_NODE \ +# --nnodes $NNODES \ +# --node_rank $NODE_RANK \ +# --master_addr $MASTER_ADDR \ +# --master_port $MASTER_PORT +##" +# +#torchrun $DISTRIBUTED_ARGS hydit/train_deepspeed.py ${params} "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g_controlnet.sh b/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g_controlnet.sh new file mode 100644 index 0000000000000000000000000000000000000000..ab4d854d6d65b6a430c69ecb87054a72878662ec --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/run_g_controlnet.sh @@ -0,0 +1,8 @@ +model='DiT-g/2' +params=" \ + --qk-norm \ + --model ${model} \ + --rope-img base512 \ + --rope-real \ + " +deepspeed hydit/train_deepspeed_controlnet.py ${params} "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/train.sh b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..10c856dfe1064716d2227a4ca53385da709c5160 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train.sh @@ -0,0 +1,46 @@ +task_flag="dit_g2_full_1024p" # the task flag is used to identify folders. +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=dataset/porcelain/jsons/porcelain_mt.json # index file for dataloader +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=1 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + + +sh $(dirname "$0")/run_g.sh \ + --task-flag ${task_flag} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 1 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + --multireso \ + --reso-step 64 \ + --max-training-steps 5000 \ + --norm 'layer' \ + --autocast-dtype 'bf16' + "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_controlnet.sh b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_controlnet.sh new file mode 100644 index 0000000000000000000000000000000000000000..09c21603e5f31283d5a154b849a6c149d9256e2b --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_controlnet.sh @@ -0,0 +1,46 @@ +task_flag="canny_controlnet" # the task flag is used to identify folders. +control_type=canny +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=/path/to/your/indexfile # index file for dataloader +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=2 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + + +sh $(dirname "$0")/run_g_controlnet.sh \ + --task-flag ${task_flag} \ + --control-type ${control_type} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --multireso \ + --reso-step 64 \ + --ema-to-module \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + "$@" \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed.py new file mode 100644 index 0000000000000000000000000000000000000000..05ce0655f821dbb21d80d3d04297f43b61020f82 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed.py @@ -0,0 +1,545 @@ +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import deepspeed +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.distributed.optim import ZeroRedundancyOptimizer +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import create_logger, set_seeds, create_exp_folder, model_resume, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model + +from utils.npu_utils import is_npu_available, seed_all, AUTOCAST_MAPPING + +if is_npu_available(): + import torch_npu + from torch_npu.npu.amp import autocast + from torch_npu.contrib import transfer_to_npu +else: + from torch.cuda.amp import autocast + + +def deepspeed_initialize(args, logger, model, opt, deepspeed_config): + logger.info(f"Initialize deepspeed...") + logger.info(f" Using deepspeed optimizer") + + def get_learning_rate_scheduler(warmup_min_lr, lr, warmup_num_steps, opt): + return WarmupLR(opt, warmup_min_lr, lr, warmup_num_steps) + + logger.info(f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + model, opt, _, scheduler = deepspeed.initialize( + model=model, + model_parameters=get_trainable_params(model), + config_params=deepspeed_config, + args=args, + lr_scheduler=partial(get_learning_rate_scheduler, args.warmup_min_lr, + args.lr, args.warmup_num_steps) if args.warmup_num_steps > 0 else None, + ) + return model, opt, scheduler + + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + model.module.module.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{save_name}") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + dist.barrier() + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/zero_dp_rank_*_tp_rank_00_pp_rank_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach() + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_(vae_scaling_factor) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + ) + + return latents, model_kwargs + + +def main(args): + if args.training_parts == "lora": + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + deepspeed.init_distributed('nccl') #adapt to deepspeed 0.14.4 + world_size = dist.get_world_size() + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = world_size * batch_size * grad_accu_steps + + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.global_seed * world_size + rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={world_size}.") + deepspeed_config = deepspeed_config_from_args(args, global_batch_size) + + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + args_dict['world_size'] = world_size + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[ + 1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + assert args.deepspeed, f"Must enable deepspeed in this script: train_deepspeed.py" + with deepspeed.zero.Init(data_parallel_group=torch.distributed.group.WORLD, + remote_device=None if args.remote_device == 'none' else args.remote_device, + config_dict_or_path=deepspeed_config, + mpu=None, + enabled=args.zero_stage == 3): + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + logger.info(f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained(TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder(mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info(f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + logger.info(" Using deepspeed optimizer") + opt = None + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True, persistent_workers=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + if args.resume is not None or len(args.resume) > 0: + model, ema, start_epoch, start_epoch_step, train_steps = model_resume(args, model, ema, logger, len(loader)) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + model, opt, scheduler = deepspeed_initialize(args, logger, model, opt, deepspeed_config) + + # =========================================================================== + # Training + # =========================================================================== + with autocast(dtype=AUTOCAST_MAPPING[args.autocast_dtype]): + model.train() + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + dist.barrier() + + iters_per_epoch = len(loader) + logger.info(" ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info(f" Number parameters: {sum(p.numel() for p in model.parameters()):,}") + logger.info(f" Number trainable params: {sum(p.numel() for p in get_trainable_params(model)):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info( + f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info(f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info(f" Training epochs: {start_epoch}/{args.epochs}") + logger.info(f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info( + f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info(f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info(f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info(f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info(" *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info(f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + for batch in loader: + step += 1 + if args.cal_e2e: + if train_steps == 100: + e2e_start = time.time() + if train_steps == 200: + logger.info(f"(100-200步端到端耗时={time.time() - e2e_start:.4f})") + exit() + + latents, model_kwargs = prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, + freqs_cis_img) + + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses(model=model, x_start=latents, model_kwargs=model_kwargs, + step=train_steps) + loss = loss_dict["loss"].mean() + model.backward(loss) + last_batch_iteration = (train_steps + 1) // (global_batch_size // (batch_size * world_size)) + model.step(lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader) - 1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + # =========================================================================== + # Log loss values: + # =========================================================================== + running_loss += loss.item() + log_steps += 1 + train_steps += 1 + if train_steps % args.log_every == 0: + # Measure training speed: + torch.cuda.synchronize() + end_time = time.time() + steps_per_sec = log_steps / (end_time - start_time) + # Reduce loss history over all processes: + avg_loss = torch.tensor(running_loss / log_steps, device=device) + dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + avg_loss = avg_loss.item() / world_size + # get lr from deepspeed fused optimizer + logger.info(f"(step={train_steps:07d}) " + + ( + f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + f"Train Loss: {avg_loss:.4f}, " + f"Lr: {opt.param_groups[0]['lr']:.6g}, " + f"Steps/Sec: {steps_per_sec:.2f}, " + f"Millisec/Step: {(end_time - start_time) * 1000:.2f}," + f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # Reset monitoring variables: + running_loss = 0 + log_steps = 0 + start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0) and train_steps > 0: + save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + dist.destroy_process_group() + + +if __name__ == "__main__": + args = get_args() + if args.seed_all: + seed_all(is_gpu=not is_npu_available(), mode=True) + # Start + main(args) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..4df1b58acd793b21100ee98fc8557ef983b6b51c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/train_deepspeed_controlnet.py @@ -0,0 +1,584 @@ +import gc +import json +import os +import random +import sys +import time +from functools import partial +from glob import glob +from pathlib import Path +import numpy as np + +import deepspeed +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.distributed.optim import ZeroRedundancyOptimizer +from torchvision.transforms import functional as TF +from diffusers.models import AutoencoderKL +from transformers import BertModel, BertTokenizer, logging as tf_logging + +from hydit.config import get_args +from hydit.constants import VAE_EMA_PATH, TEXT_ENCODER, TOKENIZER, T5_ENCODER +from hydit.lr_scheduler import WarmupLR +from hydit.data_loader.arrow_load_stream import TextImageArrowStream +from hydit.diffusion import create_diffusion +from hydit.ds_config import deepspeed_config_from_args +from hydit.modules.ema import EMA +from hydit.modules.fp16_layers import Float16Module +from hydit.modules.models import HUNYUAN_DIT_MODELS +from hydit.modules.controlnet import HunYuanControlNet +from hydit.modules.posemb_layers import init_image_posemb +from hydit.utils.tools import create_logger, set_seeds, create_exp_folder, model_resume, get_trainable_params +from IndexKits.index_kits import ResolutionGroup +from IndexKits.index_kits.sampler import DistributedSamplerWithStartIndex, BlockDistributedSampler +from peft import LoraConfig, get_peft_model + +from hydit.annotator.dwpose import DWposeDetector +torch.optim.lr_scheduler.LRScheduler = torch.optim.lr_scheduler._LRScheduler +from transformers import pipeline +import cv2 +from PIL import Image + +depth_estimator = pipeline('depth-estimation', device='cuda:{}'.format(int(os.getenv('LOCAL_RANK', '0')))) +pose_detector = DWposeDetector() + +def deepspeed_initialize(args, logger, model, opt, deepspeed_config): + logger.info(f"Initialize deepspeed...") + logger.info(f" Using deepspeed optimizer") + + def get_learning_rate_scheduler(warmup_min_lr, lr, warmup_num_steps, opt): + return WarmupLR(opt, warmup_min_lr, lr, warmup_num_steps) + + logger.info(f" Building scheduler with warmup_min_lr={args.warmup_min_lr}, warmup_num_steps={args.warmup_num_steps}") + model, opt, _, scheduler = deepspeed.initialize( + model=model, + model_parameters=get_trainable_params(model), + config_params=deepspeed_config, + args=args, + lr_scheduler=partial(get_learning_rate_scheduler, args.warmup_min_lr, args.lr, args.warmup_num_steps) if args.warmup_num_steps > 0 else None, + ) + return model, opt, scheduler + +def save_checkpoint(args, rank, logger, model, ema, epoch, train_steps, checkpoint_dir): + def save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt"): + cur_ckpt_save_dir = f"{checkpoint_dir}/{tag}" + if rank == 0: + if args.use_fp16: + model.module.module.save_pretrained(cur_ckpt_save_dir) + else: + model.module.save_pretrained(cur_ckpt_save_dir) + + checkpoint_path = "[Not rank 0. Disabled output.]" + + client_state = { + "steps": train_steps, + "epoch": epoch, + "args": args + } + if ema is not None: + client_state['ema'] = ema.state_dict() + + dst_paths = [] + if train_steps % args.ckpt_every == 0: + checkpoint_path = f"{checkpoint_dir}/{train_steps:07d}.pt" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{train_steps:07d}.pt") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{train_steps:07d}.pt") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + if train_steps % args.ckpt_latest_every == 0 or train_steps == args.max_training_steps: + save_name = "latest.pt" + checkpoint_path = f"{checkpoint_dir}/{save_name}" + try: + if args.training_parts == "lora": + save_lora_weight(checkpoint_dir, client_state, tag=f"{save_name}") + else: + model.save_checkpoint(checkpoint_dir, client_state=client_state, tag=f"{save_name}") + dst_paths.append(checkpoint_path) + logger.info(f"Saved checkpoint to {checkpoint_path}") + except: + logger.error(f"Saved failed to {checkpoint_path}") + + dist.barrier() + if rank == 0 and len(dst_paths) > 0: + # Delete optimizer states to avoid occupying too much disk space. + for dst_path in dst_paths: + for opt_state_path in glob(f"{dst_path}/*_00_optim_states.pt"): + os.remove(opt_state_path) + + return checkpoint_path + +def get_canny(np_img, low_threshold = 100, high_threshold = 200): + # tensor = deNormalize() + # image = tensor_to_img(tensor) + image = cv2.Canny(np_img, low_threshold,high_threshold) + image = image[:,:,None] + image = np.concatenate([image,image,image], axis=2) + ## 输出边缘图像 + ## 归一化为张量 + # canny_tensor = img_to_norm_tensor(canny_img) + return image + +def get_depth(np_img): + pil_img = Image.fromarray(np_img) + depth = depth_estimator(pil_img)['depth'] + depth = np.array(depth) + depth = depth[:, :, None] + depth = np.concatenate([depth, depth, depth], axis=2) + return depth + +def get_pose(np_img): + return pose_detector(np_img)[0] + +@torch.no_grad() +def prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img): + image, text_embedding, text_embedding_mask, text_embedding_t5, text_embedding_mask_t5, kwargs = batch + + # clip & mT5 text embedding + text_embedding = text_embedding.to(device) + text_embedding_mask = text_embedding_mask.to(device) + encoder_hidden_states = text_encoder( + text_embedding.to(device), + attention_mask=text_embedding_mask.to(device), + )[0] + text_embedding_t5 = text_embedding_t5.to(device).squeeze(1) + text_embedding_mask_t5 = text_embedding_mask_t5.to(device).squeeze(1) + with torch.no_grad(): + output_t5 = text_encoder_t5( + input_ids=text_embedding_t5, + attention_mask=text_embedding_mask_t5 if T5_ENCODER['attention_mask'] else None, + output_hidden_states=True + ) + encoder_hidden_states_t5 = output_t5['hidden_states'][T5_ENCODER['layer_index']].detach() + + # additional condition + image_meta_size = kwargs['image_meta_size'].to(device) + style = kwargs['style'].to(device) + + np_img = image.squeeze(0).add(1).mul(255 / 2).permute(1, 2, 0).cpu().numpy().astype('uint8') + if args.control_type == 'canny': + condition = get_canny(np_img) + elif args.control_type == 'depth': + condition = get_depth(np_img) + elif args.control_type == 'pose': + condition = get_pose(np_img) + else: + raise NotImplementedError + condtion = Image.fromarray(condition) + condition = TF.to_tensor(condition) + condition = TF.normalize(condition, [0.5], [0.5]) + condition = condition.unsqueeze(0).to(device) + + if args.extra_fp16: + image = image.half() + image_meta_size = image_meta_size.half() if image_meta_size is not None else None + + # Map input images to latent space + normalize latents: + image = image.to(device) + vae_scaling_factor = vae.config.scaling_factor + latents = vae.encode(image).latent_dist.sample().mul_(vae_scaling_factor) + condition = vae.encode(condition).latent_dist.sample().mul_(vae_scaling_factor) + + # positional embedding + _, _, height, width = image.shape + reso = f"{height}x{width}" + cos_cis_img, sin_cis_img = freqs_cis_img[reso] + + # Model conditions + model_kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + text_embedding_mask=text_embedding_mask, + encoder_hidden_states_t5=encoder_hidden_states_t5, + text_embedding_mask_t5=text_embedding_mask_t5, + image_meta_size=image_meta_size, + style=style, + cos_cis_img=cos_cis_img, + sin_cis_img=sin_cis_img, + condition=condition, + ) + + return latents, model_kwargs + +def main(args): + if args.training_parts == "lora": + args.use_ema = False + args.use_ema = False + + assert torch.cuda.is_available(), "Training currently requires at least one GPU." + + dist.init_process_group("nccl") + world_size = dist.get_world_size() + batch_size = args.batch_size + grad_accu_steps = args.grad_accu_steps + global_batch_size = world_size * batch_size * grad_accu_steps + + rank = dist.get_rank() + device = rank % torch.cuda.device_count() + seed = args.global_seed * world_size + rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + torch.cuda.set_device(device) + print(f"Starting rank={rank}, seed={seed}, world_size={world_size}.") + deepspeed_config = deepspeed_config_from_args(args, global_batch_size) + + # Setup an experiment folder + experiment_dir, checkpoint_dir, logger = create_exp_folder(args, rank) + + # Log all the arguments + logger.info(sys.argv) + logger.info(str(args)) + # Save to a json file + args_dict = vars(args) + args_dict['world_size'] = world_size + with open(f"{experiment_dir}/args.json", 'w') as f: + json.dump(args_dict, f, indent=4) + + # Disable the message "Some weights of the model checkpoint at ... were not used when initializing BertModel." + # If needed, just comment the following line. + tf_logging.set_verbosity_error() + + # =========================================================================== + # Building HYDIT + # =========================================================================== + + logger.info("Building HYDIT Model.") + + # --------------------------------------------------------------------------- + # Training sample base size, such as 256/512/1024. Notice that this size is + # just a base size, not necessary the actual size of training samples. Actual + # size of the training samples are correlated with `resolutions` when enabling + # multi-resolution training. + # --------------------------------------------------------------------------- + image_size = args.image_size + if len(image_size) == 1: + image_size = [image_size[0], image_size[0]] + if len(image_size) != 2: + raise ValueError(f"Invalid image size: {args.image_size}") + assert image_size[0] % 8 == 0 and image_size[1] % 8 == 0, "Image size must be divisible by 8 (for the VAE encoder). " \ + f"got {image_size}" + latent_size = [image_size[0] // 8, image_size[1] // 8] + + # initialize model by deepspeed + assert args.deepspeed, f"Must enable deepspeed in this script: train_deepspeed.py" + with deepspeed.zero.Init(data_parallel_group=torch.distributed.group.WORLD, + remote_device=None if args.remote_device == 'none' else args.remote_device, + config_dict_or_path=deepspeed_config, + mpu=None, + enabled=args.zero_stage == 3): + model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=logger.info, + ) + controlnet = HunYuanControlNet(args, + input_size=latent_size, + depth=40, hidden_size=1408, patch_size=2, num_heads=16, mlp_ratio=4.3637, + log_fn=logger.info, + ) + # Multi-resolution / Single-resolution training. + if args.multireso: + resolutions = ResolutionGroup(image_size[0], + align=16, + step=args.reso_step, + target_ratios=args.target_ratios).data + else: + resolutions = ResolutionGroup(image_size[0], + align=16, + target_ratios=['1:1']).data + + freqs_cis_img = init_image_posemb(args.rope_img, + resolutions=resolutions, + patch_size=model.patch_size, + hidden_size=model.hidden_size, + num_heads=model.num_heads, + log_fn=logger.info, + rope_real=args.rope_real, + ) + + # Create EMA model and convert to fp16 if needed. + ema = None + if args.use_ema: + ema = EMA(args, model, device, logger) + + # Setup FP16 main model: + if args.use_fp16: + model = Float16Module(model, args) + controlnet = Float16Module(controlnet, args) + logger.info(f" Using main model with data type {'fp16' if args.use_fp16 else 'fp32'}") + + diffusion = create_diffusion( + noise_schedule=args.noise_schedule, + predict_type=args.predict_type, + learn_sigma=args.learn_sigma, + mse_loss_weight_type=args.mse_loss_weight_type, + beta_start=args.beta_start, + beta_end=args.beta_end, + noise_offset=args.noise_offset, + ) + + # Setup VAE + logger.info(f" Loading vae from {VAE_EMA_PATH}") + vae = AutoencoderKL.from_pretrained(VAE_EMA_PATH) + # Setup BERT text encoder + logger.info(f" Loading Bert text encoder from {TEXT_ENCODER}") + text_encoder = BertModel.from_pretrained(TEXT_ENCODER, False, revision=None) + # Setup BERT tokenizer: + logger.info(f" Loading Bert tokenizer from {TOKENIZER}") + tokenizer = BertTokenizer.from_pretrained(TOKENIZER) + # Setup T5 text encoder + from hydit.modules.text_encoder import MT5Embedder + mt5_path = T5_ENCODER['MT5'] + embedder_t5 = MT5Embedder(mt5_path, torch_dtype=T5_ENCODER['torch_dtype'], max_length=args.text_len_t5) + tokenizer_t5 = embedder_t5.tokenizer + text_encoder_t5 = embedder_t5.model + + if args.extra_fp16: + logger.info(f" Using fp16 for extra modules: vae, text_encoder") + vae = vae.half().to(device) + text_encoder = text_encoder.half().to(device) + text_encoder_t5 = text_encoder_t5.half().to(device) + else: + vae = vae.to(device) + text_encoder = text_encoder.to(device) + text_encoder_t5 = text_encoder_t5.to(device) + + logger.info(f" Optimizer parameters: lr={args.lr}, weight_decay={args.weight_decay}") + logger.info(" Using deepspeed optimizer") + opt = None + + # =========================================================================== + # Building Dataset + # =========================================================================== + + logger.info(f"Building Streaming Dataset.") + logger.info(f" Loading index file {args.index_file} (v2)") + + dataset = TextImageArrowStream(args=args, + resolution=image_size[0], + random_flip=args.random_flip, + log_fn=logger.info, + index_file=args.index_file, + multireso=args.multireso, + batch_size=batch_size, + world_size=world_size, + random_shrink_size_cond=args.random_shrink_size_cond, + merge_src_cond=args.merge_src_cond, + uncond_p=args.uncond_p, + text_ctx_len=args.text_len, + tokenizer=tokenizer, + uncond_p_t5=args.uncond_p_t5, + text_ctx_len_t5=args.text_len_t5, + tokenizer_t5=tokenizer_t5, + ) + if args.multireso: + sampler = BlockDistributedSampler(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True, batch_size=batch_size) + else: + sampler = DistributedSamplerWithStartIndex(dataset, num_replicas=world_size, rank=rank, seed=args.global_seed, + shuffle=False, drop_last=True) + loader = DataLoader(dataset, batch_size=batch_size, shuffle=False, sampler=sampler, + num_workers=args.num_workers, pin_memory=True, drop_last=True) + logger.info(f" Dataset contains {len(dataset):,} images.") + logger.info(f" Index file: {args.index_file}.") + if args.multireso: + logger.info(f' Using MultiResolutionBucketIndexV2 with step {dataset.index_manager.step} ' + f'and base size {dataset.index_manager.base_size}') + logger.info(f'\n {dataset.index_manager.resolutions}') + + # =========================================================================== + # Loading parameter + # =========================================================================== + + logger.info(f"Loading parameter") + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + # Resume checkpoint if needed + if args.resume is not None or len(args.resume) > 0: + model, ema, start_epoch, start_epoch_step, train_steps = model_resume(args, model, ema, logger, len(loader)) + + if args.training_parts == "lora": + loraconfig = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=args.target_modules + ) + if args.use_fp16: + model.module = get_peft_model(model.module, loraconfig) + else: + model = get_peft_model(model, loraconfig) + + logger.info(f" Training parts: {args.training_parts}") + + if args.use_fp16: + controlnet.module.from_dit(model.module) + controlnet.module.set_trainable() + else: + controlnet.from_dit(model) + controlnet.set_trainable() + logger.info(f" ControlNet loaded from DIT") + + + + controlnet, opt, scheduler = deepspeed_initialize(args, logger, controlnet, opt, deepspeed_config) + + # =========================================================================== + # Training + # =========================================================================== + + model.eval() + model.requires_grad_(False) + model = model.to(device) + + if args.use_ema: + ema.eval() + + print(f" Worker {rank} ready.") + dist.barrier() + + iters_per_epoch = len(loader) + logger.info(" ****************************** Running training ******************************") + logger.info(f" Number GPUs: {world_size}") + logger.info(f" Number training samples: {len(dataset):,}") + logger.info(f" Number parameters: {sum(p.numel() for p in controlnet.parameters()):,}") + logger.info(f" Number trainable params: {sum(p.numel() for p in get_trainable_params(controlnet)):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Iters per epoch: {iters_per_epoch:,}") + logger.info(f" Batch size per device: {batch_size}") + logger.info(f" Batch size all device: {batch_size * world_size * grad_accu_steps:,} (world_size * batch_size * grad_accu_steps)") + logger.info(f" Gradient Accu steps: {args.grad_accu_steps}") + logger.info(f" Total optimization steps: {args.epochs * iters_per_epoch // grad_accu_steps:,}") + + logger.info(f" Training epochs: {start_epoch}/{args.epochs}") + logger.info(f" Training epoch steps: {start_epoch_step:,}/{iters_per_epoch:,}") + logger.info(f" Training total steps: {train_steps:,}/{min(args.max_training_steps, args.epochs * iters_per_epoch):,}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Noise schedule: {args.noise_schedule}") + logger.info(f" Beta limits: ({args.beta_start}, {args.beta_end})") + logger.info(f" Learn sigma: {args.learn_sigma}") + logger.info(f" Prediction type: {args.predict_type}") + logger.info(f" Noise offset: {args.noise_offset}") + + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Using EMA model: {args.use_ema} ({args.ema_dtype})") + if args.use_ema: + logger.info(f" Using EMA decay: {ema.max_value if args.use_ema else None}") + logger.info(f" Using EMA warmup power: {ema.power if args.use_ema else None}") + logger.info(f" Using main model fp16: {args.use_fp16}") + logger.info(f" Using extra modules fp16: {args.extra_fp16}") + logger.info(" ------------------------------------------------------------------------------") + logger.info(f" Experiment directory: {experiment_dir}") + logger.info(" *******************************************************************************") + + if args.gc_interval > 0: + gc.disable() + gc.collect() + + # Variables for monitoring/logging purposes: + log_steps = 0 + running_loss = 0 + start_time = time.time() + + if args.async_ema: + ema_stream = torch.cuda.Stream() + + # Training loop + for epoch in range(start_epoch, args.epochs): + logger.info(f" Start random shuffle with seed={seed}") + # Makesure all processors use the same seed to shuffle dataset. + dataset.shuffle(seed=args.global_seed + epoch, fast=True) + logger.info(f" End of random shuffle") + + # Move sampler to start_index + if not args.multireso: + start_index = start_epoch_step * world_size * batch_size + if start_index != sampler.start_index: + sampler.start_index = start_index + # Reset start_epoch_step to zero, to ensure next epoch will start from the beginning. + start_epoch_step = 0 + logger.info(f" Iters left this epoch: {len(loader):,}") + + logger.info(f" Beginning epoch {epoch}...") + step = 0 + for batch in loader: + step += 1 + + latents, model_kwargs = prepare_model_inputs(args, batch, device, vae, text_encoder, text_encoder_t5, freqs_cis_img) + + # training model by deepspeed while use fp16 + if args.use_fp16: + if args.use_ema and args.async_ema: + with torch.cuda.stream(ema_stream): + ema.update(model.module.module, step=step) + torch.cuda.current_stream().wait_stream(ema_stream) + + loss_dict = diffusion.training_losses(model=model, x_start=latents, model_kwargs=model_kwargs, controlnet=controlnet) + loss = loss_dict["loss"].mean() + controlnet.backward(loss) + last_batch_iteration = (train_steps + 1) // (global_batch_size // (batch_size * world_size)) + controlnet.step(lr_kwargs={'last_batch_iteration': last_batch_iteration}) + + if args.use_ema and not args.async_ema or (args.async_ema and step == len(loader)-1): + if args.use_fp16: + ema.update(model.module.module, step=step) + else: + ema.update(model.module, step=step) + + # =========================================================================== + # Log loss values: + # =========================================================================== + running_loss += loss.item() + log_steps += 1 + train_steps += 1 + if train_steps % args.log_every == 0: + # Measure training speed: + torch.cuda.synchronize() + end_time = time.time() + steps_per_sec = log_steps / (end_time - start_time) + # Reduce loss history over all processes: + avg_loss = torch.tensor(running_loss / log_steps, device=device) + dist.all_reduce(avg_loss, op=dist.ReduceOp.SUM) + avg_loss = avg_loss.item() / world_size + # get lr from deepspeed fused optimizer + logger.info(f"(step={train_steps:07d}) " + + (f"(update_step={train_steps // args.grad_accu_steps:07d}) " if args.grad_accu_steps > 1 else "") + + f"Train Loss: {avg_loss:.4f}, " + f"Lr: {opt.param_groups[0]['lr']:.6g}, " + f"Steps/Sec: {steps_per_sec:.2f}, " + f"Samples/Sec: {int(steps_per_sec * batch_size * world_size):d}") + # Reset monitoring variables: + running_loss = 0 + log_steps = 0 + start_time = time.time() + + # collect gc: + if args.gc_interval > 0 and (step % args.gc_interval == 0): + gc.collect() + + if (train_steps % args.ckpt_every == 0 or train_steps % args.ckpt_latest_every == 0 # or train_steps == args.max_training_steps + ) and train_steps > 0: + save_checkpoint(args, rank, logger, controlnet, ema, epoch, train_steps, checkpoint_dir) + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking step loop at {train_steps}.") + break + + if train_steps >= args.max_training_steps: + logger.info(f"Breaking epoch loop at {epoch}.") + break + + dist.destroy_process_group() + + +if __name__ == "__main__": + # Start + main(get_args()) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/hydit/utils/tools.py b/PyTorch/built-in/mlm/HunyuanDiT/hydit/utils/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..71d829227fbd6c9ee3d0f11ea873a5ac7abd70e1 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/hydit/utils/tools.py @@ -0,0 +1,190 @@ +import random +import logging +from pathlib import Path +import shutil + +import numpy as np +from PIL import Image +import torch +import torch.distributed as dist +from tqdm.auto import tqdm +import math +import torch.nn.functional as F +import os + +def get_trainable_params(model): + params = model.parameters() + params = [p for p in params if p.requires_grad] + return params + + +def set_seeds(seed_list, device=None): + if isinstance(seed_list, (tuple, list)): + seed = sum(seed_list) + else: + seed = seed_list + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + return torch.Generator(device).manual_seed(seed) + +def get_start_epoch(resume_path, ckpt, steps_per_epoch): + if 'epoch' in ckpt: + start_epoch = ckpt['epoch'] + else: + start_epoch = 0 + if 'steps' in ckpt: + train_steps = ckpt['steps'] + else: + try: + train_steps = int(Path(resume_path).stem) + except: + train_steps = start_epoch * steps_per_epoch + + start_epoch_step = train_steps % steps_per_epoch + 1 + return start_epoch, start_epoch_step, train_steps + +def assert_shape(*args): + if len(args) < 2: + return + cond = True + fail_str = f"{args[0] if isinstance(args[0], (list, tuple)) else args[0].shape}" + for i in range(1, len(args)): + shape1 = args[i] if isinstance(args[i], (list, tuple)) else args[i].shape + shape2 = args[i - 1] if isinstance(args[i - 1], (list, tuple)) else args[i - 1].shape + cond = cond and (shape1 == shape2) + fail_str += f" vs {args[i] if isinstance(args[i], (list, tuple)) else args[i].shape}" + assert cond, fail_str + + +def create_logger(logging_dir=None, logging_file=None, ddp=True): + """ + Create a logger that writes to a log file and stdout. + """ + if not ddp or (ddp and dist.get_rank() == 0): # real logger + if logging_file is not None: + file_handler = [logging.FileHandler(logging_file)] + elif logging_dir is not None: + file_handler = [logging.FileHandler(f"{logging_dir}/log.txt")] + else: + file_handler = [] + logging.basicConfig( + level=logging.INFO, + format='[\033[34m%(asctime)s\033[0m] %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[logging.StreamHandler()] + file_handler + ) + logger = logging.getLogger(__name__) + else: + logger = logging.getLogger(__name__) + logger.addHandler(logging.NullHandler()) + return logger + +def create_exp_folder(args, rank): + if rank == 0: + os.makedirs(args.results_dir, exist_ok=True) + existed_experiments = list(Path(args.results_dir).glob("*dit*")) + if len(existed_experiments) == 0: + experiment_index = 1 + else: + existed_experiments.sort() + print('existed_experiments', existed_experiments) + experiment_index = max([int(x.stem.split('-')[0]) for x in existed_experiments]) + 1 + dist.barrier() + model_string_name = args.task_flag if args.task_flag else args.model.replace("/", "-") + experiment_dir = f"{args.results_dir}/{experiment_index:03d}-{model_string_name}" # Create an experiment folder + checkpoint_dir = f"{experiment_dir}/checkpoints" # Stores saved model checkpoints + if rank == 0: + os.makedirs(checkpoint_dir, exist_ok=True) + logger = create_logger(experiment_dir) + logger.info(f"Experiment directory created at {experiment_dir}") + else: + logger = create_logger() + experiment_dir = "" + + return experiment_dir, checkpoint_dir, logger + + +def model_resume(args, model, ema, logger, len_loader): + """ + Load pretrained weights. + """ + start_epoch = 0 + start_epoch_step = 0 + train_steps = 0 + resume_path = args.resume + if not Path(resume_path).exists(): + raise FileNotFoundError(f" Cannot find checkpoint from {resume_path}") + + logger.info(f" Resume deepspeed={args.resume_deepspeed}, " + f"Resume split={args.resume_split}, " + f"Resume from checkpoint {resume_path}") + # Resume model and ema states (not include optimizer states) from a checkpoint saved by Deepspeed version of DIT. + if args.resume_deepspeed: + assert 'mp_rank_00_model_states.pt' in os.listdir(resume_path), f' Cannot find dp chkpt from {resume_path}' + resume_ckpt = torch.load(os.path.join(resume_path, 'mp_rank_00_model_states.pt'), + map_location=lambda storage, loc: storage) + # Resume main model + if args.ema_to_module: + logger.info(" Resume main model from the ema states.") + model.load_state_dict(resume_ckpt['ema'], strict=args.strict) + else: + logger.info(" Resume main model from the main states.") + model.load_state_dict(resume_ckpt['module'], strict=args.strict) + # Resume EMA model + if args.use_ema: + if args.module_to_ema: + logger.info(" Resume EMA model from the main states.") + ema.load_state_dict(resume_ckpt['module'], strict=args.strict) + else: + logger.info(" Resume EMA model from the EMA states.") + ema.load_state_dict(resume_ckpt['ema'], strict=args.strict) + if not args.reset_loader: + start_epoch, start_epoch_step, train_steps = get_start_epoch(args.resume, resume_ckpt, len_loader) + # Resume model and ema states (not include optimizer states) from two checkpoints separated from DeepSpeed ckpt. + elif args.resume_split: + # Resume main model + if args.ema_to_module: + assert 'pytorch_model_ema.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_ema.pt from {resume_path}' + logger.info(f" Resume main model from ema states.") + resume_ckpt_ema = torch.load(os.path.join(resume_path, 'pytorch_model_ema.pt'), + map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + assert 'pytorch_model_module.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_module.pt from {resume_path}' + logger.info(f" Resume main model from main states.") + resume_ckpt_module = torch.load(os.path.join(resume_path, 'pytorch_model_module.pt'), + map_location=lambda storage, loc: storage) + model.load_state_dict(resume_ckpt_module, strict=args.strict) + # Resume ema model + if args.use_ema: + if args.module_to_ema: + if "resume_ckpt_module" in locals(): + logger.info(f" Resume ema model from main states.") + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + assert 'pytorch_model_module.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_module.pt from {resume_path}' + logger.info(f" Resume ema model from module states.") + resume_ckpt_module = torch.load(os.path.join(resume_path, 'pytorch_model_module.pt'), + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_module, strict=args.strict) + else: + if "resume_ckpt_ema" in locals(): + logger.info(f" Resume ema model from EMA states.") + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + assert 'pytorch_model_ema.pt' in os.listdir( + resume_path), f' Cannot find pytorch_model_ema.pt from {resume_path}' + logger.info(f" Resume ema model from EMA states.") + resume_ckpt_ema = torch.load(os.path.join(resume_path, 'pytorch_model_ema.pt'), + map_location=lambda storage, loc: storage) + ema.load_state_dict(resume_ckpt_ema, strict=args.strict) + else: + raise ValueError(" “If `resume` is True, then either `resume_split` or `resume_deepspeed` must be true.”") + + return model, ema, start_epoch, start_epoch_step, train_steps \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/lite/README.md b/PyTorch/built-in/mlm/HunyuanDiT/lite/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d9f52703dcb587aa219cf837178fa9506693db9a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/lite/README.md @@ -0,0 +1,35 @@ +## Using HunyuanDiT Inference with under 6GB GPU VRAM + +### Instructions +Running HunyuanDiT in under 6GB GPU VRAM is available now based on [**diffusers**](https://huggingface.co/docs/diffusers/main/en/api/pipelines/hunyuandit). Here we provide instructions and demo for your quick start. + +The 6Glite version supports Nvidia Ampere architecture series graphics cards such as RTX 3070/3080/4080/4090, A100, and so on. + +The only thing you need do is to install the following library: + +```bash +pip install -U bitsandbytes +pip install git+https://github.com/huggingface/diffusers +pip install torch==2.0.0 +``` + +Then you can enjoy your HunyuanDiT text-to-image journey under 6GB GPU VRAM directly! + +Here is a demo for you. + +```bash +cd HunyuanDiT + +# Quick start +model_id=Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled +prompt=一个宇航员在骑马 +infer_steps=50 +guidance_scale=6 +python3 lite/inference.py ${model_id} ${prompt} ${infer_steps} ${guidance_scale} +``` + +Note: To use other features in hydit requires torch 1.13.1. In this case, you may need to downgrade your torch version. + +```bash +pip install torch==1.13.1 +``` \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/lite/inference.py b/PyTorch/built-in/mlm/HunyuanDiT/lite/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..40f8bcd0bcc8cb8614124c612637f73e9364c7db --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/lite/inference.py @@ -0,0 +1,175 @@ +import random +import torch +from diffusers import HunyuanDiTPipeline +from transformers import T5EncoderModel +import time +from loguru import logger +import gc +import sys + +NEGATIVE_PROMPT = '' + +TEXT_ENCODER_CONF = { + "negative_prompt": NEGATIVE_PROMPT, + "prompt_embeds": None, + "negative_prompt_embeds": None, + "prompt_attention_mask": None, + "negative_prompt_attention_mask": None, + "max_sequence_length": 256, + "text_encoder_index": 1, +} + +def flush(): + gc.collect() + torch.cuda.empty_cache() + + +class End2End(object): + def __init__(self, model_id="Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled"): + self.model_id = model_id + self.device = "cuda" if torch.cuda.is_available() else "cpu" + # ======================================================================== + self.default_negative_prompt = NEGATIVE_PROMPT + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def load_pipeline(self): + self.pipeline= HunyuanDiTPipeline.from_pretrained( + self.model_id, + text_encoder=None, + text_encoder_2=None, + torch_dtype=torch.float16, + ).to(self.device) + + + def get_text_emb(self, prompts): + with torch.no_grad(): + text_encoder_2 = T5EncoderModel.from_pretrained( + self.model_id, + subfolder="text_encoder_2", + load_in_8bit=True, + device_map="auto", + ) + encoder_pipeline = HunyuanDiTPipeline.from_pretrained( + self.model_id, + text_encoder_2=text_encoder_2, + transformer=None, + vae=None, + torch_dtype=torch.float16, + device_map="balanced", + ) + TEXT_ENCODER_CONF["negative_prompt"]=self.default_negative_prompt + prompt_emb1 = encoder_pipeline.encode_prompt(prompts, negative_prompt=self.default_negative_prompt) + prompt_emb2 = encoder_pipeline.encode_prompt(prompts, **TEXT_ENCODER_CONF) + del text_encoder_2 + del encoder_pipeline + flush() + return prompt_emb1, prompt_emb2 + + def predict(self, + user_prompt, + seed=None, + enhanced_prompt=None, + negative_prompt=None, + infer_steps=50, + guidance_scale=6, + batch_size=1, + ): + # ======================================================================== + # Arguments: seed + # ======================================================================== + if seed is None: + seed = random.randint(0, 1_000_000) + if not isinstance(seed, int): + raise TypeError(f"`seed` must be an integer, but got {type(seed)}") + generator = torch.Generator(device=self.device).manual_seed(seed) + + # ======================================================================== + # Arguments: prompt, new_prompt, negative_prompt + # ======================================================================== + if not isinstance(user_prompt, str): + raise TypeError(f"`user_prompt` must be a string, but got {type(user_prompt)}") + user_prompt = user_prompt.strip() + prompt = user_prompt + + if enhanced_prompt is not None: + if not isinstance(enhanced_prompt, str): + raise TypeError(f"`enhanced_prompt` must be a string, but got {type(enhanced_prompt)}") + enhanced_prompt = enhanced_prompt.strip() + prompt = enhanced_prompt + + # negative prompt + if negative_prompt is not None and negative_prompt != '': + self.default_negative_prompt = negative_prompt + if not isinstance(self.default_negative_prompt, str): + raise TypeError(f"`negative_prompt` must be a string, but got {type(negative_prompt)}") + + + # ======================================================================== + + logger.debug(f""" + prompt: {user_prompt} + enhanced prompt: {enhanced_prompt} + seed: {seed} + negative_prompt: {negative_prompt} + batch_size: {batch_size} + guidance_scale: {guidance_scale} + infer_steps: {infer_steps} + """) + + + # get text embeding + flush() + prompt_emb1, prompt_emb2 = self.get_text_emb(prompt) + prompt_embeds, negative_prompt_embeds, prompt_attention_mask, negative_prompt_attention_mask = prompt_emb1 + prompt_embeds_2,negative_prompt_embeds_2,prompt_attention_mask_2,negative_prompt_attention_mask_2 = prompt_emb2 + del prompt_emb1 + del prompt_emb2 + # get pipeline + self.load_pipeline() + samples = self.pipeline( + prompt_embeds=prompt_embeds, + prompt_embeds_2=prompt_embeds_2, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_2=negative_prompt_embeds_2, + prompt_attention_mask=prompt_attention_mask, + prompt_attention_mask_2=prompt_attention_mask_2, + negative_prompt_attention_mask=negative_prompt_attention_mask, + negative_prompt_attention_mask_2=negative_prompt_attention_mask_2, + num_images_per_prompt=batch_size, + guidance_scale=guidance_scale, + num_inference_steps=infer_steps, + generator=generator, + ).images[0] + + return { + 'images': samples, + 'seed': seed, + } + + +if __name__ == "__main__": + + if len(sys.argv) != 5: + print("Usage: python lite/inference.py ${model_id} ${prompt} ${infer_steps} ${guidance_scale}") + print("model_id: Choose a diffusers repository from the official Hugging Face repository https://huggingface.co/Tencent-Hunyuan, " + "such as Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers, " + "Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers-Distilled, " + "Tencent-Hunyuan/HunyuanDiT-Diffusers, or Tencent-Hunyuan/HunyuanDiT-Diffusers-Distilled.") + print("prompt: the input prompt") + print("infer_steps: infer_steps") + print("guidance_scale: guidance_scale") + sys.exit(1) + model_id = sys.argv[1] + prompt = sys.argv[2] + infer_steps = int(sys.argv[3]) + guidance_scale = int(sys.argv[4]) + gen = End2End(model_id) + seed = 42 + results = gen.predict(prompt, + seed = seed, + infer_steps=infer_steps, + guidance_scale=guidance_scale, + ) + results['images'].save('./lite_image.png') diff --git a/PyTorch/built-in/mlm/HunyuanDiT/lora/README.md b/PyTorch/built-in/mlm/HunyuanDiT/lora/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ef549842ad4802de4b8d1f24e64c5807d39b9842 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/lora/README.md @@ -0,0 +1,217 @@ + +## Using LoRA to fine-tune HunyuanDiT + + +### Instructions + + The dependencies and installation are basically the same as the [**base model**](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1). + + We provide two types of trained LoRA weights for you to test. + + Then download the model using the following commands: + +```bash +cd HunyuanDiT +# Use the huggingface-cli tool to download the model. +huggingface-cli download Tencent-Hunyuan/HYDiT-LoRA --local-dir ./ckpts/t2i/lora + +# Quick start +python sample_t2i.py --prompt "青花瓷风格，一只猫在追蝴蝶" --no-enhance --load-key ema --lora-ckpt ./ckpts/t2i/lora/porcelain +``` + +Examples of training data and inference results are as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Examples of training data
Image 0Image 1Image 2Image 3
青花瓷风格，一只蓝色的鸟儿站在蓝色的花瓶上，周围点缀着白色花朵，背景是白色 （Porcelain style, a blue bird stands on a blue vase, surrounded by white flowers, with a white background. +）青花瓷风格，这是一幅蓝白相间的陶瓷盘子，上面描绘着一只狐狸和它的幼崽在森林中漫步，背景是白色 （Porcelain style, this is a blue and white ceramic plate depicting a fox and its cubs strolling in the forest, with a white background.）青花瓷风格，在黑色背景上，一只蓝色的狼站在蓝白相间的盘子上，周围是树木和月亮 （Porcelain style, on a black background, a blue wolf stands on a blue and white plate, surrounded by trees and the moon.）青花瓷风格，在蓝色背景上，一只蓝色蝴蝶和白色花朵被放置在中央 （Porcelain style, on a blue background, a blue butterfly and white flowers are placed in the center.）
Examples of inference results
Image 4Image 5Image 6Image 7
青花瓷风格，苏州园林 （Porcelain style, Suzhou Gardens.）青花瓷风格，一朵荷花 （Porcelain style, a lotus flower.）青花瓷风格，一只羊（Porcelain style, a sheep.）青花瓷风格，一个女孩在雨中跳舞（Porcelain style, a girl dancing in the rain.）
+ + +### Training + +We provide three types of weights for fine-tuning LoRA, `ema`, `module` and `distill`, and you can choose according to the actual effect. By default, we use `ema` weights. + +Here is an example, we load the `ema` weights into the main model and perform LoRA fine-tuning through the `--ema-to-module` parameter. + +If you want to load the `module` weights into the main model, just remove the `--ema-to-module` parameter. + +If multiple resolution are used, you need to add the `--multireso` and `--reso-step 64 ` parameter. + +```bash +model='DiT-g/2' # model type +task_flag="lora_porcelain_ema_rank64" # task flag +resume=./ckpts/t2i/model/ # resume checkpoint +index_file=dataset/porcelain/jsons/porcelain.json # the selected data indices +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=2 # gradient accumulation steps +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=100 # create a ckpt every a few steps. +ckpt_latest_every=2000 # create a ckpt named `latest.pt` every a few steps. +rank=64 # rank of lora +max_training_steps=2000 # Maximum training iteration steps + +PYTHONPATH=./ deepspeed hydit/train_deepspeed.py \ + --task-flag ${task_flag} \ + --model ${model} \ + --training_parts lora \ + --rank ${rank} \ + --resume-split \ + --resume ${resume} \ + --ema-to-module \ + --lr ${lr} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --ckpt-every ${ckpt_every} \ + --max-training-steps ${max_training_steps}\ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + --qk-norm \ + --rope-img base512 \ + --rope-real \ + "$@" +``` + +Recommended parameter settings + +| Parameter | Description | Recommended Parameter Value | Note| +|:---------------:|:---------:|:---------------------------------------------------:|:--:| +| `--batch-size` | Training batch size | 1 | Depends on GPU memory| +| `--grad-accu-steps` | Size of gradient accumulation | 2 | - | +| `--rank` | Rank of lora | 64 | Choosing from 8-128 | +| `--max-training-steps` | Training steps | 2000 | Depend on training data size, for reference apply 2000 steps on 100 images| +| `--lr` | Learning rate | 0.0001 | - | + + +### Inference + +After the training is complete, you can use the following command line for inference. +We provide the `--lora-ckpt` parameter for selecting the folder which contains lora weights and configurations. + +a. Using LoRA during inference + +```bash +python sample_t2i.py --prompt "青花瓷风格，一只小狗" --no-enhance --lora-ckpt log_EXP/001-lora_porcelain_ema_rank64/checkpoints/0001000.pt +``` + +b. Using LoRA in gradio +```bash +python app/hydit_app.py --infer-mode fa --no-enhance --lora-ckpt log_EXP/001-lora_porcelain_ema_rank64/checkpoints/0001000.pt +``` + +c. Merge LoRA weights into the main model + +We provide the `--output-merge-path` parameter to set the path for saving the merged weights. + +```bash +PYTHONPATH=./ python lora/merge.py --lora-ckpt log_EXP/001-lora_porcelain_ema_rank64/checkpoints/0001000.pt --output-merge-path ./ckpts/t2i/model/pytorch_model_merge.pt +``` + +d. Regarding how to use the LoRA weights we trained in diffusion, we provide the following script. To ensure compatibility with the diffuser, some modifications are made, which means that LoRA cannot be directly loaded. + + +```python +import torch +from diffusers import HunyuanDiTPipeline + +num_layers = 40 +def load_hunyuan_dit_lora(transformer_state_dict, lora_state_dict, lora_scale): + for i in range(num_layers): + Wqkv = torch.matmul(lora_state_dict[f"blocks.{i}.attn1.Wqkv.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn1.Wqkv.lora_A.weight"]) + q, k, v = torch.chunk(Wqkv, 3, dim=0) + transformer_state_dict[f"blocks.{i}.attn1.to_q.weight"] += lora_scale * q + transformer_state_dict[f"blocks.{i}.attn1.to_k.weight"] += lora_scale * k + transformer_state_dict[f"blocks.{i}.attn1.to_v.weight"] += lora_scale * v + + out_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn1.out_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn1.out_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn1.to_out.0.weight"] += lora_scale * out_proj + + q_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.q_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.q_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn2.to_q.weight"] += lora_scale * q_proj + + kv_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.kv_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.kv_proj.lora_A.weight"]) + k, v = torch.chunk(kv_proj, 2, dim=0) + transformer_state_dict[f"blocks.{i}.attn2.to_k.weight"] += lora_scale * k + transformer_state_dict[f"blocks.{i}.attn2.to_v.weight"] += lora_scale * v + + out_proj = torch.matmul(lora_state_dict[f"blocks.{i}.attn2.out_proj.lora_B.weight"], lora_state_dict[f"blocks.{i}.attn2.out_proj.lora_A.weight"]) + transformer_state_dict[f"blocks.{i}.attn2.to_out.0.weight"] += lora_scale * out_proj + + q_proj = torch.matmul(lora_state_dict["pooler.q_proj.lora_B.weight"], lora_state_dict["pooler.q_proj.lora_A.weight"]) + transformer_state_dict["time_extra_emb.pooler.q_proj.weight"] += lora_scale * q_proj + + return transformer_state_dict + +pipe = HunyuanDiTPipeline.from_pretrained("Tencent-Hunyuan/HunyuanDiT-v1.1-Diffusers", torch_dtype=torch.float16) +pipe.to("cuda") + +from safetensors import safe_open + +lora_state_dict = {} +with safe_open("./ckpts/t2i/lora/jade/adapter_model.safetensors", framework="pt", device=0) as f: + for k in f.keys(): + lora_state_dict[k[17:]] = f.get_tensor(k) # remove 'basemodel.model' + +transformer_state_dict = pipe.transformer.state_dict() +transformer_state_dict = load_hunyuan_dit_lora(transformer_state_dict, lora_state_dict, lora_scale=1.0) +pipe.transformer.load_state_dict(transformer_state_dict) + +prompt = "玉石绘画风格，一只猫在追蝴蝶" +image = pipe( + prompt, + num_inference_steps=100, + guidance_scale=6.0, +).images[0] +image.save('img.png') +``` + + +e. For more information, please refer to [HYDiT-LoRA](https://huggingface.co/Tencent-Hunyuan/HYDiT-LoRA). diff --git a/PyTorch/built-in/mlm/HunyuanDiT/lora/merge.py b/PyTorch/built-in/mlm/HunyuanDiT/lora/merge.py new file mode 100644 index 0000000000000000000000000000000000000000..4fd7dc664aaaaf6412ea549080501aaea6e80a69 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/lora/merge.py @@ -0,0 +1,28 @@ +import torch +import os +from hydit.config import get_args +from hydit.modules.models import HUNYUAN_DIT_MODELS + +from hydit.inference import _to_tuple + +args = get_args() + +image_size = _to_tuple(args.image_size) +latent_size = (image_size[0] // 8, image_size[1] // 8) + +model = HUNYUAN_DIT_MODELS[args.model](args, + input_size=latent_size, + log_fn=print, + ) +model_path = os.path.join(args.model_root, 't2i', 'model', f"pytorch_model_{args.load_key}.pt") +state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + +print(f"Loading model from {model_path}") +model.load_state_dict(state_dict) + +print(f"Loading lora from {args.lora_ckpt}") +model.load_adapter(args.lora_ckpt) +model.merge_and_unload() + +torch.save(model.state_dict(), args.output_merge_path) +print(f"Model saved to {args.output_merge_path}") \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/lora/train_lora.sh b/PyTorch/built-in/mlm/HunyuanDiT/lora/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..eadbbe1d0382ca6cf632b46f51f83671594ea78f --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/lora/train_lora.sh @@ -0,0 +1,50 @@ +model='DiT-g/2' # model type +task_flag="lora_porcelain_ema_rank64" # task flag +resume=./ckpts/t2i/model/ # resume checkpoint +index_file=dataset/porcelain/jsons/porcelain.json # the selected data indices +results_dir=./log_EXP # save root for results +batch_size=1 # training batch size +image_size=1024 # training image resolution +grad_accu_steps=2 # gradient accumulation steps +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=100 # create a ckpt every a few steps. +ckpt_latest_every=2000 # create a ckpt named `latest.pt` every a few steps. +rank=64 # rank of lora +max_training_steps=2000 # Maximum training iteration steps + +PYTHONPATH=./ deepspeed hydit/train_deepspeed.py \ + --task-flag ${task_flag} \ + --model ${model} \ + --training-parts lora \ + --rank ${rank} \ + --resume-split \ + --resume ${resume} \ + --ema-to-module \ + --lr ${lr} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --batch-size ${batch_size} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --ckpt-every ${ckpt_every} \ + --max-training-steps ${max_training_steps}\ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 10 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + --qk-norm \ + --rope-img base512 \ + --rope-real \ + "$@" diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/README.md b/PyTorch/built-in/mlm/HunyuanDiT/mllm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..860245d590987b9bb6807a59b575d0441e202dd6 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/README.md @@ -0,0 +1,109 @@ +# Hunyuan-MLLM +We provide two multimodal large language models, Hunyuan-Captioner and Dialogen. The former provides fine-grained text descriptions for training data, while the latter enhances the user's prompt input during inference and supports multi-turn text-to-image generation. + +## Contents +- [Hunyuan-Captioner](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#hunyuan-captioner) + - [Instructions](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#instructions) + - [Examples](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#examples) + - [Inference](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#inference) + - [Gradio](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#gradio) +- [DialogGen](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#dialoggen) + - [Inference](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#inference-1) + - [Gradio](https://github.com/Tencent/HunyuanDiT/tree/main/mllm#gradio-1) + +## Hunyuan-Captioner +Hunyuan-Captioner meets the need of text-to-image techniques by maintaining a high degree of image-text consistency. It can generate high-quality image descriptions from a variety of angles, including object description, objects relationships, background information, image style, etc. Our code is based on [LLaVA](https://github.com/haotian-liu/LLaVA) implementation. + +### Examples + +Image 3 + + +### Instructions +a. Install dependencies + +The dependencies and installation are basically the same as the [**base model**](https://huggingface.co/Tencent-Hunyuan/HunyuanDiT-v1.1). + +b. Data download +```shell +cd HunyuanDiT +wget -O ./dataset/data_demo.zip https://dit.hunyuan.tencent.com/download/HunyuanDiT/data_demo.zip +unzip ./dataset/data_demo.zip -d ./dataset +mkdir ./dataset/porcelain/arrows ./dataset/porcelain/jsons +``` + +c. Model download +```shell +# Use the huggingface-cli tool to download the model. +huggingface-cli download Tencent-Hunyuan/HunyuanCaptioner --local-dir ./ckpts/captioner +``` + +### Inference + +Current supported prompt templates: + +|Mode | Prompt template |Description | +| --- | --- | --- | +|caption_zh | 描述这张图片 |Caption in Chinese | +|insert_content | 根据提示词“{}”,描述这张图片 |Insert specific knowledge into caption| +|caption_en | Please describe the content of this image |Caption in English | +| | | | + + +a. Single picture inference in Chinese + +```bash +python mllm/caption_demo.py --mode "caption_zh" --image_file "mllm/images/demo1.png" --model_path "./ckpts/captioner" +``` + +b. Insert specific knowledge into caption + +```bash +python mllm/caption_demo.py --mode "insert_content" --content "宫保鸡丁" --image_file "mllm/images/demo2.png" --model_path "./ckpts/captioner" +``` + +c. Single picture inference in English + +```bash +python mllm/caption_demo.py --mode "caption_en" --image_file "mllm/images/demo3.png" --model_path "./ckpts/captioner" +``` + +d. Multiple pictures inference in Chinese + +```bash +### Convert multiple pictures to csv file. +python mllm/make_csv.py --img_dir "mllm/images" --input_file "mllm/images/demo.csv" + +### Multiple pictures inference +python mllm/caption_demo.py --mode "caption_zh" --input_file "mllm/images/demo.csv" --output_file "mllm/images/demo_res.csv" --model_path "./ckpts/captioner" +``` + +(Optional) To convert the output csv file to Arrow format, please refer to [Data Preparation #3](#data-preparation) for detailed instructions. + +### Gradio +To launch a Gradio demo locally, please execute the following commands sequentially. Ensure each command is running in the background. For more detailed instructions, please refer to [LLaVA](https://github.com/haotian-liu/LLaVA). +```bash +cd mllm +python -m llava.serve.controller --host 0.0.0.0 --port 10000 +python -m llava.serve.gradio_web_server --controller http://0.0.0.0:10000 --model-list-mode reload --port 443 +python -m llava.serve.model_worker --host 0.0.0.0 --controller http://0.0.0.0:10000 --port 40000 --worker http://0.0.0.0:40000 --model-path "../ckpts/captioner" --model-name LlavaMistral +``` +Then the demo can be accessed through http://0.0.0.0:443. It should be noted that the 0.0.0.0 here needs to be X.X.X.X with your server IP. + + + + +## Hunyuan-DialogGen +We additionally provide inference commands for [DialogGen](https://github.com/Centaurusalpha/DialogGen). +### Inference +```bash +cd HunyuanDiT +python mllm/dialoggen_demo.py --prompt "画一只小猫" +``` + +### Gradio +```bash +# Start a multi-turn T2I generation UI. +# If your GPU memory is less than 32GB, use '--load-4bit' to enable 4-bit quantization, which requires at least 22GB of memory. +python app/multiTurnT2I_app.py +``` \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/caption_demo.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/caption_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..d5d823cf04ab72cb9ec0c6f63c7586a45fb294af --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/caption_demo.py @@ -0,0 +1,182 @@ +import argparse +import torch +import sys +import os +import pandas as pd +import tqdm +# 添加当前命令行运行的目录到 sys.path +sys.path.append(os.getcwd()+"/mllm") + + +from llava.constants import ( + IMAGE_TOKEN_INDEX, + DEFAULT_IMAGE_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IM_END_TOKEN, + IMAGE_PLACEHOLDER, +) +from llava.conversation import conv_templates, SeparatorStyle +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init +from llava.mm_utils import ( + process_images, + tokenizer_image_token, + get_model_name_from_path, +) + +import requests +from PIL import Image +from io import BytesIO +import re + + +def image_parser(image_file, sep=','): + out = image_file.split(sep) + return out + + +def load_image(image_file): + if image_file.startswith("http") or image_file.startswith("https"): + response = requests.get(image_file) + image = Image.open(BytesIO(response.content)).convert("RGB") + else: + image = Image.open(image_file).convert("RGB") + return image + + +def load_images(image_files): + out = [] + for image_file in image_files: + image = load_image(image_file) + out.append(image) + return out + + +def init_dialoggen_model(model_path, model_base=None, load_4bit=False): + model_name = get_model_name_from_path(model_path) + tokenizer, model, image_processor, context_len = load_pretrained_model( + model_path, model_base, model_name, llava_type_model=True, load_4bit=load_4bit) + return {"tokenizer": tokenizer, + "model": model, + "image_processor": image_processor} + + +def eval_model(models, + query='详细描述一下这张图片', + image_file=None, + sep=',', + temperature=0.2, + top_p=None, + num_beams=1, + max_new_tokens=512, + return_history=False, + history=None, + skip_special=False + ): + # Model + disable_torch_init() + + qs = query + image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + if IMAGE_PLACEHOLDER in qs: + if models["model"].config.mm_use_im_start_end: + qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs) + else: + qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs) + else: + if models["model"].config.mm_use_im_start_end: + qs = image_token_se + "\n" + qs + else: + qs = DEFAULT_IMAGE_TOKEN + "\n" + qs + + if not history: + conv = conv_templates['llava_v1'].copy() + else: + conv = history + + if skip_special: + conv.append_message(conv.roles[0], query) + else: + conv.append_message(conv.roles[0], qs) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + if image_file is not None: + image_files = image_parser(image_file, sep=sep) + images = load_images(image_files) + image_sizes = [x.size for x in images] + images_tensor = process_images( + images, + models["image_processor"], + models["model"].config + ).to(models["model"].device, dtype=torch.float16) + else: + # fomatted input as training data + image_sizes = [(1024, 1024)] + images_tensor = torch.zeros(1, 5, 3, models["image_processor"].crop_size["height"], models["image_processor"].crop_size["width"]) + images_tensor = images_tensor.to(models["model"].device, dtype=torch.float16) + + input_ids = ( + tokenizer_image_token(prompt, models["tokenizer"], IMAGE_TOKEN_INDEX, return_tensors="pt") + .unsqueeze(0) + .cuda() + ) + with torch.inference_mode(): + output_ids = models["model"].generate( + input_ids, + images=images_tensor, + image_sizes=image_sizes, + do_sample=True if temperature > 0 else False, + temperature=temperature, + top_p=top_p, + num_beams=num_beams, + max_new_tokens=max_new_tokens, + use_cache=True, + ) + + outputs = models["tokenizer"].batch_decode(output_ids, skip_special_tokens=True)[0].strip() + if return_history: + return outputs, conv + return outputs + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--model_path', type=str, default='/apdcephfs_data_cq5_1/share_300167803/dengxinchi/project/LLaVA/checkpoints/exps/caption/caption_v3_prolr2-5_lr2-5_v5_checkpoint-50000_merge_lora') + parser.add_argument('--mode', choices=['caption_zh','caption_en','insert_content'], default="caption_zh") + parser.add_argument('--content', type=str, default=None) + parser.add_argument('--input_file', type=str, default=None) # 'images/demo.csv' + parser.add_argument('--output_file', type=str, default=None) # 'images/demo_res.csv' + parser.add_argument('--image_file', type=str, default='images/demo1.jpeg') # 'images/demo1.jpeg' + args = parser.parse_args() + + if args.mode == 'caption_zh': + query = "描述这张图片" + elif args.mode == 'caption_en': + query = 'Please describe the content of this image' + elif args.mode == 'insert_content': + assert args.content is not None + query = f'根据提示词“{args.content}”,描述这张图片' + + + models = init_dialoggen_model(args.model_path) + + if args.input_file != None: + df = pd.read_csv(args.input_file) + text_zh = [] + for i in tqdm.tqdm(range(len(df))): + img_path = df.loc[i]["img_path"] + res = eval_model(models, + query=query, + image_file=img_path, + ) + text_zh.append(res) + df["text_zh"] = text_zh + df.to_csv(args.output_file, index=False, encoding='utf-8-sig') + else: + res = eval_model(models, + query=query, + image_file=args.image_file, + ) + print(res) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/dialoggen_demo.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/dialoggen_demo.py new file mode 100644 index 0000000000000000000000000000000000000000..d02f9bb5d2e97ad5255ed8c6010a171155b5f70c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/dialoggen_demo.py @@ -0,0 +1,189 @@ +import argparse +import torch +import sys +import os +# 添加当前命令行运行的目录到 sys.path +sys.path.append(os.getcwd()+"/mllm") + + +from llava.constants import ( + IMAGE_TOKEN_INDEX, + DEFAULT_IMAGE_TOKEN, + DEFAULT_IM_START_TOKEN, + DEFAULT_IM_END_TOKEN, + IMAGE_PLACEHOLDER, +) +from llava.conversation import conv_templates, SeparatorStyle +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init +from llava.mm_utils import ( + process_images, + tokenizer_image_token, + get_model_name_from_path, +) + +import requests +from PIL import Image +from io import BytesIO +import re + + +def image_parser(image_file, sep=','): + out = image_file.split(sep) + return out + + +def load_image(image_file): + if image_file.startswith("http") or image_file.startswith("https"): + response = requests.get(image_file) + image = Image.open(BytesIO(response.content)).convert("RGB") + else: + image = Image.open(image_file).convert("RGB") + return image + + +def load_images(image_files): + out = [] + for image_file in image_files: + image = load_image(image_file) + out.append(image) + return out + + +def init_dialoggen_model(model_path, model_base=None, load_4bit=False): + model_name = get_model_name_from_path(model_path) + tokenizer, model, image_processor, context_len = load_pretrained_model( + model_path, model_base, model_name, llava_type_model=True, load_4bit=load_4bit) + return {"tokenizer": tokenizer, + "model": model, + "image_processor": image_processor} + + +def eval_model(models, + query='详细描述一下这张图片', + image_file=None, + sep=',', + temperature=0.2, + top_p=None, + num_beams=1, + max_new_tokens=512, + return_history=False, + history=None, + skip_special=False + ): + # Model + disable_torch_init() + + qs = query + image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + if IMAGE_PLACEHOLDER in qs: + if models["model"].config.mm_use_im_start_end: + qs = re.sub(IMAGE_PLACEHOLDER, image_token_se, qs) + else: + qs = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, qs) + else: + if models["model"].config.mm_use_im_start_end: + qs = image_token_se + "\n" + qs + else: + qs = DEFAULT_IMAGE_TOKEN + "\n" + qs + + if not history: + conv = conv_templates['llava_v1'].copy() + else: + conv = history + + if skip_special: + conv.append_message(conv.roles[0], query) + else: + conv.append_message(conv.roles[0], qs) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + if image_file is not None: + image_files = image_parser(image_file, sep=sep) + images = load_images(image_files) + image_sizes = [x.size for x in images] + images_tensor = process_images( + images, + models["image_processor"], + models["model"].config + ).to(models["model"].device, dtype=torch.float16) + else: + # fomatted input as training data + image_sizes = [(1024, 1024)] + images_tensor = torch.zeros(1, 5, 3, models["image_processor"].crop_size["height"], models["image_processor"].crop_size["width"]) + images_tensor = images_tensor.to(models["model"].device, dtype=torch.float16) + + input_ids = ( + tokenizer_image_token(prompt, models["tokenizer"], IMAGE_TOKEN_INDEX, return_tensors="pt") + .unsqueeze(0) + .cuda() + ) + with torch.inference_mode(): + output_ids = models["model"].generate( + input_ids, + images=images_tensor, + image_sizes=image_sizes, + do_sample=True if temperature > 0 else False, + temperature=temperature, + top_p=top_p, + num_beams=num_beams, + max_new_tokens=max_new_tokens, + use_cache=True, + ) + + outputs = models["tokenizer"].batch_decode(output_ids, skip_special_tokens=True)[0].strip() + if return_history: + return outputs, conv + return outputs + + +def remove_prefix(text): + if text.startswith("<画图>"): + return text[len("<画图>"):], True + elif text.startswith("对不起"): + # 拒绝画图 + return "", False + else: + return text, True + + +class DialogGen(object): + def __init__(self, model_path, load_4bit=False): + self.models = init_dialoggen_model(model_path, load_4bit=load_4bit) + self.query_template = "请先判断用户的意图，若为画图则在输出前加入<画图>:{}" + + def __call__(self, prompt, return_history=False, history=None, skip_special=False): + enhanced_prompt = eval_model( + models=self.models, + query=self.query_template.format(prompt), + image_file=None, + return_history=return_history, + history=history, + skip_special=skip_special + ) + if return_history: + return enhanced_prompt + + enhanced_prompt, compliance = remove_prefix(enhanced_prompt) + if not compliance: + return False, "" + return True, enhanced_prompt + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--model_path', type=str, default='./ckpts/dialoggen') + parser.add_argument('--prompt', type=str, default='画一只小猫') + parser.add_argument('--image_file', type=str, default=None) # 'images/demo1.jpeg' + args = parser.parse_args() + + query = f"请先判断用户的意图，若为画图则在输出前加入<画图>:{args.prompt}" + + models = init_dialoggen_model(args.model_path) + + res = eval_model(models, + query=query, + image_file=args.image_file, + ) + print(res) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4d1f016db1028101d45ba7d68cb3f0bcb558c2bb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/__init__.py @@ -0,0 +1 @@ +from .model import LlavaLlamaForCausalLM diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/constants.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..374be090510b302de9882d880c755787a8eafe11 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/constants.py @@ -0,0 +1,13 @@ +CONTROLLER_HEART_BEAT_EXPIRATION = 30 +WORKER_HEART_BEAT_INTERVAL = 15 + +LOGDIR = "." + +# Model Constants +IGNORE_INDEX = -100 +IMAGE_TOKEN_INDEX = -200 +DEFAULT_IMAGE_TOKEN = "" +DEFAULT_IMAGE_PATCH_TOKEN = "" +DEFAULT_IM_START_TOKEN = "" +DEFAULT_IM_END_TOKEN = "" +IMAGE_PLACEHOLDER = "" diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/conversation.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/conversation.py new file mode 100644 index 0000000000000000000000000000000000000000..00c56867dd1fd88094df9556f3d1c57e71a7ada8 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/conversation.py @@ -0,0 +1,396 @@ +import dataclasses +from enum import auto, Enum +from typing import List, Tuple +import base64 +from io import BytesIO +from PIL import Image + + +class SeparatorStyle(Enum): + """Different separator style.""" + SINGLE = auto() + TWO = auto() + MPT = auto() + PLAIN = auto() + LLAMA_2 = auto() + + +@dataclasses.dataclass +class Conversation: + """A class that keeps all conversation history.""" + system: str + roles: List[str] + messages: List[List[str]] + offset: int + sep_style: SeparatorStyle = SeparatorStyle.SINGLE + sep: str = "###" + sep2: str = None + version: str = "Unknown" + + skip_next: bool = False + + def get_prompt(self): + messages = self.messages + if len(messages) > 0 and type(messages[0][1]) is tuple: + messages = self.messages.copy() + init_role, init_msg = messages[0].copy() + init_msg = init_msg[0].replace("", "").strip() + if 'mmtag' in self.version: + messages[0] = (init_role, init_msg) + messages.insert(0, (self.roles[0], "")) + messages.insert(1, (self.roles[1], "Received.")) + else: + messages[0] = (init_role, "\n" + init_msg) + + if self.sep_style == SeparatorStyle.SINGLE: + ret = self.system + self.sep + for role, message in messages: + if message: + if type(message) is tuple: + message, _, _ = message + ret += role + ": " + message + self.sep + else: + ret += role + ":" + elif self.sep_style == SeparatorStyle.TWO: + seps = [self.sep, self.sep2] + ret = self.system + seps[0] + for i, (role, message) in enumerate(messages): + if message: + if type(message) is tuple: + message, _, _ = message + ret += role + ": " + message + seps[i % 2] + else: + ret += role + ":" + elif self.sep_style == SeparatorStyle.MPT: + ret = self.system + self.sep + for role, message in messages: + if message: + if type(message) is tuple: + message, _, _ = message + ret += role + message + self.sep + else: + ret += role + elif self.sep_style == SeparatorStyle.LLAMA_2: + wrap_sys = lambda msg: f"<>\n{msg}\n<>\n\n" if len(msg) > 0 else msg + wrap_inst = lambda msg: f"[INST] {msg} [/INST]" + ret = "" + + for i, (role, message) in enumerate(messages): + if i == 0: + assert message, "first message should not be none" + assert role == self.roles[0], "first message should come from user" + if message: + if type(message) is tuple: + message, _, _ = message + if i == 0: message = wrap_sys(self.system) + message + if i % 2 == 0: + message = wrap_inst(message) + ret += self.sep + message + else: + ret += " " + message + " " + self.sep2 + else: + ret += "" + ret = ret.lstrip(self.sep) + elif self.sep_style == SeparatorStyle.PLAIN: + seps = [self.sep, self.sep2] + ret = self.system + for i, (role, message) in enumerate(messages): + if message: + if type(message) is tuple: + message, _, _ = message + ret += message + seps[i % 2] + else: + ret += "" + else: + raise ValueError(f"Invalid style: {self.sep_style}") + + return ret + + def append_message(self, role, message): + self.messages.append([role, message]) + + def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672): + if image_process_mode == "Pad": + def expand2square(pil_img, background_color=(122, 116, 104)): + width, height = pil_img.size + if width == height: + return pil_img + elif width > height: + result = Image.new(pil_img.mode, (width, width), background_color) + result.paste(pil_img, (0, (width - height) // 2)) + return result + else: + result = Image.new(pil_img.mode, (height, height), background_color) + result.paste(pil_img, ((height - width) // 2, 0)) + return result + image = expand2square(image) + elif image_process_mode in ["Default", "Crop"]: + pass + elif image_process_mode == "Resize": + image = image.resize((336, 336)) + else: + raise ValueError(f"Invalid image_process_mode: {image_process_mode}") + if max(image.size) > max_len: + max_hw, min_hw = max(image.size), min(image.size) + aspect_ratio = max_hw / min_hw + shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw)) + longest_edge = int(shortest_edge * aspect_ratio) + W, H = image.size + if H > W: + H, W = longest_edge, shortest_edge + else: + H, W = shortest_edge, longest_edge + image = image.resize((W, H)) + if return_pil: + return image + else: + buffered = BytesIO() + image.save(buffered, format=image_format) + img_b64_str = base64.b64encode(buffered.getvalue()).decode() + return img_b64_str + + def get_images(self, return_pil=False): + images = [] + for i, (role, msg) in enumerate(self.messages[self.offset:]): + if i % 2 == 0: + if type(msg) is tuple: + msg, image, image_process_mode = msg + image = self.process_image(image, image_process_mode, return_pil=return_pil) + images.append(image) + return images + + def to_gradio_chatbot(self): + ret = [] + for i, (role, msg) in enumerate(self.messages[self.offset:]): + if i % 2 == 0: + if type(msg) is tuple: + msg, image, image_process_mode = msg + img_b64_str = self.process_image( + image, "Default", return_pil=False, + image_format='JPEG') + img_str = f'user upload image' + msg = img_str + msg.replace('', '').strip() + ret.append([msg, None]) + else: + ret.append([msg, None]) + else: + ret[-1][-1] = msg + return ret + + def copy(self): + return Conversation( + system=self.system, + roles=self.roles, + messages=[[x, y] for x, y in self.messages], + offset=self.offset, + sep_style=self.sep_style, + sep=self.sep, + sep2=self.sep2, + version=self.version) + + def dict(self): + if len(self.get_images()) > 0: + return { + "system": self.system, + "roles": self.roles, + "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages], + "offset": self.offset, + "sep": self.sep, + "sep2": self.sep2, + } + return { + "system": self.system, + "roles": self.roles, + "messages": self.messages, + "offset": self.offset, + "sep": self.sep, + "sep2": self.sep2, + } + + +conv_vicuna_v0 = Conversation( + system="A chat between a curious human and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the human's questions.", + roles=("Human", "Assistant"), + messages=( + ("Human", "What are the key differences between renewable and non-renewable energy sources?"), + ("Assistant", + "Renewable energy sources are those that can be replenished naturally in a relatively " + "short amount of time, such as solar, wind, hydro, geothermal, and biomass. " + "Non-renewable energy sources, on the other hand, are finite and will eventually be " + "depleted, such as coal, oil, and natural gas. Here are some key differences between " + "renewable and non-renewable energy sources:\n" + "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable " + "energy sources are finite and will eventually run out.\n" + "2. Environmental impact: Renewable energy sources have a much lower environmental impact " + "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, " + "and other negative effects.\n" + "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically " + "have lower operational costs than non-renewable sources.\n" + "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote " + "locations than non-renewable sources.\n" + "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different " + "situations and needs, while non-renewable sources are more rigid and inflexible.\n" + "6. Sustainability: Renewable energy sources are more sustainable over the long term, while " + "non-renewable sources are not, and their depletion can lead to economic and social instability.\n") + ), + offset=2, + sep_style=SeparatorStyle.SINGLE, + sep="###", +) + +conv_vicuna_v1 = Conversation( + system="A chat between a curious user and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the user's questions.", + roles=("USER", "ASSISTANT"), + version="v1", + messages=(), + offset=0, + sep_style=SeparatorStyle.TWO, + sep=" ", + sep2="", +) + +conv_llama_2 = Conversation( + system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. + +If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""", + roles=("USER", "ASSISTANT"), + version="llama_v2", + messages=(), + offset=0, + sep_style=SeparatorStyle.LLAMA_2, + sep="", + sep2="", +) + +conv_llava_llama_2 = Conversation( + system="You are a helpful language and vision assistant. " + "You are able to understand the visual content that the user provides, " + "and assist the user with a variety of tasks using natural language.", + roles=("USER", "ASSISTANT"), + version="llama_v2", + messages=(), + offset=0, + sep_style=SeparatorStyle.LLAMA_2, + sep="", + sep2="", +) + +conv_mpt = Conversation( + system="""<|im_start|>system +A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""", + roles=("<|im_start|>user\n", "<|im_start|>assistant\n"), + version="mpt", + messages=(), + offset=0, + sep_style=SeparatorStyle.MPT, + sep="<|im_end|>", +) + +conv_llava_plain = Conversation( + system="", + roles=("", ""), + messages=( + ), + offset=0, + sep_style=SeparatorStyle.PLAIN, + sep="\n", +) + +conv_llava_v0 = Conversation( + system="A chat between a curious human and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the human's questions.", + roles=("Human", "Assistant"), + messages=( + ), + offset=0, + sep_style=SeparatorStyle.SINGLE, + sep="###", +) + +conv_llava_v0_mmtag = Conversation( + system="A chat between a curious user and an artificial intelligence assistant. " + "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language." + "The visual content will be provided with the following format: visual content.", + roles=("Human", "Assistant"), + messages=( + ), + offset=0, + sep_style=SeparatorStyle.SINGLE, + sep="###", + version="v0_mmtag", +) + +conv_llava_v1 = Conversation( + system="A chat between a curious human and an artificial intelligence assistant. " + "The assistant gives helpful, detailed, and polite answers to the human's questions.", + roles=("USER", "ASSISTANT"), + version="v1", + messages=(), + offset=0, + sep_style=SeparatorStyle.TWO, + sep=" ", + sep2="", +) + +conv_llava_v1_mmtag = Conversation( + system="A chat between a curious user and an artificial intelligence assistant. " + "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language." + "The visual content will be provided with the following format: visual content.", + roles=("USER", "ASSISTANT"), + messages=(), + offset=0, + sep_style=SeparatorStyle.TWO, + sep=" ", + sep2="", + version="v1_mmtag", +) + +conv_mistral_instruct = Conversation( + system="", + roles=("USER", "ASSISTANT"), + version="llama_v2", + messages=(), + offset=0, + sep_style=SeparatorStyle.LLAMA_2, + sep="", + sep2="", +) + +conv_chatml_direct = Conversation( + system="""<|im_start|>system +Answer the questions.""", + roles=("<|im_start|>user\n", "<|im_start|>assistant\n"), + version="mpt", + messages=(), + offset=0, + sep_style=SeparatorStyle.MPT, + sep="<|im_end|>", +) + +default_conversation = conv_vicuna_v1 +conv_templates = { + "default": conv_vicuna_v0, + "v0": conv_vicuna_v0, + "v1": conv_vicuna_v1, + "vicuna_v1": conv_vicuna_v1, + "llama_2": conv_llama_2, + "mistral_instruct": conv_mistral_instruct, + "chatml_direct": conv_chatml_direct, + "mistral_direct": conv_chatml_direct, + + "plain": conv_llava_plain, + "v0_plain": conv_llava_plain, + "llava_v0": conv_llava_v0, + "v0_mmtag": conv_llava_v0_mmtag, + "llava_v1": conv_llava_v1, + "v1_mmtag": conv_llava_v1_mmtag, + "llava_llama_2": conv_llava_llama_2, + + "mpt": conv_mpt, +} + + +if __name__ == "__main__": + print(default_conversation.get_prompt()) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/mm_utils.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/mm_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9717a81f8e481a452bbd99f4aa0baad95d0306df --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/mm_utils.py @@ -0,0 +1,247 @@ +from PIL import Image +from io import BytesIO +import base64 +import torch +import math +import ast + +from transformers import StoppingCriteria +from llava.constants import IMAGE_TOKEN_INDEX + + +def select_best_resolution(original_size, possible_resolutions): + """ + Selects the best resolution from a list of possible resolutions based on the original size. + + Args: + original_size (tuple): The original size of the image in the format (width, height). + possible_resolutions (list): A list of possible resolutions in the format [(width1, height1), (width2, height2), ...]. + + Returns: + tuple: The best fit resolution in the format (width, height). + """ + original_width, original_height = original_size + best_fit = None + max_effective_resolution = 0 + min_wasted_resolution = float('inf') + + for width, height in possible_resolutions: + scale = min(width / original_width, height / original_height) + downscaled_width, downscaled_height = int(original_width * scale), int(original_height * scale) + effective_resolution = min(downscaled_width * downscaled_height, original_width * original_height) + wasted_resolution = (width * height) - effective_resolution + + if effective_resolution > max_effective_resolution or (effective_resolution == max_effective_resolution and wasted_resolution < min_wasted_resolution): + max_effective_resolution = effective_resolution + min_wasted_resolution = wasted_resolution + best_fit = (width, height) + + return best_fit + + +def resize_and_pad_image(image, target_resolution): + """ + Resize and pad an image to a target resolution while maintaining aspect ratio. + + Args: + image (PIL.Image.Image): The input image. + target_resolution (tuple): The target resolution (width, height) of the image. + + Returns: + PIL.Image.Image: The resized and padded image. + """ + original_width, original_height = image.size + target_width, target_height = target_resolution + + scale_w = target_width / original_width + scale_h = target_height / original_height + + if scale_w < scale_h: + new_width = target_width + new_height = min(math.ceil(original_height * scale_w), target_height) + else: + new_height = target_height + new_width = min(math.ceil(original_width * scale_h), target_width) + + # Resize the image + resized_image = image.resize((new_width, new_height)) + + new_image = Image.new('RGB', (target_width, target_height), (0, 0, 0)) + paste_x = (target_width - new_width) // 2 + paste_y = (target_height - new_height) // 2 + new_image.paste(resized_image, (paste_x, paste_y)) + + return new_image + + +def divide_to_patches(image, patch_size): + """ + Divides an image into patches of a specified size. + + Args: + image (PIL.Image.Image): The input image. + patch_size (int): The size of each patch. + + Returns: + list: A list of PIL.Image.Image objects representing the patches. + """ + patches = [] + width, height = image.size + for i in range(0, height, patch_size): + for j in range(0, width, patch_size): + box = (j, i, j + patch_size, i + patch_size) + patch = image.crop(box) + patches.append(patch) + + return patches + + +def get_anyres_image_grid_shape(image_size, grid_pinpoints, patch_size): + """ + Calculate the shape of the image patch grid after the preprocessing for images of any resolution. + + Args: + image_size (tuple): The size of the input image in the format (width, height). + grid_pinpoints (str): A string representation of a list of possible resolutions. + patch_size (int): The size of each image patch. + + Returns: + tuple: The shape of the image patch grid in the format (width, height). + """ + if type(grid_pinpoints) is list: + possible_resolutions = grid_pinpoints + else: + possible_resolutions = ast.literal_eval(grid_pinpoints) + width, height = select_best_resolution(image_size, possible_resolutions) + return width // patch_size, height // patch_size + + +def process_anyres_image(image, processor, grid_pinpoints): + """ + Process an image with variable resolutions. + + Args: + image (PIL.Image.Image): The input image to be processed. + processor: The image processor object. + grid_pinpoints (str): A string representation of a list of possible resolutions. + + Returns: + torch.Tensor: A tensor containing the processed image patches. + """ + if type(grid_pinpoints) is list: + possible_resolutions = grid_pinpoints + else: + possible_resolutions = ast.literal_eval(grid_pinpoints) + best_resolution = select_best_resolution(image.size, possible_resolutions) + image_padded = resize_and_pad_image(image, best_resolution) + + patches = divide_to_patches(image_padded, processor.crop_size['height']) + + image_original_resize = image.resize((processor.size['shortest_edge'], processor.size['shortest_edge'])) + + image_patches = [image_original_resize] + patches + image_patches = [processor.preprocess(image_patch, return_tensors='pt')['pixel_values'][0] + for image_patch in image_patches] + return torch.stack(image_patches, dim=0) + + +def load_image_from_base64(image): + return Image.open(BytesIO(base64.b64decode(image))) + + +def expand2square(pil_img, background_color): + width, height = pil_img.size + if width == height: + return pil_img + elif width > height: + result = Image.new(pil_img.mode, (width, width), background_color) + result.paste(pil_img, (0, (width - height) // 2)) + return result + else: + result = Image.new(pil_img.mode, (height, height), background_color) + result.paste(pil_img, ((height - width) // 2, 0)) + return result + + +def process_images(images, image_processor, model_cfg): + image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None) + new_images = [] + if image_aspect_ratio == 'pad': + for image in images: + image = expand2square(image, tuple(int(x*255) for x in image_processor.image_mean)) + image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0] + new_images.append(image) + elif image_aspect_ratio == "anyres": + for image in images: + image = process_anyres_image(image, image_processor, model_cfg.image_grid_pinpoints) + new_images.append(image) + else: + return image_processor(images, return_tensors='pt')['pixel_values'] + if all(x.shape == new_images[0].shape for x in new_images): + new_images = torch.stack(new_images, dim=0) + return new_images + + +def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None): + prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('')] + + def insert_separator(X, sep): + return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1] + + input_ids = [] + offset = 0 + if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id: + offset = 1 + input_ids.append(prompt_chunks[0][0]) + + for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)): + input_ids.extend(x[offset:]) + + if return_tensors is not None: + if return_tensors == 'pt': + return torch.tensor(input_ids, dtype=torch.long) + raise ValueError(f'Unsupported tensor type: {return_tensors}') + return input_ids + + +def get_model_name_from_path(model_path): + model_path = model_path.strip("/") + model_paths = model_path.split("/") + if model_paths[-1].startswith('checkpoint-'): + return model_paths[-2] + "_" + model_paths[-1] + else: + return model_paths[-1] + +class KeywordsStoppingCriteria(StoppingCriteria): + def __init__(self, keywords, tokenizer, input_ids): + self.keywords = keywords + self.keyword_ids = [] + self.max_keyword_len = 0 + for keyword in keywords: + cur_keyword_ids = tokenizer(keyword).input_ids + if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id: + cur_keyword_ids = cur_keyword_ids[1:] + if len(cur_keyword_ids) > self.max_keyword_len: + self.max_keyword_len = len(cur_keyword_ids) + self.keyword_ids.append(torch.tensor(cur_keyword_ids)) + self.tokenizer = tokenizer + self.start_len = input_ids.shape[1] + + def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: + offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len) + self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids] + for keyword_id in self.keyword_ids: + truncated_output_ids = output_ids[0, -keyword_id.shape[0]:] + if torch.equal(truncated_output_ids, keyword_id): + return True + outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0] + for keyword in self.keywords: + if keyword in outputs: + return True + return False + + def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: + outputs = [] + for i in range(output_ids.shape[0]): + outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores)) + return all(outputs) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd91789f0cde61dd13a7f9a5f7a69488ad07279 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/__init__.py @@ -0,0 +1,6 @@ +try: + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_mpt import LlavaMptForCausalLM, LlavaMptConfig + from .language_model.llava_mistral import LlavaMistralForCausalLM, LlavaMistralConfig +except: + pass diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/apply_delta.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/apply_delta.py new file mode 100644 index 0000000000000000000000000000000000000000..666dd9691bde7d54ddf2871e311d6f621e29f099 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/apply_delta.py @@ -0,0 +1,48 @@ +""" +Usage: +python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta +""" +import argparse + +import torch +from tqdm import tqdm +from transformers import AutoTokenizer, AutoModelForCausalLM +from llava import LlavaLlamaForCausalLM + + +def apply_delta(base_model_path, target_model_path, delta_path): + print("Loading base model") + base = AutoModelForCausalLM.from_pretrained( + base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + + print("Loading delta") + delta = LlavaLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + delta_tokenizer = AutoTokenizer.from_pretrained(delta_path) + + print("Applying delta") + for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"): + if name not in base.state_dict(): + assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model' + continue + if param.data.shape == base.state_dict()[name].shape: + param.data += base.state_dict()[name] + else: + assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \ + f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}' + bparam = base.state_dict()[name] + param.data[:bparam.shape[0], :bparam.shape[1]] += bparam + + print("Saving target model") + delta.save_pretrained(target_model_path) + delta_tokenizer.save_pretrained(target_model_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base-model-path", type=str, required=True) + parser.add_argument("--target-model-path", type=str, required=True) + parser.add_argument("--delta-path", type=str, required=True) + + args = parser.parse_args() + + apply_delta(args.base_model_path, args.target_model_path, args.delta_path) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/builder.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..263d5d1dea46912d7f8eb767007e89c4471eacf4 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/builder.py @@ -0,0 +1,166 @@ +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import warnings +import shutil + +from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig +import torch +from llava.model import * +from llava.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN + + +def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda", use_flash_attn=False, llava_type_model=True, **kwargs): + kwargs = {"device_map": device_map, **kwargs} + + if device != "cuda": + kwargs['device_map'] = {"": device} + if load_8bit: + kwargs['load_in_8bit'] = True + elif load_4bit: + kwargs['load_in_4bit'] = True + kwargs['quantization_config'] = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4' + ) + else: + kwargs['torch_dtype'] = torch.float16 + + if use_flash_attn: + kwargs['attn_implementation'] = 'flash_attention_2' + + if 'llava' in model_name.lower(): + # Load LLaVA model + if 'lora' in model_name.lower() and model_base is None: + warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument. Detailed instruction: https://github.com/haotian-liu/LLaVA#launch-a-model-worker-lora-weights-unmerged.') + if 'lora' in model_name.lower() and model_base is not None: + from llava.model.language_model.llava_llama import LlavaConfig + lora_cfg_pretrained = LlavaConfig.from_pretrained(model_path) + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False) + print('Loading LLaVA from base model...') + model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs) + token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features + if model.lm_head.weight.shape[0] != token_num: + model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype)) + model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype)) + + print('Loading additional LLaVA weights...') + if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')): + non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu') + else: + # this is probably from HF Hub + from huggingface_hub import hf_hub_download + def load_from_hf(repo_id, filename, subfolder=None): + cache_file = hf_hub_download( + repo_id=repo_id, + filename=filename, + subfolder=subfolder) + return torch.load(cache_file, map_location='cpu') + non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin') + non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()} + if any(k.startswith('model.model.') for k in non_lora_trainables): + non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()} + model.load_state_dict(non_lora_trainables, strict=False) + + from peft import PeftModel + print('Loading LoRA weights...') + model = PeftModel.from_pretrained(model, model_path) + print('Merging LoRA weights...') + model = model.merge_and_unload() + print('Model is loaded...') + elif model_base is not None: + # this may be mm projector only + print('Loading LLaVA from base model...') + if 'mpt' in model_name.lower(): + if not os.path.isfile(os.path.join(model_path, 'configuration_mpt.py')): + shutil.copyfile(os.path.join(model_base, 'configuration_mpt.py'), os.path.join(model_path, 'configuration_mpt.py')) + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=True) + cfg_pretrained = AutoConfig.from_pretrained(model_path, trust_remote_code=True) + model = LlavaMptForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs) + else: + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False) + cfg_pretrained = AutoConfig.from_pretrained(model_path) + model = LlavaLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs) + + mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu') + mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()} + model.load_state_dict(mm_projector_weights, strict=False) + else: + if 'mpt' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True) + model = LlavaMptForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs) + elif 'mistral' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained(model_path) + model = LlavaMistralForCausalLM.from_pretrained( + model_path, + low_cpu_mem_usage=True, + **kwargs + ) + else: + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + model = LlavaLlamaForCausalLM.from_pretrained( + model_path, + low_cpu_mem_usage=True, + **kwargs + ) + else: + # Load language model + if model_base is not None: + # PEFT model + from peft import PeftModel + tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False) + model = AutoModelForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, **kwargs) + print(f"Loading LoRA weights from {model_path}") + model = PeftModel.from_pretrained(model, model_path) + print(f"Merging weights") + model = model.merge_and_unload() + print('Convert to FP16...') + model.to(torch.float16) + else: + use_fast = False + if 'mpt' in model_name.lower(): + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True) + model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, trust_remote_code=True, **kwargs) + else: + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) + model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs) + + image_processor = None + + if llava_type_model: + mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False) + mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True) + if mm_use_im_patch_token: + tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True) + if mm_use_im_start_end: + tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True) + model.resize_token_embeddings(len(tokenizer)) + + vision_tower = model.get_vision_tower() + if not vision_tower.is_loaded: + vision_tower.load_model(device_map=device_map) + if device_map != 'auto': + vision_tower.to(device=device_map, dtype=torch.float16) + image_processor = vision_tower.image_processor + + if hasattr(model.config, "max_sequence_length"): + context_len = model.config.max_sequence_length + else: + context_len = 2048 + + return tokenizer, model, image_processor, context_len diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/consolidate.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/consolidate.py new file mode 100644 index 0000000000000000000000000000000000000000..1e324210e229eeba23b75791bba82df7c6e639eb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/consolidate.py @@ -0,0 +1,29 @@ +""" +Usage: +python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate +""" +import argparse + +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM +from llava.model import * +from llava.model.utils import auto_upgrade + + +def consolidate_ckpt(src_path, dst_path): + print("Loading model") + auto_upgrade(src_path) + src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + src_tokenizer = AutoTokenizer.from_pretrained(src_path, use_fast=False) + src_model.save_pretrained(dst_path) + src_tokenizer.save_pretrained(dst_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--src", type=str, required=True) + parser.add_argument("--dst", type=str, required=True) + + args = parser.parse_args() + + consolidate_ckpt(args.src, args.dst) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_llama.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_llama.py new file mode 100644 index 0000000000000000000000000000000000000000..069d0d1c10da42f5d278598e8534f166d1f9f5ff --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_llama.py @@ -0,0 +1,158 @@ +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from transformers import AutoConfig, AutoModelForCausalLM, \ + LlamaConfig, LlamaModel, LlamaForCausalLM + +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.generation.utils import GenerateOutput + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +class LlavaConfig(LlamaConfig): + model_type = "llava_llama" + + +class LlavaLlamaModel(LlavaMetaModel, LlamaModel): + config_class = LlavaConfig + + def __init__(self, config: LlamaConfig): + super(LlavaLlamaModel, self).__init__(config) + + +class LlavaLlamaForCausalLM(LlamaForCausalLM, LlavaMetaForCausalLM): + config_class = LlavaConfig + + def __init__(self, config): + super(LlamaForCausalLM, self).__init__(config) + self.model = LlavaLlamaModel(config) + self.pretraining_tp = config.pretraining_tp + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_model(self): + return self.model + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + images: Optional[torch.FloatTensor] = None, + image_sizes: Optional[List[List[int]]] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + + if inputs_embeds is None: + ( + input_ids, + position_ids, + attention_mask, + past_key_values, + inputs_embeds, + labels + ) = self.prepare_inputs_labels_for_multimodal( + input_ids, + position_ids, + attention_mask, + past_key_values, + labels, + images, + image_sizes + ) + + return super().forward( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + labels=labels, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict + ) + + @torch.no_grad() + def generate( + self, + inputs: Optional[torch.Tensor] = None, + images: Optional[torch.Tensor] = None, + image_sizes: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[GenerateOutput, torch.LongTensor]: + position_ids = kwargs.pop("position_ids", None) + attention_mask = kwargs.pop("attention_mask", None) + if "inputs_embeds" in kwargs: + raise NotImplementedError("`inputs_embeds` is not supported") + + if images is not None: + ( + inputs, + position_ids, + attention_mask, + _, + inputs_embeds, + _ + ) = self.prepare_inputs_labels_for_multimodal( + inputs, + position_ids, + attention_mask, + None, + None, + images, + image_sizes=image_sizes + ) + else: + inputs_embeds = self.get_model().embed_tokens(inputs) + + return super().generate( + position_ids=position_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + **kwargs + ) + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, + inputs_embeds=None, **kwargs): + images = kwargs.pop("images", None) + image_sizes = kwargs.pop("image_sizes", None) + inputs = super().prepare_inputs_for_generation( + input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs + ) + if images is not None: + inputs['images'] = images + if image_sizes is not None: + inputs['image_sizes'] = image_sizes + return inputs + +AutoConfig.register("llava_llama", LlavaConfig) +AutoModelForCausalLM.register(LlavaConfig, LlavaLlamaForCausalLM) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mistral.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mistral.py new file mode 100644 index 0000000000000000000000000000000000000000..0def682ea3c497e36aa85f1c53eb2cfab6e2fb87 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mistral.py @@ -0,0 +1,158 @@ +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +from torch.nn import CrossEntropyLoss + +from transformers import AutoConfig, AutoModelForCausalLM, \ + MistralConfig, MistralModel, MistralForCausalLM + +from transformers.modeling_outputs import CausalLMOutputWithPast +from transformers.generation.utils import GenerateOutput + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +class LlavaMistralConfig(MistralConfig): + model_type = "llava_mistral" + + +class LlavaMistralModel(LlavaMetaModel, MistralModel): + config_class = LlavaMistralConfig + + def __init__(self, config: MistralConfig): + super(LlavaMistralModel, self).__init__(config) + + +class LlavaMistralForCausalLM(MistralForCausalLM, LlavaMetaForCausalLM): + config_class = LlavaMistralConfig + + def __init__(self, config): + super(MistralForCausalLM, self).__init__(config) + self.model = LlavaMistralModel(config) + + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_model(self): + return self.model + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + images: Optional[torch.FloatTensor] = None, + image_sizes: Optional[List[List[int]]] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithPast]: + + if inputs_embeds is None: + ( + input_ids, + position_ids, + attention_mask, + past_key_values, + inputs_embeds, + labels + ) = self.prepare_inputs_labels_for_multimodal( + input_ids, + position_ids, + attention_mask, + past_key_values, + labels, + images, + image_sizes + ) + + return super().forward( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + labels=labels, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict + ) + + @torch.no_grad() + def generate( + self, + inputs: Optional[torch.Tensor] = None, + images: Optional[torch.Tensor] = None, + image_sizes: Optional[torch.Tensor] = None, + **kwargs, + ) -> Union[GenerateOutput, torch.LongTensor]: + position_ids = kwargs.pop("position_ids", None) + attention_mask = kwargs.pop("attention_mask", None) + if "inputs_embeds" in kwargs: + raise NotImplementedError("`inputs_embeds` is not supported") + + if images is not None: + ( + inputs, + position_ids, + attention_mask, + _, + inputs_embeds, + _ + ) = self.prepare_inputs_labels_for_multimodal( + inputs, + position_ids, + attention_mask, + None, + None, + images, + image_sizes=image_sizes + ) + else: + inputs_embeds = self.get_model().embed_tokens(inputs) + + return super().generate( + position_ids=position_ids, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + **kwargs + ) + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, + inputs_embeds=None, **kwargs): + images = kwargs.pop("images", None) + image_sizes = kwargs.pop("image_sizes", None) + inputs = super().prepare_inputs_for_generation( + input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs + ) + if images is not None: + inputs['images'] = images + if image_sizes is not None: + inputs['image_sizes'] = image_sizes + return inputs + +AutoConfig.register("llava_mistral", LlavaMistralConfig) +AutoModelForCausalLM.register(LlavaMistralConfig, LlavaMistralForCausalLM) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mpt.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mpt.py new file mode 100644 index 0000000000000000000000000000000000000000..02e5237ece031af23fcd76b5b4e0d9b0bc5f55cc --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/language_model/llava_mpt.py @@ -0,0 +1,97 @@ +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Optional, Tuple + +import torch + +from transformers import AutoConfig, AutoModelForCausalLM, \ + MptConfig, MptForCausalLM, MptModel +from llava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +class LlavaMptConfig(MptConfig): + model_type = "llava_mpt" + + +class LlavaMptModel(LlavaMetaModel, MptModel): + config_class = LlavaMptConfig + + def __init__(self, config: MptConfig): + config.hidden_size = config.d_model + super(LlavaMptModel, self).__init__(config) + + def embed_tokens(self, x): + return self.wte(x) + + +class LlavaMptForCausalLM(MptForCausalLM, LlavaMetaForCausalLM): + config_class = LlavaMptConfig + supports_gradient_checkpointing = True + + def __init__(self, config): + super(MptForCausalLM, self).__init__(config) + + self.transformer = LlavaMptModel(config) + self.lm_head = torch.nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_model(self): + return self.transformer + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, LlavaMptModel): + module.gradient_checkpointing = value + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + past_key_values: Optional[Tuple[Tuple[torch.Tensor, torch.Tensor], ...]] = None, + attention_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + images=None): + + input_ids, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images) + + return super().forward( + input_ids, + past_key_values=past_key_values, + attention_mask=attention_mask, + inputs_embeds=inputs_embeds, + labels=labels, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs): + images = kwargs.pop("images", None) + _inputs = super().prepare_inputs_for_generation( + input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs + ) + _inputs['images'] = images + return _inputs + + +AutoConfig.register("llava_mpt", LlavaMptConfig) +AutoModelForCausalLM.register(LlavaMptConfig, LlavaMptForCausalLM) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/llava_arch.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/llava_arch.py new file mode 100644 index 0000000000000000000000000000000000000000..7b299d3c416a0f5ffea3d03d7be5a32b77319533 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/llava_arch.py @@ -0,0 +1,368 @@ +# Copyright 2023 Haotian Liu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from abc import ABC, abstractmethod + +import torch +import torch.nn as nn + +from .multimodal_encoder.builder import build_vision_tower +from .multimodal_projector.builder import build_vision_projector + +from llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN + +from llava.mm_utils import get_anyres_image_grid_shape + + +class LlavaMetaModel: + + def __init__(self, config): + super(LlavaMetaModel, self).__init__(config) + + if hasattr(config, "mm_vision_tower"): + self.vision_tower = build_vision_tower(config, delay_load=True) + self.mm_projector = build_vision_projector(config) + + if 'unpad' in getattr(config, 'mm_patch_merge_type', ''): + self.image_newline = nn.Parameter( + torch.empty(config.hidden_size, dtype=self.dtype) + ) + + def get_vision_tower(self): + vision_tower = getattr(self, 'vision_tower', None) + if type(vision_tower) is list: + vision_tower = vision_tower[0] + return vision_tower + + def initialize_vision_modules(self, model_args, fsdp=None): + vision_tower = model_args.vision_tower + mm_vision_select_layer = model_args.mm_vision_select_layer + mm_vision_select_feature = model_args.mm_vision_select_feature + pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter + mm_patch_merge_type = model_args.mm_patch_merge_type + + self.config.mm_vision_tower = vision_tower + + if self.get_vision_tower() is None: + vision_tower = build_vision_tower(model_args) + + if fsdp is not None and len(fsdp) > 0: + self.vision_tower = [vision_tower] + else: + self.vision_tower = vision_tower + else: + if fsdp is not None and len(fsdp) > 0: + vision_tower = self.vision_tower[0] + else: + vision_tower = self.vision_tower + vision_tower.load_model() + + self.config.use_mm_proj = True + self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear') + self.config.mm_hidden_size = vision_tower.hidden_size + self.config.mm_vision_select_layer = mm_vision_select_layer + self.config.mm_vision_select_feature = mm_vision_select_feature + self.config.mm_patch_merge_type = mm_patch_merge_type + + if getattr(self, 'mm_projector', None) is None: + self.mm_projector = build_vision_projector(self.config) + + if 'unpad' in mm_patch_merge_type: + embed_std = 1 / torch.sqrt(torch.tensor(self.config.hidden_size, dtype=self.dtype)) + self.image_newline = nn.Parameter( + torch.randn(self.config.hidden_size, dtype=self.dtype) * embed_std + ) + else: + # In case it is frozen by LoRA + for p in self.mm_projector.parameters(): + p.requires_grad = True + + if pretrain_mm_mlp_adapter is not None: + mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu') + def get_w(weights, keyword): + return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k} + + self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector')) + + +def unpad_image(tensor, original_size): + """ + Unpads a PyTorch tensor of a padded and resized image. + + Args: + tensor (torch.Tensor): The image tensor, assumed to be in CxHxW format. + original_size (tuple): The original size of the image (height, width). + + Returns: + torch.Tensor: The unpadded image tensor. + """ + original_width, original_height = original_size + current_height, current_width = tensor.shape[1:] + + original_aspect_ratio = original_width / original_height + current_aspect_ratio = current_width / current_height + + if original_aspect_ratio > current_aspect_ratio: + scale_factor = current_width / original_width + new_height = int(original_height * scale_factor) + padding = (current_height - new_height) // 2 + unpadded_tensor = tensor[:, padding:current_height - padding, :] + else: + scale_factor = current_height / original_height + new_width = int(original_width * scale_factor) + padding = (current_width - new_width) // 2 + unpadded_tensor = tensor[:, :, padding:current_width - padding] + + return unpadded_tensor + + +class LlavaMetaForCausalLM(ABC): + + @abstractmethod + def get_model(self): + pass + + def get_vision_tower(self): + return self.get_model().get_vision_tower() + + def encode_images(self, images): + image_features = self.get_model().get_vision_tower()(images) + image_features = self.get_model().mm_projector(image_features) + return image_features + + def prepare_inputs_labels_for_multimodal( + self, input_ids, position_ids, attention_mask, past_key_values, labels, + images, image_sizes=None + ): + vision_tower = self.get_vision_tower() + if vision_tower is None or images is None or input_ids.shape[1] == 1: + return input_ids, position_ids, attention_mask, past_key_values, None, labels + + if type(images) is list or images.ndim == 5: + if type(images) is list: + images = [x.unsqueeze(0) if x.ndim == 3 else x for x in images] + concat_images = torch.cat([image for image in images], dim=0) + image_features = self.encode_images(concat_images) + split_sizes = [image.shape[0] for image in images] + image_features = torch.split(image_features, split_sizes, dim=0) + mm_patch_merge_type = getattr(self.config, 'mm_patch_merge_type', 'flat') + image_aspect_ratio = getattr(self.config, 'image_aspect_ratio', 'square') + if mm_patch_merge_type == 'flat': + image_features = [x.flatten(0, 1) for x in image_features] + elif mm_patch_merge_type.startswith('spatial'): + new_image_features = [] + for image_idx, image_feature in enumerate(image_features): + if image_feature.shape[0] > 1: + base_image_feature = image_feature[0] + image_feature = image_feature[1:] + height = width = self.get_vision_tower().num_patches_per_side + assert height * width == base_image_feature.shape[0] + if image_aspect_ratio == 'anyres': + num_patch_width, num_patch_height = get_anyres_image_grid_shape(image_sizes[image_idx], self.config.image_grid_pinpoints, self.get_vision_tower().config.image_size) + image_feature = image_feature.view(num_patch_height, num_patch_width, height, width, -1) + else: + raise NotImplementedError + if 'unpad' in mm_patch_merge_type: + image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous() + image_feature = image_feature.flatten(1, 2).flatten(2, 3) + image_feature = unpad_image(image_feature, image_sizes[image_idx]) + image_feature = torch.cat(( + image_feature, + self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device) + ), dim=-1) + image_feature = image_feature.flatten(1, 2).transpose(0, 1) + else: + image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous() + image_feature = image_feature.flatten(0, 3) + image_feature = torch.cat((base_image_feature, image_feature), dim=0) + else: + image_feature = image_feature[0] + if 'unpad' in mm_patch_merge_type: + image_feature = torch.cat(( + image_feature, + self.model.image_newline[None].to(image_feature.device) + ), dim=0) + new_image_features.append(image_feature) + image_features = new_image_features + else: + raise ValueError(f"Unexpected mm_patch_merge_type: {self.config.mm_patch_merge_type}") + else: + image_features = self.encode_images(images) + + # TODO: image start / end is not implemented here to support pretraining. + if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False): + raise NotImplementedError + + # Let's just add dummy tensors if they do not exist, + # it is a headache to deal with None all the time. + # But it is not ideal, and if you have a better idea, + # please open an issue / submit a PR, thanks. + _labels = labels + _position_ids = position_ids + _attention_mask = attention_mask + if attention_mask is None: + attention_mask = torch.ones_like(input_ids, dtype=torch.bool) + else: + attention_mask = attention_mask.bool() + if position_ids is None: + position_ids = torch.arange(0, input_ids.shape[1], dtype=torch.long, device=input_ids.device) + if labels is None: + labels = torch.full_like(input_ids, IGNORE_INDEX) + + # remove the padding using attention_mask -- FIXME + _input_ids = input_ids + input_ids = [cur_input_ids[cur_attention_mask] for cur_input_ids, cur_attention_mask in zip(input_ids, attention_mask)] + labels = [cur_labels[cur_attention_mask] for cur_labels, cur_attention_mask in zip(labels, attention_mask)] + + new_input_embeds = [] + new_labels = [] + cur_image_idx = 0 + for batch_idx, cur_input_ids in enumerate(input_ids): + num_images = (cur_input_ids == IMAGE_TOKEN_INDEX).sum() + if num_images == 0: + cur_image_features = image_features[cur_image_idx] + cur_input_embeds_1 = self.get_model().embed_tokens(cur_input_ids) + cur_input_embeds = torch.cat([cur_input_embeds_1, cur_image_features[0:0]], dim=0) + new_input_embeds.append(cur_input_embeds) + new_labels.append(labels[batch_idx]) + cur_image_idx += 1 + continue + + image_token_indices = [-1] + torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0].tolist() + [cur_input_ids.shape[0]] + cur_input_ids_noim = [] + cur_labels = labels[batch_idx] + cur_labels_noim = [] + for i in range(len(image_token_indices) - 1): + cur_input_ids_noim.append(cur_input_ids[image_token_indices[i]+1:image_token_indices[i+1]]) + cur_labels_noim.append(cur_labels[image_token_indices[i]+1:image_token_indices[i+1]]) + split_sizes = [x.shape[0] for x in cur_labels_noim] + cur_input_embeds = self.get_model().embed_tokens(torch.cat(cur_input_ids_noim)) + cur_input_embeds_no_im = torch.split(cur_input_embeds, split_sizes, dim=0) + cur_new_input_embeds = [] + cur_new_labels = [] + + for i in range(num_images + 1): + cur_new_input_embeds.append(cur_input_embeds_no_im[i]) + cur_new_labels.append(cur_labels_noim[i]) + if i < num_images: + cur_image_features = image_features[cur_image_idx] + cur_image_idx += 1 + cur_new_input_embeds.append(cur_image_features) + cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=cur_labels.device, dtype=cur_labels.dtype)) + + cur_new_input_embeds = [x.to(self.device) for x in cur_new_input_embeds] + + cur_new_input_embeds = torch.cat(cur_new_input_embeds) + cur_new_labels = torch.cat(cur_new_labels) + + new_input_embeds.append(cur_new_input_embeds) + new_labels.append(cur_new_labels) + + # Truncate sequences to max length as image embeddings can make the sequence longer + tokenizer_model_max_length = getattr(self.config, 'tokenizer_model_max_length', None) + if tokenizer_model_max_length is not None: + new_input_embeds = [x[:tokenizer_model_max_length] for x in new_input_embeds] + new_labels = [x[:tokenizer_model_max_length] for x in new_labels] + + # Combine them + max_len = max(x.shape[0] for x in new_input_embeds) + batch_size = len(new_input_embeds) + + new_input_embeds_padded = [] + new_labels_padded = torch.full((batch_size, max_len), IGNORE_INDEX, dtype=new_labels[0].dtype, device=new_labels[0].device) + attention_mask = torch.zeros((batch_size, max_len), dtype=attention_mask.dtype, device=attention_mask.device) + position_ids = torch.zeros((batch_size, max_len), dtype=position_ids.dtype, device=position_ids.device) + + for i, (cur_new_embed, cur_new_labels) in enumerate(zip(new_input_embeds, new_labels)): + cur_len = cur_new_embed.shape[0] + if getattr(self.config, 'tokenizer_padding_side', 'right') == "left": + new_input_embeds_padded.append(torch.cat(( + torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device), + cur_new_embed + ), dim=0)) + if cur_len > 0: + new_labels_padded[i, -cur_len:] = cur_new_labels + attention_mask[i, -cur_len:] = True + position_ids[i, -cur_len:] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) + else: + new_input_embeds_padded.append(torch.cat(( + cur_new_embed, + torch.zeros((max_len - cur_len, cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device) + ), dim=0)) + if cur_len > 0: + new_labels_padded[i, :cur_len] = cur_new_labels + attention_mask[i, :cur_len] = True + position_ids[i, :cur_len] = torch.arange(0, cur_len, dtype=position_ids.dtype, device=position_ids.device) + + new_input_embeds = torch.stack(new_input_embeds_padded, dim=0) + + if _labels is None: + new_labels = None + else: + new_labels = new_labels_padded + + if _attention_mask is None: + attention_mask = None + else: + attention_mask = attention_mask.to(dtype=_attention_mask.dtype) + + if _position_ids is None: + position_ids = None + + return None, position_ids, attention_mask, past_key_values, new_input_embeds, new_labels + + def initialize_vision_tokenizer(self, model_args, tokenizer): + if model_args.mm_use_im_patch_token: + tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True) + self.resize_token_embeddings(len(tokenizer)) + + if model_args.mm_use_im_start_end: + num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True) + self.resize_token_embeddings(len(tokenizer)) + + if num_new_tokens > 0: + input_embeddings = self.get_input_embeddings().weight.data + output_embeddings = self.get_output_embeddings().weight.data + + input_embeddings_avg = input_embeddings[:-num_new_tokens].mean( + dim=0, keepdim=True) + output_embeddings_avg = output_embeddings[:-num_new_tokens].mean( + dim=0, keepdim=True) + + input_embeddings[-num_new_tokens:] = input_embeddings_avg + output_embeddings[-num_new_tokens:] = output_embeddings_avg + + if model_args.tune_mm_mlp_adapter: + for p in self.get_input_embeddings().parameters(): + p.requires_grad = True + for p in self.get_output_embeddings().parameters(): + p.requires_grad = False + + if model_args.pretrain_mm_mlp_adapter: + mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu') + embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight'] + assert num_new_tokens == 2 + if input_embeddings.shape == embed_tokens_weight.shape: + input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:] + elif embed_tokens_weight.shape[0] == num_new_tokens: + input_embeddings[-num_new_tokens:] = embed_tokens_weight + else: + raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.") + elif model_args.mm_use_im_patch_token: + if model_args.tune_mm_mlp_adapter: + for p in self.get_input_embeddings().parameters(): + p.requires_grad = False + for p in self.get_output_embeddings().parameters(): + p.requires_grad = False diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/make_delta.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/make_delta.py new file mode 100644 index 0000000000000000000000000000000000000000..4ae55d59c2c8bab80299272314a41bbeb959d8ed --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/make_delta.py @@ -0,0 +1,52 @@ +""" +Usage: +python3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/model_weights/llava-7b --delta ~/model_weights/llava-7b-delta --hub-repo-id liuhaotian/llava-7b-delta +""" +import argparse + +import torch +from tqdm import tqdm +from transformers import AutoTokenizer, AutoModelForCausalLM +from llava.model.utils import auto_upgrade + + +def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id): + print("Loading base model") + base = AutoModelForCausalLM.from_pretrained( + base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + + print("Loading target model") + auto_upgrade(target_model_path) + target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) + + print("Calculating delta") + for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"): + if name not in base.state_dict(): + assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model' + continue + if param.data.shape == base.state_dict()[name].shape: + param.data -= base.state_dict()[name] + else: + assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}' + bparam = base.state_dict()[name] + param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam + + print("Saving delta") + if hub_repo_id: + kwargs = {"push_to_hub": True, "repo_id": hub_repo_id} + else: + kwargs = {} + target.save_pretrained(delta_path, **kwargs) + target_tokenizer = AutoTokenizer.from_pretrained(target_model_path) + target_tokenizer.save_pretrained(delta_path, **kwargs) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--base-model-path", type=str, required=True) + parser.add_argument("--target-model-path", type=str, required=True) + parser.add_argument("--delta-path", type=str, required=True) + parser.add_argument("--hub-repo-id", type=str, default=None) + args = parser.parse_args() + + make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/builder.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..e89507c49df413945453959d48b51a71b9031ef7 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/builder.py @@ -0,0 +1,11 @@ +import os +from .clip_encoder import CLIPVisionTower + + +def build_vision_tower(vision_tower_cfg, **kwargs): + vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None)) + is_absolute_path_exists = os.path.exists(vision_tower) + if is_absolute_path_exists or vision_tower.startswith("openai") or vision_tower.startswith("laion") or "ShareGPT4V" in vision_tower: + return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs) + + raise ValueError(f'Unknown vision tower: {vision_tower}') diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/clip_encoder.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/clip_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..89ce2d9e084a2b324dc5e19fb2d2d889f3a60602 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_encoder/clip_encoder.py @@ -0,0 +1,88 @@ +import torch +import torch.nn as nn + +from transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig + + +class CLIPVisionTower(nn.Module): + def __init__(self, vision_tower, args, delay_load=False): + super().__init__() + + self.is_loaded = False + + self.vision_tower_name = vision_tower + self.select_layer = args.mm_vision_select_layer + self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch') + + if not delay_load: + self.load_model() + elif getattr(args, 'unfreeze_mm_vision_tower', False): + self.load_model() + else: + self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name) + + def load_model(self, device_map=None): + if self.is_loaded: + print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name)) + return + + self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name) + self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name, device_map=device_map) + self.vision_tower.requires_grad_(False) + + self.is_loaded = True + + def feature_select(self, image_forward_outs): + image_features = image_forward_outs.hidden_states[self.select_layer] + if self.select_feature == 'patch': + image_features = image_features[:, 1:] + elif self.select_feature == 'cls_patch': + image_features = image_features + else: + raise ValueError(f'Unexpected select feature: {self.select_feature}') + return image_features + + @torch.no_grad() + def forward(self, images): + if type(images) is list: + image_features = [] + for image in images: + image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True) + image_feature = self.feature_select(image_forward_out).to(image.dtype) + image_features.append(image_feature) + else: + image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True) + image_features = self.feature_select(image_forward_outs).to(images.dtype) + + return image_features + + @property + def dummy_feature(self): + return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype) + + @property + def dtype(self): + return self.vision_tower.dtype + + @property + def device(self): + return self.vision_tower.device + + @property + def config(self): + if self.is_loaded: + return self.vision_tower.config + else: + return self.cfg_only + + @property + def hidden_size(self): + return self.config.hidden_size + + @property + def num_patches_per_side(self): + return self.config.image_size // self.config.patch_size + + @property + def num_patches(self): + return (self.config.image_size // self.config.patch_size) ** 2 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_projector/builder.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_projector/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..31cd4f48e6055cd6d00a162af30b1c8139e26b57 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/multimodal_projector/builder.py @@ -0,0 +1,51 @@ +import torch +import torch.nn as nn +import re + + +class IdentityMap(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x, *args, **kwargs): + return x + + @property + def config(self): + return {"mm_projector_type": 'identity'} + + +class SimpleResBlock(nn.Module): + def __init__(self, channels): + super().__init__() + self.pre_norm = nn.LayerNorm(channels) + + self.proj = nn.Sequential( + nn.Linear(channels, channels), + nn.GELU(), + nn.Linear(channels, channels) + ) + def forward(self, x): + x = self.pre_norm(x) + return x + self.proj(x) + + +def build_vision_projector(config, delay_load=False, **kwargs): + projector_type = getattr(config, 'mm_projector_type', 'linear') + + if projector_type == 'linear': + return nn.Linear(config.mm_hidden_size, config.hidden_size) + + mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type) + if mlp_gelu_match: + mlp_depth = int(mlp_gelu_match.group(1)) + modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)] + for _ in range(1, mlp_depth): + modules.append(nn.GELU()) + modules.append(nn.Linear(config.hidden_size, config.hidden_size)) + return nn.Sequential(*modules) + + if projector_type == 'identity': + return IdentityMap() + + raise ValueError(f'Unknown projector type: {projector_type}') diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/utils.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2563f89c6cedf5e73508afec8f9979105df9b745 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/model/utils.py @@ -0,0 +1,20 @@ +from transformers import AutoConfig + + +def auto_upgrade(config): + cfg = AutoConfig.from_pretrained(config) + if 'llava' in config and 'llava' not in cfg.model_type: + assert cfg.model_type == 'llama' + print("You are using newer LLaVA code base, while the checkpoint of v0 is from older code base.") + print("You must upgrade the checkpoint to the new code base (this can be done automatically).") + confirm = input("Please confirm that you want to upgrade the checkpoint. [Y/N]") + if confirm.lower() in ["y", "yes"]: + print("Upgrading checkpoint...") + assert len(cfg.architectures) == 1 + setattr(cfg.__class__, "model_type", "llava") + cfg.architectures[0] = 'LlavaLlamaForCausalLM' + cfg.save_pretrained(config) + print("Checkpoint upgraded.") + else: + print("Checkpoint upgrade aborted.") + exit(1) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/cli.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..f4a52e9eb1a5fd711d2dce0afd452f3b425755f0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/cli.py @@ -0,0 +1,141 @@ +import argparse +import torch + +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from llava.conversation import conv_templates, SeparatorStyle +from llava.model.builder import load_pretrained_model +from llava.utils import disable_torch_init +from llava.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path + +from PIL import Image + +import requests +from PIL import Image +from io import BytesIO +from transformers import TextStreamer + + +def load_image(image_file): + if image_file.startswith('http://') or image_file.startswith('https://'): + response = requests.get(image_file) + image = Image.open(BytesIO(response.content)).convert('RGB') + else: + image = Image.open(image_file).convert('RGB') + return image + + +def main(args): + # Model + disable_torch_init() + + model_name = get_model_name_from_path(args.model_path) + tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, + args.load_8bit, args.load_4bit, + device=args.device) + + if "llama-2" in model_name.lower(): + conv_mode = "llava_llama_2" + elif "mistral" in model_name.lower(): + conv_mode = "mistral_instruct" + elif "v1.6-34b" in model_name.lower(): + conv_mode = "chatml_direct" + elif "v1" in model_name.lower(): + conv_mode = "llava_v1" + elif "mpt" in model_name.lower(): + conv_mode = "mpt" + else: + conv_mode = "llava_v0" + + if args.conv_mode is not None and conv_mode != args.conv_mode: + print( + '[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, + args.conv_mode, + args.conv_mode)) + else: + args.conv_mode = conv_mode + + conv = conv_templates[args.conv_mode].copy() + if "mpt" in model_name.lower(): + roles = ('user', 'assistant') + else: + roles = conv.roles + + if args.image_file is not None: + image = load_image(args.image_file) + image_size = image.size + # Similar operation in model_worker.py + image_tensor = process_images([image], image_processor, model.config) + if type(image_tensor) is list: + image_tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor] + else: + image_tensor = image_tensor.to(model.device, dtype=torch.float16) + else: + image = True + image_size = (1024, 1024) + image_tensor = torch.zeros(1, 5, 3, 336, 336) + image_tensor = image_tensor.to(model.device, dtype=torch.float16) + + + while True: + try: + inp = input(f"{roles[0]}: ") + except EOFError: + inp = "" + if not inp: + print("exit...") + break + + print(f"{roles[1]}: ", end="") + + if image is not None: + # first message + if model.config.mm_use_im_start_end: + inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + inp + else: + inp = inp.replace(DEFAULT_IMAGE_TOKEN, '').strip() + inp = DEFAULT_IMAGE_TOKEN + '\n' + inp + inp = inp.strip() + image = None + + conv.append_message(conv.roles[0], inp) + conv.append_message(conv.roles[1], None) + prompt = conv.get_prompt() + + input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to( + model.device) + stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2 + keywords = [stop_str] + streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True) + + with torch.inference_mode(): + output_ids = model.generate( + input_ids, + images=image_tensor, + image_sizes=[image_size], + do_sample=True if args.temperature > 0 else False, + temperature=args.temperature, + max_new_tokens=args.max_new_tokens, + streamer=streamer, + use_cache=True) + + outputs = tokenizer.decode(output_ids[0]).strip() + conv.messages[-1][-1] = outputs + + if args.debug: + print("\n", {"prompt": prompt, "outputs": outputs}, "\n") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--model-path", type=str, default="facebook/opt-350m") + parser.add_argument("--model-base", type=str, default=None) + parser.add_argument("--image-file", type=str, default=None) + parser.add_argument("--device", type=str, default="cuda") + parser.add_argument("--conv-mode", type=str, default=None) + parser.add_argument("--temperature", type=float, default=0.2) + parser.add_argument("--max-new-tokens", type=int, default=512) + parser.add_argument("--load-8bit", action="store_true") + parser.add_argument("--load-4bit", action="store_true") + parser.add_argument("--debug", action="store_true") + args = parser.parse_args() + main(args) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/controller.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/controller.py new file mode 100644 index 0000000000000000000000000000000000000000..d4bf1b4c47ccdb1401b18f8397868ec016d1c43a --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/controller.py @@ -0,0 +1,298 @@ +""" +A controller manages distributed workers. +It sends worker addresses to clients. +""" +import argparse +import asyncio +import dataclasses +from enum import Enum, auto +import json +import logging +import time +from typing import List, Union +import threading + +from fastapi import FastAPI, Request +from fastapi.responses import StreamingResponse +import numpy as np +import requests +import uvicorn + +from llava.constants import CONTROLLER_HEART_BEAT_EXPIRATION +from llava.utils import build_logger, server_error_msg + + +logger = build_logger("controller", "controller.log") + + +class DispatchMethod(Enum): + LOTTERY = auto() + SHORTEST_QUEUE = auto() + + @classmethod + def from_str(cls, name): + if name == "lottery": + return cls.LOTTERY + elif name == "shortest_queue": + return cls.SHORTEST_QUEUE + else: + raise ValueError(f"Invalid dispatch method") + + +@dataclasses.dataclass +class WorkerInfo: + model_names: List[str] + speed: int + queue_length: int + check_heart_beat: bool + last_heart_beat: str + + +def heart_beat_controller(controller): + while True: + time.sleep(CONTROLLER_HEART_BEAT_EXPIRATION) + controller.remove_stable_workers_by_expiration() + + +class Controller: + def __init__(self, dispatch_method: str): + # Dict[str -> WorkerInfo] + self.worker_info = {} + self.dispatch_method = DispatchMethod.from_str(dispatch_method) + + self.heart_beat_thread = threading.Thread( + target=heart_beat_controller, args=(self,), daemon=True) + self.heart_beat_thread.start() + + logger.info("Init controller") + + def register_worker(self, worker_name: str, check_heart_beat: bool, + worker_status: dict): + if worker_name not in self.worker_info: + logger.info(f"Register a new worker: {worker_name}") + else: + logger.info(f"Register an existing worker: {worker_name}") + + if not worker_status: + worker_status = self.get_worker_status(worker_name) + if not worker_status: + return False + + self.worker_info[worker_name] = WorkerInfo( + worker_status["model_names"], worker_status["speed"], worker_status["queue_length"], + check_heart_beat, time.time()) + + logger.info(f"Register done: {worker_name}, {worker_status}") + return True + + def get_worker_status(self, worker_name: str): + try: + r = requests.post(worker_name + "/worker_get_status", timeout=5) + except requests.exceptions.RequestException as e: + logger.error(f"Get status fails: {worker_name}, {e}") + return None + + if r.status_code != 200: + logger.error(f"Get status fails: {worker_name}, {r}") + return None + + return r.json() + + def remove_worker(self, worker_name: str): + del self.worker_info[worker_name] + + def refresh_all_workers(self): + old_info = dict(self.worker_info) + self.worker_info = {} + + for w_name, w_info in old_info.items(): + if not self.register_worker(w_name, w_info.check_heart_beat, None): + logger.info(f"Remove stale worker: {w_name}") + + def list_models(self): + model_names = set() + + for w_name, w_info in self.worker_info.items(): + model_names.update(w_info.model_names) + + return list(model_names) + + def get_worker_address(self, model_name: str): + if self.dispatch_method == DispatchMethod.LOTTERY: + worker_names = [] + worker_speeds = [] + for w_name, w_info in self.worker_info.items(): + if model_name in w_info.model_names: + worker_names.append(w_name) + worker_speeds.append(w_info.speed) + worker_speeds = np.array(worker_speeds, dtype=np.float32) + norm = np.sum(worker_speeds) + if norm < 1e-4: + return "" + worker_speeds = worker_speeds / norm + if True: # Directly return address + pt = np.random.choice(np.arange(len(worker_names)), + p=worker_speeds) + worker_name = worker_names[pt] + return worker_name + + # Check status before returning + while True: + pt = np.random.choice(np.arange(len(worker_names)), + p=worker_speeds) + worker_name = worker_names[pt] + + if self.get_worker_status(worker_name): + break + else: + self.remove_worker(worker_name) + worker_speeds[pt] = 0 + norm = np.sum(worker_speeds) + if norm < 1e-4: + return "" + worker_speeds = worker_speeds / norm + continue + return worker_name + elif self.dispatch_method == DispatchMethod.SHORTEST_QUEUE: + worker_names = [] + worker_qlen = [] + for w_name, w_info in self.worker_info.items(): + if model_name in w_info.model_names: + worker_names.append(w_name) + worker_qlen.append(w_info.queue_length / w_info.speed) + if len(worker_names) == 0: + return "" + min_index = np.argmin(worker_qlen) + w_name = worker_names[min_index] + self.worker_info[w_name].queue_length += 1 + logger.info(f"names: {worker_names}, queue_lens: {worker_qlen}, ret: {w_name}") + return w_name + else: + raise ValueError(f"Invalid dispatch method: {self.dispatch_method}") + + def receive_heart_beat(self, worker_name: str, queue_length: int): + if worker_name not in self.worker_info: + logger.info(f"Receive unknown heart beat. {worker_name}") + return False + + self.worker_info[worker_name].queue_length = queue_length + self.worker_info[worker_name].last_heart_beat = time.time() + logger.info(f"Receive heart beat. {worker_name}") + return True + + def remove_stable_workers_by_expiration(self): + expire = time.time() - CONTROLLER_HEART_BEAT_EXPIRATION + to_delete = [] + for worker_name, w_info in self.worker_info.items(): + if w_info.check_heart_beat and w_info.last_heart_beat < expire: + to_delete.append(worker_name) + + for worker_name in to_delete: + self.remove_worker(worker_name) + + def worker_api_generate_stream(self, params): + worker_addr = self.get_worker_address(params["model"]) + if not worker_addr: + logger.info(f"no worker: {params['model']}") + ret = { + "text": server_error_msg, + "error_code": 2, + } + yield json.dumps(ret).encode() + b"\0" + + try: + response = requests.post(worker_addr + "/worker_generate_stream", + json=params, stream=True, timeout=5) + for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): + if chunk: + yield chunk + b"\0" + except requests.exceptions.RequestException as e: + logger.info(f"worker timeout: {worker_addr}") + ret = { + "text": server_error_msg, + "error_code": 3, + } + yield json.dumps(ret).encode() + b"\0" + + + # Let the controller act as a worker to achieve hierarchical + # management. This can be used to connect isolated sub networks. + def worker_api_get_status(self): + model_names = set() + speed = 0 + queue_length = 0 + + for w_name in self.worker_info: + worker_status = self.get_worker_status(w_name) + if worker_status is not None: + model_names.update(worker_status["model_names"]) + speed += worker_status["speed"] + queue_length += worker_status["queue_length"] + + return { + "model_names": list(model_names), + "speed": speed, + "queue_length": queue_length, + } + + +app = FastAPI() + + +@app.post("/register_worker") +async def register_worker(request: Request): + data = await request.json() + controller.register_worker( + data["worker_name"], data["check_heart_beat"], + data.get("worker_status", None)) + + +@app.post("/refresh_all_workers") +async def refresh_all_workers(): + models = controller.refresh_all_workers() + + +@app.post("/list_models") +async def list_models(): + models = controller.list_models() + return {"models": models} + + +@app.post("/get_worker_address") +async def get_worker_address(request: Request): + data = await request.json() + addr = controller.get_worker_address(data["model"]) + return {"address": addr} + + +@app.post("/receive_heart_beat") +async def receive_heart_beat(request: Request): + data = await request.json() + exist = controller.receive_heart_beat( + data["worker_name"], data["queue_length"]) + return {"exist": exist} + + +@app.post("/worker_generate_stream") +async def worker_api_generate_stream(request: Request): + params = await request.json() + generator = controller.worker_api_generate_stream(params) + return StreamingResponse(generator) + + +@app.post("/worker_get_status") +async def worker_api_get_status(request: Request): + return controller.worker_api_get_status() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=21001) + parser.add_argument("--dispatch-method", type=str, choices=[ + "lottery", "shortest_queue"], default="shortest_queue") + args = parser.parse_args() + logger.info(f"args: {args}") + + controller = Controller(args.dispatch_method) + uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/gradio_web_server.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/gradio_web_server.py new file mode 100644 index 0000000000000000000000000000000000000000..1c016cc0261c957c850e3a3bb1dbbcbed2976efc --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/gradio_web_server.py @@ -0,0 +1,436 @@ +import argparse +import datetime +import json +import os +import time + +import gradio as gr +import requests + +from llava.conversation import (default_conversation, conv_templates, + SeparatorStyle) +from llava.constants import LOGDIR +from llava.utils import (build_logger, server_error_msg, + violates_moderation, moderation_msg) +import hashlib + + +logger = build_logger("gradio_web_server", "gradio_web_server.log") + +headers = {"User-Agent": "LLaVA Client"} + +no_change_btn = gr.Button() +enable_btn = gr.Button(interactive=True) +disable_btn = gr.Button(interactive=False) + +priority = { + "vicuna-13b": "aaaaaaa", + "koala-13b": "aaaaaab", +} + + +def get_conv_log_filename(): + t = datetime.datetime.now() + name = os.path.join(LOGDIR, f"{t.year}-{t.month:02d}-{t.day:02d}-conv.json") + return name + + +def get_model_list(): + ret = requests.post(args.controller_url + "/refresh_all_workers") + assert ret.status_code == 200 + ret = requests.post(args.controller_url + "/list_models") + models = ret.json()["models"] + models.sort(key=lambda x: priority.get(x, x)) + logger.info(f"Models: {models}") + return models + + +get_window_url_params = """ +function() { + const params = new URLSearchParams(window.location.search); + url_params = Object.fromEntries(params); + console.log(url_params); + return url_params; + } +""" + + +def load_demo(url_params, request: gr.Request): + logger.info(f"load_demo. ip: {request.client.host}. params: {url_params}") + + dropdown_update = gr.Dropdown(visible=True) + if "model" in url_params: + model = url_params["model"] + if model in models: + dropdown_update = gr.Dropdown(value=model, visible=True) + + state = default_conversation.copy() + return state, dropdown_update + + +def load_demo_refresh_model_list(request: gr.Request): + logger.info(f"load_demo. ip: {request.client.host}") + models = get_model_list() + state = default_conversation.copy() + dropdown_update = gr.Dropdown( + choices=models, + value=models[0] if len(models) > 0 else "", + ) + + return state, dropdown_update + + +def vote_last_response(state, vote_type, model_selector, request: gr.Request): + with open(get_conv_log_filename(), "a") as fout: + data = { + "tstamp": round(time.time(), 4), + "type": vote_type, + "model": model_selector, + "state": state.dict(), + "ip": request.client.host, + } + fout.write(json.dumps(data) + "\n") + + +def upvote_last_response(state, model_selector, request: gr.Request): + logger.info(f"upvote. ip: {request.client.host}") + vote_last_response(state, "upvote", model_selector, request) + return ("",) + (disable_btn,) * 3 + + +def downvote_last_response(state, model_selector, request: gr.Request): + logger.info(f"downvote. ip: {request.client.host}") + vote_last_response(state, "downvote", model_selector, request) + return ("",) + (disable_btn,) * 3 + + +def flag_last_response(state, model_selector, request: gr.Request): + logger.info(f"flag. ip: {request.client.host}") + vote_last_response(state, "flag", model_selector, request) + return ("",) + (disable_btn,) * 3 + + +def regenerate(state, image_process_mode, request: gr.Request): + logger.info(f"regenerate. ip: {request.client.host}") + state.messages[-1][-1] = None + prev_human_msg = state.messages[-2] + if type(prev_human_msg[1]) in (tuple, list): + prev_human_msg[1] = (*prev_human_msg[1][:2], image_process_mode) + state.skip_next = False + return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5 + + +def clear_history(request: gr.Request): + logger.info(f"clear_history. ip: {request.client.host}") + state = default_conversation.copy() + return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5 + + +def add_text(state, text, image, image_process_mode, request: gr.Request): + logger.info(f"add_text. ip: {request.client.host}. len: {len(text)}") + if len(text) <= 0 and image is None: + state.skip_next = True + return (state, state.to_gradio_chatbot(), "", None) + (no_change_btn,) * 5 + if args.moderate: + flagged = violates_moderation(text) + if flagged: + state.skip_next = True + return (state, state.to_gradio_chatbot(), moderation_msg, None) + ( + no_change_btn,) * 5 + + text = text[:1536] # Hard cut-off + if image is not None: + text = text[:1200] # Hard cut-off for images + if '' not in text: + # text = '' + text + text = text + '\n' + text = (text, image, image_process_mode) + state = default_conversation.copy() + state.append_message(state.roles[0], text) + state.append_message(state.roles[1], None) + state.skip_next = False + return (state, state.to_gradio_chatbot(), "", None) + (disable_btn,) * 5 + + +def http_bot(state, model_selector, temperature, top_p, max_new_tokens, request: gr.Request): + logger.info(f"http_bot. ip: {request.client.host}") + start_tstamp = time.time() + model_name = model_selector + + if state.skip_next: + # This generate call is skipped due to invalid inputs + yield (state, state.to_gradio_chatbot()) + (no_change_btn,) * 5 + return + + if len(state.messages) == state.offset + 2: + # First round of conversation + if "llava" in model_name.lower(): + if 'llama-2' in model_name.lower(): + template_name = "llava_llama_2" + elif "mistral" in model_name.lower() or "mixtral" in model_name.lower(): + if 'orca' in model_name.lower(): + template_name = "mistral_orca" + elif 'hermes' in model_name.lower(): + template_name = "chatml_direct" + else: + template_name = "mistral_instruct" + elif 'llava-v1.6-34b' in model_name.lower(): + template_name = "chatml_direct" + elif "v1" in model_name.lower(): + if 'mmtag' in model_name.lower(): + template_name = "v1_mmtag" + elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower(): + template_name = "v1_mmtag" + else: + template_name = "llava_v1" + elif "mpt" in model_name.lower(): + template_name = "mpt" + else: + if 'mmtag' in model_name.lower(): + template_name = "v0_mmtag" + elif 'plain' in model_name.lower() and 'finetune' not in model_name.lower(): + template_name = "v0_mmtag" + else: + template_name = "llava_v0" + elif "mpt" in model_name: + template_name = "mpt_text" + elif "llama-2" in model_name: + template_name = "llama_2" + else: + template_name = "vicuna_v1" + new_state = conv_templates[template_name].copy() + new_state.append_message(new_state.roles[0], state.messages[-2][1]) + new_state.append_message(new_state.roles[1], None) + state = new_state + + # Query worker address + controller_url = args.controller_url + ret = requests.post(controller_url + "/get_worker_address", + json={"model": model_name}) + worker_addr = ret.json()["address"] + logger.info(f"model_name: {model_name}, worker_addr: {worker_addr}") + + # No available worker + if worker_addr == "": + state.messages[-1][-1] = server_error_msg + yield (state, state.to_gradio_chatbot(), disable_btn, disable_btn, disable_btn, enable_btn, enable_btn) + return + + # Construct prompt + prompt = state.get_prompt() + + all_images = state.get_images(return_pil=True) + all_image_hash = [hashlib.md5(image.tobytes()).hexdigest() for image in all_images] + for image, hash in zip(all_images, all_image_hash): + t = datetime.datetime.now() + filename = os.path.join(LOGDIR, "serve_images", f"{t.year}-{t.month:02d}-{t.day:02d}", f"{hash}.jpg") + if not os.path.isfile(filename): + os.makedirs(os.path.dirname(filename), exist_ok=True) + image.save(filename) + + # Make requests + pload = { + "model": model_name, + "prompt": prompt, + "temperature": float(temperature), + "top_p": float(top_p), + "max_new_tokens": min(int(max_new_tokens), 1536), + "stop": state.sep if state.sep_style in [SeparatorStyle.SINGLE, SeparatorStyle.MPT] else state.sep2, + "images": f'List of {len(state.get_images())} images: {all_image_hash}', + } + logger.info(f"==== request ====\n{pload}") + + pload['images'] = state.get_images() + + state.messages[-1][-1] = "▌" + yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5 + + try: + # Stream output + response = requests.post(worker_addr + "/worker_generate_stream", + headers=headers, json=pload, stream=True, timeout=10) + for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): + if chunk: + data = json.loads(chunk.decode()) + if data["error_code"] == 0: + output = data["text"][len(prompt):].strip() + state.messages[-1][-1] = output + "▌" + yield (state, state.to_gradio_chatbot()) + (disable_btn,) * 5 + else: + output = data["text"] + f" (error_code: {data['error_code']})" + state.messages[-1][-1] = output + yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn) + return + time.sleep(0.03) + except requests.exceptions.RequestException as e: + state.messages[-1][-1] = server_error_msg + yield (state, state.to_gradio_chatbot()) + (disable_btn, disable_btn, disable_btn, enable_btn, enable_btn) + return + + state.messages[-1][-1] = state.messages[-1][-1][:-1] + yield (state, state.to_gradio_chatbot()) + (enable_btn,) * 5 + + finish_tstamp = time.time() + logger.info(f"{output}") + + with open(get_conv_log_filename(), "a") as fout: + data = { + "tstamp": round(finish_tstamp, 4), + "type": "chat", + "model": model_name, + "start": round(start_tstamp, 4), + "finish": round(finish_tstamp, 4), + "state": state.dict(), + "images": all_image_hash, + "ip": request.client.host, + } + fout.write(json.dumps(data) + "\n") + + +block_css = """ + +#buttons button { + min-width: min(120px,100%); +} + +""" + +def build_demo(embed_mode, cur_dir=None, concurrency_count=10): + textbox = gr.Textbox(show_label=False, placeholder="Enter text and press ENTER", container=False) + with gr.Blocks(title="Hunyuan", theme=gr.themes.Default(), css=block_css) as demo: + state = gr.State() + + + with gr.Row(): + with gr.Column(scale=3): + with gr.Row(elem_id="model_selector_row"): + model_selector = gr.Dropdown( + choices=models, + value=models[0] if len(models) > 0 else "", + interactive=True, + show_label=False, + container=False) + + imagebox = gr.Image(type="pil") + image_process_mode = gr.Radio( + ["Crop", "Resize", "Pad", "Default"], + value="Default", + label="Preprocess for non-square image", visible=False) + + if cur_dir is None: + cur_dir = os.path.dirname(os.path.abspath(__file__)) + + with gr.Accordion("Parameters", open=False) as parameter_row: + temperature = gr.Slider(minimum=0.0, maximum=1.0, value=0.2, step=0.1, interactive=True, label="Temperature",) + top_p = gr.Slider(minimum=0.0, maximum=1.0, value=0.7, step=0.1, interactive=True, label="Top P",) + max_output_tokens = gr.Slider(minimum=0, maximum=1024, value=512, step=64, interactive=True, label="Max output tokens",) + + with gr.Column(scale=8): + chatbot = gr.Chatbot( + elem_id="chatbot", + label="Hunyuan", + height=650, + layout="panel", + ) + with gr.Row(): + with gr.Column(scale=8): + textbox.render() + with gr.Column(scale=1, min_width=50): + submit_btn = gr.Button(value="Send", variant="primary") + with gr.Row(elem_id="buttons") as button_row: + regenerate_btn = gr.Button(value="🔄 Regenerate", interactive=False) + clear_btn = gr.Button(value="🗑️ Clear", interactive=False) + + + url_params = gr.JSON(visible=False) + + + btn_list = [regenerate_btn, clear_btn] + + + regenerate_btn.click( + regenerate, + [state, image_process_mode], + [state, chatbot, textbox, imagebox] + btn_list + ).then( + http_bot, + [state, model_selector, temperature, top_p, max_output_tokens], + [state, chatbot] + btn_list, + # concurrency_limit=concurrency_count + ) + + clear_btn.click( + clear_history, + None, + [state, chatbot, textbox, imagebox] + btn_list, + queue=False + ) + + textbox.submit( + add_text, + [state, textbox, imagebox, image_process_mode], + [state, chatbot, textbox, imagebox] + btn_list, + queue=False + ).then( + http_bot, + [state, model_selector, temperature, top_p, max_output_tokens], + [state, chatbot] + btn_list, + + ) + + submit_btn.click( + add_text, + [state, textbox, imagebox, image_process_mode], + [state, chatbot, textbox, imagebox] + btn_list + ).then( + http_bot, + [state, model_selector, temperature, top_p, max_output_tokens], + [state, chatbot] + btn_list, + + ) + + if args.model_list_mode == "once": + demo.load( + load_demo, + [url_params], + [state, model_selector], + js=get_window_url_params + ) + elif args.model_list_mode == "reload": + demo.load( + load_demo_refresh_model_list, + None, + [state, model_selector], + queue=False + ) + else: + raise ValueError(f"Unknown model list mode: {args.model_list_mode}") + + return demo + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="0.0.0.0") + parser.add_argument("--port", type=int) + parser.add_argument("--controller-url", type=str, default="http://localhost:21001") + parser.add_argument("--concurrency-count", type=int, default=16) + parser.add_argument("--model-list-mode", type=str, default="once", + choices=["once", "reload"]) + parser.add_argument("--share", action="store_true") + parser.add_argument("--moderate", action="store_true") + parser.add_argument("--embed", action="store_true") + args = parser.parse_args() + logger.info(f"args: {args}") + + models = get_model_list() + + logger.info(args) + demo = build_demo(args.embed, concurrency_count=args.concurrency_count) + demo.queue( + api_open=False + ).launch( + server_name=args.host, + server_port=args.port, + share=args.share + ) diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/model_worker.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/model_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..9144329893c51f402ff2e2f65d9fb7baf177bd52 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/model_worker.py @@ -0,0 +1,288 @@ +""" +A model worker executes the model. +""" +import argparse +import asyncio +import json +import time +import threading +import uuid + +from fastapi import FastAPI, Request, BackgroundTasks +from fastapi.responses import StreamingResponse +import requests +import torch +import uvicorn +from functools import partial + +from llava.constants import WORKER_HEART_BEAT_INTERVAL +from llava.utils import (build_logger, server_error_msg, + pretty_print_semaphore) +from llava.model.builder import load_pretrained_model +from llava.mm_utils import process_images, load_image_from_base64, tokenizer_image_token +from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN +from transformers import TextIteratorStreamer +from threading import Thread + + +GB = 1 << 30 + +worker_id = str(uuid.uuid4())[:6] +logger = build_logger("model_worker", f"model_worker_{worker_id}.log") +global_counter = 0 + +model_semaphore = None + + +def heart_beat_worker(controller): + + while True: + time.sleep(WORKER_HEART_BEAT_INTERVAL) + controller.send_heart_beat() + + +class ModelWorker: + def __init__(self, controller_addr, worker_addr, + worker_id, no_register, + model_path, model_base, model_name, + load_8bit, load_4bit, device, use_flash_attn=False): + self.controller_addr = controller_addr + self.worker_addr = worker_addr + self.worker_id = worker_id + if model_path.endswith("/"): + model_path = model_path[:-1] + if model_name is None: + model_paths = model_path.split("/") + if model_paths[-1].startswith('checkpoint-'): + self.model_name = model_paths[-2] + "_" + model_paths[-1] + else: + self.model_name = model_paths[-1] + else: + self.model_name = model_name + + self.device = device + logger.info(f"Loading the model {self.model_name} on worker {worker_id} ...") + self.tokenizer, self.model, self.image_processor, self.context_len = load_pretrained_model( + model_path, model_base, self.model_name, load_8bit, load_4bit, device=self.device, use_flash_attn=use_flash_attn) + self.is_multimodal = 'llava' in self.model_name.lower() + + if not no_register: + self.register_to_controller() + self.heart_beat_thread = threading.Thread( + target=heart_beat_worker, args=(self,), daemon=True) + self.heart_beat_thread.start() + + def register_to_controller(self): + logger.info("Register to controller") + + url = self.controller_addr + "/register_worker" + data = { + "worker_name": self.worker_addr, + "check_heart_beat": True, + "worker_status": self.get_status() + } + r = requests.post(url, json=data) + assert r.status_code == 200 + + def send_heart_beat(self): + logger.info(f"Send heart beat. Models: {[self.model_name]}. " + f"Semaphore: {pretty_print_semaphore(model_semaphore)}. " + f"global_counter: {global_counter}") + + url = self.controller_addr + "/receive_heart_beat" + + while True: + try: + ret = requests.post(url, json={ + "worker_name": self.worker_addr, + "queue_length": self.get_queue_length()}, timeout=5) + exist = ret.json()["exist"] + break + except requests.exceptions.RequestException as e: + logger.error(f"heart beat error: {e}") + time.sleep(5) + + if not exist: + self.register_to_controller() + + def get_queue_length(self): + if model_semaphore is None: + return 0 + else: + return args.limit_model_concurrency - model_semaphore._value + (len( + model_semaphore._waiters) if model_semaphore._waiters is not None else 0) + + def get_status(self): + return { + "model_names": [self.model_name], + "speed": 1, + "queue_length": self.get_queue_length(), + } + + @torch.inference_mode() + def generate_stream(self, params): + tokenizer, model, image_processor = self.tokenizer, self.model, self.image_processor + + prompt = params["prompt"] + ori_prompt = prompt + images = params.get("images", None) + num_image_tokens = 0 + if images is not None and len(images) > 0 and self.is_multimodal: + if len(images) > 0: + if len(images) != prompt.count(DEFAULT_IMAGE_TOKEN): + raise ValueError("Number of images does not match number of tokens in prompt") + + images = [load_image_from_base64(image) for image in images] + image_sizes = [image.size for image in images] + images = process_images(images, image_processor, model.config) + + if type(images) is list: + images = [image.to(self.model.device, dtype=torch.float16) for image in images] + else: + images = images.to(self.model.device, dtype=torch.float16) + + replace_token = DEFAULT_IMAGE_TOKEN + if getattr(self.model.config, 'mm_use_im_start_end', False): + replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN + prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token) + + num_image_tokens = prompt.count(replace_token) * model.get_vision_tower().num_patches + else: + images = None + image_sizes = None + image_args = {"images": images, "image_sizes": image_sizes} + else: + images = None + image_args = {} + + temperature = float(params.get("temperature", 1.0)) + top_p = float(params.get("top_p", 1.0)) + max_context_length = getattr(model.config, 'max_position_embeddings', 2048) + max_new_tokens = min(int(params.get("max_new_tokens", 256)), 1024) + stop_str = params.get("stop", None) + do_sample = True if temperature > 0.001 else False + + input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(self.device) + keywords = [stop_str] + # stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids) + streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True, timeout=15) + + max_new_tokens = min(max_new_tokens, max_context_length - input_ids.shape[-1] - num_image_tokens) + + if max_new_tokens < 1: + yield json.dumps({"text": ori_prompt + "Exceeds max token length. Please start a new conversation, thanks.", "error_code": 0}).encode() + b"\0" + return + + thread = Thread(target=model.generate, kwargs=dict( + inputs=input_ids, + do_sample=do_sample, + temperature=temperature, + top_p=top_p, + max_new_tokens=max_new_tokens, + streamer=streamer, + use_cache=True, + **image_args + )) + thread.start() + + generated_text = ori_prompt + for new_text in streamer: + generated_text += new_text + if generated_text.endswith(stop_str): + generated_text = generated_text[:-len(stop_str)] + yield json.dumps({"text": generated_text, "error_code": 0}).encode() + b"\0" + + def generate_stream_gate(self, params): + try: + for x in self.generate_stream(params): + yield x + except ValueError as e: + print("Caught ValueError:", e) + ret = { + "text": server_error_msg, + "error_code": 1, + } + yield json.dumps(ret).encode() + b"\0" + except torch.cuda.CudaError as e: + print("Caught torch.cuda.CudaError:", e) + ret = { + "text": server_error_msg, + "error_code": 1, + } + yield json.dumps(ret).encode() + b"\0" + except Exception as e: + print("Caught Unknown Error", e) + ret = { + "text": server_error_msg, + "error_code": 1, + } + yield json.dumps(ret).encode() + b"\0" + + +app = FastAPI() + + +def release_model_semaphore(fn=None): + model_semaphore.release() + if fn is not None: + fn() + + +@app.post("/worker_generate_stream") +async def generate_stream(request: Request): + global model_semaphore, global_counter + global_counter += 1 + params = await request.json() + + if model_semaphore is None: + model_semaphore = asyncio.Semaphore(args.limit_model_concurrency) + await model_semaphore.acquire() + worker.send_heart_beat() + generator = worker.generate_stream_gate(params) + background_tasks = BackgroundTasks() + background_tasks.add_task(partial(release_model_semaphore, fn=worker.send_heart_beat)) + return StreamingResponse(generator, background=background_tasks) + + +@app.post("/worker_get_status") +async def get_status(request: Request): + return worker.get_status() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=21002) + parser.add_argument("--worker-address", type=str, + default="http://localhost:21002") + parser.add_argument("--controller-address", type=str, + default="http://localhost:21001") + parser.add_argument("--model-path", type=str, default="facebook/opt-350m") + parser.add_argument("--model-base", type=str, default=None) + parser.add_argument("--model-name", type=str) + parser.add_argument("--device", type=str, default="cuda") + parser.add_argument("--multi-modal", action="store_true", help="Multimodal mode is automatically detected with model name, please make sure `llava` is included in the model path.") + parser.add_argument("--limit-model-concurrency", type=int, default=5) + parser.add_argument("--stream-interval", type=int, default=1) + parser.add_argument("--no-register", action="store_true") + parser.add_argument("--load-8bit", action="store_true") + parser.add_argument("--load-4bit", action="store_true") + parser.add_argument("--use-flash-attn", action="store_true") + args = parser.parse_args() + logger.info(f"args: {args}") + + if args.multi_modal: + logger.warning("Multimodal mode is automatically detected with model name, please make sure `llava` is included in the model path.") + + worker = ModelWorker(args.controller_address, + args.worker_address, + worker_id, + args.no_register, + args.model_path, + args.model_base, + args.model_name, + args.load_8bit, + args.load_4bit, + args.device, + use_flash_attn=args.use_flash_attn) + uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/register_worker.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/register_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..2c2c40295e0351f25709ba25554c9329f15bf0d2 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/register_worker.py @@ -0,0 +1,26 @@ +""" +Manually register workers. + +Usage: +python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name http://localhost:21002 +""" + +import argparse + +import requests + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--controller-address", type=str) + parser.add_argument("--worker-name", type=str) + parser.add_argument("--check-heart-beat", action="store_true") + args = parser.parse_args() + + url = args.controller_address + "/register_worker" + data = { + "worker_name": args.worker_name, + "check_heart_beat": args.check_heart_beat, + "worker_status": None, + } + r = requests.post(url, json=data) + assert r.status_code == 200 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/sglang_worker.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/sglang_worker.py new file mode 100644 index 0000000000000000000000000000000000000000..a3297b7c295abddedfaac7f6fbe882d7b672487d --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/sglang_worker.py @@ -0,0 +1,244 @@ +""" +A model worker executes the model. +""" +import argparse +import asyncio +from concurrent.futures import ThreadPoolExecutor +import json +import time +import threading +import uuid + +from fastapi import FastAPI, Request, BackgroundTasks +from fastapi.responses import StreamingResponse +import requests +import re +import uvicorn +from functools import partial + +from llava.constants import WORKER_HEART_BEAT_INTERVAL +from llava.utils import (build_logger, server_error_msg, + pretty_print_semaphore) +from llava.mm_utils import process_images, load_image_from_base64, tokenizer_image_token, expand2square +from llava.constants import DEFAULT_IMAGE_TOKEN + +import sglang as sgl +from sglang.backend.runtime_endpoint import RuntimeEndpoint + + +GB = 1 << 30 + +worker_id = str(uuid.uuid4())[:6] +logger = build_logger("model_worker", f"model_worker_{worker_id}.log") +global_counter = 0 + +model_semaphore = None + + +def heart_beat_worker(controller): + while True: + time.sleep(WORKER_HEART_BEAT_INTERVAL) + controller.send_heart_beat() + + +@sgl.function +def pipeline(s, prompt, max_tokens): + for p in prompt: + if type(p) is str: + s += p + else: + s += sgl.image(p) + s += sgl.gen("response", max_tokens=max_tokens) + + +class ModelWorker: + def __init__(self, controller_addr, worker_addr, sgl_endpoint, + worker_id, no_register, model_name): + self.controller_addr = controller_addr + self.worker_addr = worker_addr + self.worker_id = worker_id + + # Select backend + backend = RuntimeEndpoint(sgl_endpoint) + sgl.set_default_backend(backend) + model_path = backend.model_info["model_path"] + + if model_path.endswith("/"): + model_path = model_path[:-1] + if model_name is None: + model_paths = model_path.split("/") + if model_paths[-1].startswith('checkpoint-'): + self.model_name = model_paths[-2] + "_" + model_paths[-1] + else: + self.model_name = model_paths[-1] + else: + self.model_name = model_name + + logger.info(f"Loading the SGLANG model {self.model_name} on worker {worker_id} ...") + + if not no_register: + self.register_to_controller() + self.heart_beat_thread = threading.Thread( + target=heart_beat_worker, args=(self,), daemon=True) + self.heart_beat_thread.start() + + def register_to_controller(self): + logger.info("Register to controller") + + url = self.controller_addr + "/register_worker" + data = { + "worker_name": self.worker_addr, + "check_heart_beat": True, + "worker_status": self.get_status() + } + r = requests.post(url, json=data) + assert r.status_code == 200 + + def send_heart_beat(self): + logger.info(f"Send heart beat. Models: {[self.model_name]}. " + f"Semaphore: {pretty_print_semaphore(model_semaphore)}. " + f"global_counter: {global_counter}") + + url = self.controller_addr + "/receive_heart_beat" + + while True: + try: + ret = requests.post(url, json={ + "worker_name": self.worker_addr, + "queue_length": self.get_queue_length()}, timeout=5) + exist = ret.json()["exist"] + break + except requests.exceptions.RequestException as e: + logger.error(f"heart beat error: {e}") + time.sleep(5) + + if not exist: + self.register_to_controller() + + def get_queue_length(self): + if model_semaphore is None: + return 0 + else: + return args.limit_model_concurrency - model_semaphore._value + (len( + model_semaphore._waiters) if model_semaphore._waiters is not None else 0) + + def get_status(self): + return { + "model_names": [self.model_name], + "speed": 1, + "queue_length": self.get_queue_length(), + } + + async def generate_stream(self, params): + ori_prompt = prompt = params["prompt"] + images = params.get("images", None) + if images is not None and len(images) > 0: + if len(images) > 0: + if len(images) != prompt.count(DEFAULT_IMAGE_TOKEN): + raise ValueError("Number of images does not match number of tokens in prompt") + + images = [load_image_from_base64(image) for image in images] + + # FIXME: for image-start/end token + # replace_token = DEFAULT_IMAGE_TOKEN + # if getattr(self.model.config, 'mm_use_im_start_end', False): + # replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN + # prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replace_token) + prompt = prompt.replace(' ' + DEFAULT_IMAGE_TOKEN + '\n', DEFAULT_IMAGE_TOKEN) + prompt_split = prompt.split(DEFAULT_IMAGE_TOKEN) + prompt = [] + for i in range(len(prompt_split)): + prompt.append(prompt_split[i]) + if i < len(images): + prompt.append(images[i]) + else: + prompt = [prompt] + + temperature = float(params.get("temperature", 1.0)) + top_p = float(params.get("top_p", 1.0)) + # max_context_length = getattr(model.config, 'max_position_embeddings', 2048) + max_new_tokens = min(int(params.get("max_new_tokens", 256)), 1024) + stop_str = params.get("stop", None) + stop_str = [stop_str] if stop_str is not None else None + + print({'prompt': prompt, 'max_new_tokens': max_new_tokens, 'temperature': temperature, 'top_p': top_p}) + state = pipeline.run(prompt, max_new_tokens, temperature=temperature, top_p=top_p, stream=True) + + generated_text = ori_prompt + async for text_outputs in state.text_async_iter(var_name="response"): + generated_text += text_outputs + yield json.dumps({"text": generated_text, "error_code": 0}).encode() + b"\0" + + async def generate_stream_gate(self, params): + try: + async for x in self.generate_stream(params): + yield x + except ValueError as e: + print("Caught ValueError:", e) + ret = { + "text": server_error_msg, + "error_code": 1, + } + yield json.dumps(ret).encode() + b"\0" + except Exception as e: + print("Caught Unknown Error", e) + ret = { + "text": server_error_msg, + "error_code": 1, + } + yield json.dumps(ret).encode() + b"\0" + + +app = FastAPI() + + +def release_model_semaphore(fn=None): + model_semaphore.release() + if fn is not None: + fn() + + +@app.post("/worker_generate_stream") +async def generate_stream(request: Request): + global model_semaphore, global_counter + global_counter += 1 + params = await request.json() + + if model_semaphore is None: + model_semaphore = asyncio.Semaphore(args.limit_model_concurrency) + await model_semaphore.acquire() + worker.send_heart_beat() + generator = worker.generate_stream_gate(params) + background_tasks = BackgroundTasks() + background_tasks.add_task(partial(release_model_semaphore, fn=worker.send_heart_beat)) + return StreamingResponse(generator, background=background_tasks) + + +@app.post("/worker_get_status") +async def get_status(request: Request): + return worker.get_status() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=21002) + parser.add_argument("--worker-address", type=str, + default="http://localhost:21002") + parser.add_argument("--controller-address", type=str, + default="http://localhost:21001") + parser.add_argument("--model-name", type=str) + parser.add_argument("--sgl-endpoint", type=str) + parser.add_argument("--limit-model-concurrency", type=int, default=5) + parser.add_argument("--stream-interval", type=int, default=1) + parser.add_argument("--no-register", action="store_true") + args = parser.parse_args() + logger.info(f"args: {args}") + + worker = ModelWorker(args.controller_address, + args.worker_address, + args.sgl_endpoint, + worker_id, + args.no_register, + args.model_name) + uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/test_message.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/test_message.py new file mode 100644 index 0000000000000000000000000000000000000000..6b090faed0e630b03b2294545050f1f4f5032cad --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/serve/test_message.py @@ -0,0 +1,62 @@ +import argparse +import json + +import requests + +from llava.conversation import default_conversation + + +def main(): + if args.worker_address: + worker_addr = args.worker_address + else: + controller_addr = args.controller_address + ret = requests.post(controller_addr + "/refresh_all_workers") + ret = requests.post(controller_addr + "/list_models") + models = ret.json()["models"] + models.sort() + print(f"Models: {models}") + + ret = requests.post(controller_addr + "/get_worker_address", + json={"model": args.model_name}) + worker_addr = ret.json()["address"] + print(f"worker_addr: {worker_addr}") + + if worker_addr == "": + return + + conv = default_conversation.copy() + conv.append_message(conv.roles[0], args.message) + prompt = conv.get_prompt() + + headers = {"User-Agent": "LLaVA Client"} + pload = { + "model": args.model_name, + "prompt": prompt, + "max_new_tokens": args.max_new_tokens, + "temperature": 0.7, + "stop": conv.sep, + } + response = requests.post(worker_addr + "/worker_generate_stream", headers=headers, + json=pload, stream=True) + + print(prompt.replace(conv.sep, "\n"), end="") + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"): + if chunk: + data = json.loads(chunk.decode("utf-8")) + output = data["text"].split(conv.sep)[-1] + print(output, end="\r") + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--controller-address", type=str, default="http://localhost:21001") + parser.add_argument("--worker-address", type=str) + parser.add_argument("--model-name", type=str, default="facebook/opt-350m") + parser.add_argument("--max-new-tokens", type=int, default=32) + parser.add_argument("--message", type=str, default= + "Tell me a story with more than 1000 words.") + args = parser.parse_args() + + main() diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/utils.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4006cf917e26c365080b0844c56fab78c48457c0 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/llava/utils.py @@ -0,0 +1,126 @@ +import datetime +import logging +import logging.handlers +import os +import sys + +import requests + +from llava.constants import LOGDIR + +server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**" +moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN." + +handler = None + + +def build_logger(logger_name, logger_filename): + global handler + + formatter = logging.Formatter( + fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + # Set the format of root handlers + if not logging.getLogger().handlers: + logging.basicConfig(level=logging.INFO) + logging.getLogger().handlers[0].setFormatter(formatter) + + # Redirect stdout and stderr to loggers + stdout_logger = logging.getLogger("stdout") + stdout_logger.setLevel(logging.INFO) + sl = StreamToLogger(stdout_logger, logging.INFO) + sys.stdout = sl + + stderr_logger = logging.getLogger("stderr") + stderr_logger.setLevel(logging.ERROR) + sl = StreamToLogger(stderr_logger, logging.ERROR) + sys.stderr = sl + + # Get logger + logger = logging.getLogger(logger_name) + logger.setLevel(logging.INFO) + + # Add a file handler for all loggers + if handler is None: + os.makedirs(LOGDIR, exist_ok=True) + filename = os.path.join(LOGDIR, logger_filename) + handler = logging.handlers.TimedRotatingFileHandler( + filename, when='D', utc=True, encoding='UTF-8') + handler.setFormatter(formatter) + + for name, item in logging.root.manager.loggerDict.items(): + if isinstance(item, logging.Logger): + item.addHandler(handler) + + return logger + + +class StreamToLogger(object): + """ + Fake file-like stream object that redirects writes to a logger instance. + """ + def __init__(self, logger, log_level=logging.INFO): + self.terminal = sys.stdout + self.logger = logger + self.log_level = log_level + self.linebuf = '' + + def __getattr__(self, attr): + return getattr(self.terminal, attr) + + def write(self, buf): + temp_linebuf = self.linebuf + buf + self.linebuf = '' + for line in temp_linebuf.splitlines(True): + # From the io.TextIOWrapper docs: + # On output, if newline is None, any '\n' characters written + # are translated to the system default line separator. + # By default sys.stdout.write() expects '\n' newlines and then + # translates them so this is still cross platform. + if line[-1] == '\n': + self.logger.log(self.log_level, line.rstrip()) + else: + self.linebuf += line + + def flush(self): + if self.linebuf != '': + self.logger.log(self.log_level, self.linebuf.rstrip()) + self.linebuf = '' + + +def disable_torch_init(): + """ + Disable the redundant torch default initialization to accelerate model creation. + """ + import torch + setattr(torch.nn.Linear, "reset_parameters", lambda self: None) + setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None) + + +def violates_moderation(text): + """ + Check whether the text violates OpenAI moderation API. + """ + url = "https://api.openai.com/v1/moderations" + headers = {"Content-Type": "application/json", + "Authorization": "Bearer " + os.environ["OPENAI_API_KEY"]} + text = text.replace("\n", "") + data = "{" + '"input": ' + f'"{text}"' + "}" + data = data.encode("utf-8") + try: + ret = requests.post(url, headers=headers, data=data, timeout=5) + flagged = ret.json()["results"][0]["flagged"] + except requests.exceptions.RequestException as e: + flagged = False + except KeyError as e: + flagged = False + + return flagged + + +def pretty_print_semaphore(semaphore): + if semaphore is None: + return "None" + return f"Semaphore(value={semaphore._value}, locked={semaphore.locked()})" diff --git a/PyTorch/built-in/mlm/HunyuanDiT/mllm/make_csv.py b/PyTorch/built-in/mlm/HunyuanDiT/mllm/make_csv.py new file mode 100644 index 0000000000000000000000000000000000000000..66678090997df93a97b559e608952d06b5765299 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/mllm/make_csv.py @@ -0,0 +1,15 @@ +import os +import argparse +import pandas as pd + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--img_dir', type=str, default='mllm/images') + parser.add_argument('--input_file', type=str, default='mllm/images/demo.csv') + args = parser.parse_args() + + df = pd.DataFrame(columns=['img_path']) + df['img_path'] = [os.path.join(args.img_dir, fn) for fn in os.listdir(args.img_dir) if fn.endswith(".png")] + + df.to_csv(args.input_file, index=False) + print("csv file saved to: ", args.input_file) \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/npu_sample.sh b/PyTorch/built-in/mlm/HunyuanDiT/npu_sample.sh new file mode 100644 index 0000000000000000000000000000000000000000..df4d8f0fa7c2c440e622a217f13e4a57517d5a14 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/npu_sample.sh @@ -0,0 +1,8 @@ +source /home/l50041210/cann-b20/ascend-toolkit/set_env.sh +export ASCEND_RT_VISIBLE_DEVICES=0 +#将Host日志输出到串口,0-关闭/1-开启 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +#设置默认日志级别,0-debug/1-info/2-warning/3-error +export ASCEND_GLOBAL_LOG_LEVEL=3 + +python sample_t2i.py --infer-mode fa --prompt "渔舟唱晚" --image-size 1280 768 --no-enhance \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/npu_train.sh b/PyTorch/built-in/mlm/HunyuanDiT/npu_train.sh new file mode 100644 index 0000000000000000000000000000000000000000..57e991e0dc377a14ab7af0f5b3db3763b123b3cb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/npu_train.sh @@ -0,0 +1,7 @@ +source /home/l50041210/cann-b020/ascend-toolkit/set_env.sh +#将Host日志输出到串口,0-关闭/1-开启 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +#设置默认日志级别,0-debug/1-info/2-warning/3-error +export ASCEND_GLOBAL_LOG_LEVEL=3 + +PYTHONPATH=./ sh hydit/train.sh \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/requirements.txt b/PyTorch/built-in/mlm/HunyuanDiT/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8de35fd32eb220912bf782b94dad64beb2776e20 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/requirements.txt @@ -0,0 +1,22 @@ +--extra-index-url https://pypi.ngc.nvidia.com +timm==0.9.5 +diffusers==0.21.2 +peft==0.10.0 +protobuf==3.19.0 +torchvision==0.14.1 +transformers==4.39.1 +peft==0.10.0 +accelerate==0.29.3 +loguru==0.7.2 +einops==0.7.0 +sentencepiece==0.1.99 +cuda-python==11.7.1 +onnxruntime==1.12.1 +onnx==1.12.0 +nvidia-pyindex==1.0.9 +onnx-graphsurgeon==0.3.27 +polygraphy==0.47.1 +pandas==2.0.3 +gradio==3.50.2 +deepspeed==0.6.3 +pyarrow==16.1.0 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/sample_controlnet.py b/PyTorch/built-in/mlm/HunyuanDiT/sample_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..c70d3eede8e586e23327004dd62fdfbae08349f3 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/sample_controlnet.py @@ -0,0 +1,89 @@ +from pathlib import Path + +from loguru import logger + +from mllm.dialoggen_demo import DialogGen +from hydit.config import get_args +from hydit.inference_controlnet import End2End + +from torchvision import transforms as T +import numpy as np + +norm_transform = T.Compose( + [ + T.ToTensor(), + T.Normalize([0.5], [0.5]), + ] + ) + +from PIL import Image +def inferencer(): + args = get_args() + models_root_path = Path(args.model_root) + if not models_root_path.exists(): + raise ValueError(f"`models_root` not exists: {models_root_path}") + + # Load models + gen = End2End(args, models_root_path) + + # Try to enhance prompt + if args.enhance: + logger.info("Loading DialogGen model (for prompt enhancement)...") + enhancer = DialogGen(str(models_root_path / "dialoggen"), args.load_4bit) + logger.info("DialogGen model loaded.") + else: + enhancer = None + + return args, gen, enhancer + + +if __name__ == "__main__": + + args, gen, enhancer = inferencer() + + if enhancer: + logger.info("Prompt Enhancement...") + success, enhanced_prompt = enhancer(args.prompt) + if not success: + logger.info("Sorry, the prompt is not compliant, refuse to draw.") + exit() + logger.info(f"Enhanced prompt: {enhanced_prompt}") + else: + enhanced_prompt = None + + # Run inference + logger.info("Generating images...") + height, width = args.image_size + + condition = Image.open(args.condition_image_path).convert('RGB').resize((width, height)) + image = norm_transform(condition) + image = image.unsqueeze(0).cuda() + + results = gen.predict(args.prompt, + height=height, + width=width, + image=image, + seed=args.seed, + enhanced_prompt=enhanced_prompt, + negative_prompt=args.negative, + infer_steps=args.infer_steps, + guidance_scale=args.cfg_scale, + batch_size=args.batch_size, + src_size_cond=args.size_cond, + ) + images = results['images'] + + # Save images + save_dir = Path('results') + save_dir.mkdir(exist_ok=True) + # Find the first available index + all_files = list(save_dir.glob('*.png')) + if all_files: + start = max([int(f.stem) for f in all_files]) + 1 + else: + start = 0 + + for idx, pil_img in enumerate(images): + save_path = save_dir / f"{idx + start}.png" + pil_img.save(save_path) + logger.info(f"Save to {save_path}") diff --git a/PyTorch/built-in/mlm/HunyuanDiT/sample_t2i.py b/PyTorch/built-in/mlm/HunyuanDiT/sample_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..1575fe980e522a887fbc2a73d67c65a4430978f7 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/sample_t2i.py @@ -0,0 +1,72 @@ +from pathlib import Path + +from loguru import logger + +from mllm.dialoggen_demo import DialogGen +from hydit.config import get_args +from hydit.inference import End2End + + +def inferencer(): + args = get_args() + models_root_path = Path(args.model_root) + if not models_root_path.exists(): + raise ValueError(f"`models_root` not exists: {models_root_path}") + + # Load models + gen = End2End(args, models_root_path) + + # Try to enhance prompt + if args.enhance: + logger.info("Loading DialogGen model (for prompt enhancement)...") + enhancer = DialogGen(str(models_root_path / "dialoggen"), args.load_4bit) + logger.info("DialogGen model loaded.") + else: + enhancer = None + + return args, gen, enhancer + + +if __name__ == "__main__": + args, gen, enhancer = inferencer() + + if enhancer: + logger.info("Prompt Enhancement...") + success, enhanced_prompt = enhancer(args.prompt) + if not success: + logger.info("Sorry, the prompt is not compliant, refuse to draw.") + exit() + logger.info(f"Enhanced prompt: {enhanced_prompt}") + else: + enhanced_prompt = None + + # Run inference + logger.info("Generating images...") + height, width = args.image_size + results = gen.predict(args.prompt, + height=height, + width=width, + seed=args.seed, + enhanced_prompt=enhanced_prompt, + negative_prompt=args.negative, + infer_steps=args.infer_steps, + guidance_scale=args.cfg_scale, + batch_size=args.batch_size, + src_size_cond=args.size_cond, + ) + images = results['images'] + + # Save images + save_dir = Path('results') + save_dir.mkdir(exist_ok=True) + # Find the first available index + all_files = list(save_dir.glob('*.png')) + if all_files: + start = max([int(f.stem) for f in all_files]) + 1 + else: + start = 0 + + for idx, pil_img in enumerate(images): + save_path = save_dir / f"{idx + start}.png" + pil_img.save(save_path) + logger.info(f"Save to {save_path}") diff --git a/PyTorch/built-in/mlm/HunyuanDiT/test/env_npu.sh b/PyTorch/built-in/mlm/HunyuanDiT/test/env_npu.sh new file mode 100644 index 0000000000000000000000000000000000000000..cd24c60a82fa5e910db8306e559579206d7465d9 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/test/env_npu.sh @@ -0,0 +1,36 @@ +#!/bin/bash +CANN_INSTALL_PATH_CONF='/etc/Ascend/ascend_cann_install.info' + +if [ -f $CANN_INSTALL_PATH_CONF ]; then + CANN_INSTALL_PATH=$(cat $CANN_INSTALL_PATH_CONF | grep Install_Path | cut -d "=" -f 2) +else + CANN_INSTALL_PATH="/usr/local/Ascend" +fi + +if [ -d ${CANN_INSTALL_PATH}/ascend-toolkit/latest ]; then + source ${CANN_INSTALL_PATH}/ascend-toolkit/set_env.sh +else + source ${CANN_INSTALL_PATH}/nnae/set_env.sh +fi + +count=$(npu-smi info -l | grep -c "NPU ID") + +for ((i=0; i<${count}; i=i+1)) +do + msnpureport -g error -d ${i} +done + +#将Host日志输出到串口,0-关闭/1-开启 +export ASCEND_SLOG_PRINT_TO_STDOUT=0 +#设置默认日志级别,0-debug/1-info/2-warning/3-error +export ASCEND_GLOBAL_LOG_LEVEL=3 +#设置Event日志开启标志,0-关闭/1-开启 +export ASCEND_GLOBAL_EVENT_ENABLE=0 +#设置是否开启taskque,0-关闭/1-开启 +export TASK_QUEUE_ENABLE=1 +#设置是否开启combined标志,0-关闭/1-开启 +export COMBINED_ENABLE=1 +#HCCL白名单开关,1-关闭/0-开启 +export HCCL_WHITELIST_DISABLE=1 +export HCCL_IF_IP=$(hostname -I |awk '{print $1}') +export HCCL_CONNECT_TIMEOUT=1200 \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/test/train_full_8p_bf16.sh b/PyTorch/built-in/mlm/HunyuanDiT/test/train_full_8p_bf16.sh new file mode 100644 index 0000000000000000000000000000000000000000..596ac19052029386566f1a0b10b81191da787a13 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/test/train_full_8p_bf16.sh @@ -0,0 +1,126 @@ +# 微调生成的ckpt路径 +Network="HunyuanDiT" +BATCH_SIZE=1 +max_train_steps=5000 +task_flag="dit_g2_full_1024p" # the task flag is used to identify folders. +resume=./ckpts/t2i/model/ # checkpoint root for resume +index_file=dataset/porcelain/jsons/porcelain_mt.json # index file for dataloader +results_dir=./log_EXP # save root for results +image_size=1024 # training image resolution +grad_accu_steps=1 # gradient accumulation +warmup_num_steps=0 # warm-up steps +lr=0.0001 # learning rate +ckpt_every=10000 # create a ckpt every a few steps. +ckpt_latest_every=5000 # create a ckpt named `latest.pt` every a few steps. + +export WORLD_SIZE=8 +export MASTER_PORT=29500 +export MASTER_ADDR=127.0.0.1 + +for para in $* +do + if [[ $para == --batch_size* ]]; then + BATCH_SIZE=$(echo ${para#*=}) + elif [[ $para == --max_train_steps* ]]; then + max_train_steps=$(echo ${para#*=}) + fi +done + +# cd到与test文件夹同层级目录下执行脚本，提高兼容性；test_path_dir为包含test文件夹的路径 +cur_path=$(pwd) +cur_path_last_dirname=${cur_path##*/} +if [ x"${cur_path_last_dirname}" == x"test" ]; then + test_path_dir=${cur_path} + cd .. + cur_path=$(pwd) +else + test_path_dir=${cur_path}/test +fi + +source ${test_path_dir}/env_npu.sh + +#创建DeviceID输出目录，不需要修改 +output_path=${cur_path}/test/output/${ASCEND_DEVICE_ID} + +mkdir -p ${output_path} + +#训练开始时间，不需要修改 +start_time=$(date +%s) +echo "start_time: ${start_time}" + +model='DiT-g/2' +params=" \ + --qk-norm \ + --model ${model} \ + --rope-img base512 \ + --rope-real \ + " +deepspeed --num_gpus ${WORLD_SIZE} --num_nodes 1 --master_port=${MASTER_PORT} hydit/train_deepspeed.py ${params} \ + --task-flag ${task_flag} \ + --noise-schedule scaled_linear --beta-start 0.00085 --beta-end 0.03 \ + --predict-type v_prediction \ + --uncond-p 0.44 \ + --uncond-p-t5 0.44 \ + --index-file ${index_file} \ + --random-flip \ + --lr ${lr} \ + --batch-size ${BATCH_SIZE} \ + --image-size ${image_size} \ + --global-seed 999 \ + --grad-accu-steps ${grad_accu_steps} \ + --warmup-num-steps ${warmup_num_steps} \ + --use-flash-attn \ + --use-fp16 \ + --use-ema \ + --ema-dtype fp32 \ + --results-dir ${results_dir} \ + --resume-split \ + --resume ${resume} \ + --ckpt-every ${ckpt_every} \ + --ckpt-latest-every ${ckpt_latest_every} \ + --log-every 1 \ + --deepspeed \ + --deepspeed-optimizer \ + --use-zero-stage 2 \ + --multireso \ + --reso-step 64 \ + --max-training-steps ${max_train_steps} \ + > ${test_path_dir}/output/$ASCEND_DEVICE_ID/train_${ASCEND_DEVICE_ID}.log 2>&1 & + +wait + +#训练结束时间，不需要修改 +end_time=$(date +%s) +e2e_time=$(($end_time - $start_time)) + + +# 训练用例信息，不需要修改 +BatchSize=${BATCH_SIZE} +DeviceType=$(uname -m) +CaseName=${Network}_bs${BatchSize}_${WORLD_SIZE}'p'_'acc' + +# 结果打印，不需要修改 +echo "------------------ Final result ------------------" +# 输出性能FPS，需要模型审视修改 +FPS=`grep -a 'FPS' ${test_path_dir}/output/${ASCEND_DEVICE_ID}/train_${ASCEND_DEVICE_ID}.log|awk -F "FPS " '{print $2}' | tail -100 | awk '{a+=$1} END {if (NR != 0) printf("%.3f",a/NR)}'` +# 打印，不需要修改 +echo "Final Performance images/sec : $FPS" +echo "E2E Training Duration sec : $e2e_time" + + +# 性能看护结果汇总 +# 获取性能数据，不需要修改 +# 吞吐量 +ActualFPS=${FPS} +#单迭代训练时长 +TrainingTime=$(awk 'BEGIN{printf "%.2f\n", '${BATCH_SIZE}'*8/'${FPS}'}') + + +# 关键信息打印到${CaseName}.log中，不需要修改 +echo "Network = ${Network}" >${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "RankSize = ${WORLD_SIZE}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "BatchSize = ${BatchSize}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "DeviceType = ${DeviceType}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "CaseName = ${CaseName}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}.log +echo "TrainingTime = ${TrainingTime}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}_perf_report.log +echo "E2ETrainingTime = ${e2e_time}" >>${test_path_dir}/output/$ASCEND_DEVICE_ID/${CaseName}_perf_report.log \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/trt/build_engine.sh b/PyTorch/built-in/mlm/HunyuanDiT/trt/build_engine.sh new file mode 100644 index 0000000000000000000000000000000000000000..02549cc75558fd184e9a85d5c3ca49d48e00b8ed --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/trt/build_engine.sh @@ -0,0 +1,70 @@ +# ============================================================================== +# Description: Export ONNX model and build TensorRT engine. +# ============================================================================== + +# Check if the model root path is exists or provided. +if [ -z "$1" ]; then + if [ -d "ckpts" ]; then + echo "The model root directory is not provided. Use the default path 'ckpts'." + export MODEL_ROOT=ckpts + else + echo "Default model path 'ckpts' does not exist. Please provide the path of the model root directory." + exit 1 + fi +elif [ ! -d "$1" ]; then + echo "The model root directory ($1) does not exist." + exit 1 +else + export MODEL_ROOT=$(cd "$1"; pwd) +fi + +export ONNX_WORKDIR=${MODEL_ROOT}/onnx_model +echo "MODEL_ROOT=${MODEL_ROOT}" +echo "ONNX_WORKDIR=${ONNX_WORKDIR}" + +# Remove old directories. +if [ -d "${ONNX_WORKDIR}" ]; then + echo "Remove old ONNX directories..." + rm -r ${ONNX_WORKDIR} +fi + +# Inspect the project directory. +SCRIPT_PATH="$( cd "$( dirname "$0" )" && pwd )" +PROJECT_DIR=$(dirname "$SCRIPT_PATH") +export PYTHONPATH=${PROJECT_DIR}:${PYTHONPATH} +echo "PYTHONPATH=${PYTHONPATH}" +cd ${PROJECT_DIR} +echo "Change directory to ${PROJECT_DIR}" + +# ---------------------------------------- +# 1. Export ONNX model. +# ---------------------------------------- + +# Sleep for reading the message. +sleep 2s + +echo "Exporting ONNX model..." +python trt/export_onnx.py --model-root ${MODEL_ROOT} --onnx-workdir ${ONNX_WORKDIR} +echo "Exporting ONNX model finished" + +# ---------------------------------------- +# 2. Build TensorRT engine. +# ---------------------------------------- + +echo "Building TensorRT engine..." +ENGINE_DIR="${MODEL_ROOT}/t2i/model_trt/engine" +mkdir -p ${ENGINE_DIR} +ENGINE_PATH=${ENGINE_DIR}/model_onnx.plan +PLUGIN_PATH=${MODEL_ROOT}/t2i/model_trt/fmha_plugins/9.2_plugin_cuda11/fMHAPlugin.so + +trtexec \ + --onnx=${ONNX_WORKDIR}/export_modified_fmha/model.onnx \ + --fp16 \ + --saveEngine=${ENGINE_PATH} \ + --minShapes=x:2x4x90x90,t:2,encoder_hidden_states:2x77x1024,text_embedding_mask:2x77,encoder_hidden_states_t5:2x256x2048,text_embedding_mask_t5:2x256,image_meta_size:2x6,style:2,cos_cis_img:2025x88,sin_cis_img:2025x88 \ + --optShapes=x:2x4x128x128,t:2,encoder_hidden_states:2x77x1024,text_embedding_mask:2x77,encoder_hidden_states_t5:2x256x2048,text_embedding_mask_t5:2x256,image_meta_size:2x6,style:2,cos_cis_img:4096x88,sin_cis_img:4096x88 \ + --maxShapes=x:2x4x160x160,t:2,encoder_hidden_states:2x77x1024,text_embedding_mask:2x77,encoder_hidden_states_t5:2x256x2048,text_embedding_mask_t5:2x256,image_meta_size:2x6,style:2,cos_cis_img:6400x88,sin_cis_img:6400x88 \ + --shapes=x:2x4x128x128,t:2,encoder_hidden_states:2x77x1024,text_embedding_mask:2x77,encoder_hidden_states_t5:2x256x2048,text_embedding_mask_t5:2x256,image_meta_size:2x6,style:2,cos_cis_img:4096x88,sin_cis_img:4096x88 \ + --verbose \ + --builderOptimizationLevel=4 \ + --staticPlugins=${PLUGIN_PATH} diff --git a/PyTorch/built-in/mlm/HunyuanDiT/trt/export_onnx.py b/PyTorch/built-in/mlm/HunyuanDiT/trt/export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..fd2c8f749f55cbcc1f6e61666087bad8406cc2af --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/trt/export_onnx.py @@ -0,0 +1,246 @@ +from pathlib import Path + +import torch +from loguru import logger + +from hydit.config import get_args +from hydit.modules.models import HunYuanDiT, HUNYUAN_DIT_CONFIG + +import numpy as np +import onnx +import onnx_graphsurgeon as gs +import polygraphy.backend.onnx.loader + + +def _to_tuple(val): + if isinstance(val, (list, tuple)): + if len(val) == 1: + val = [val[0], val[0]] + elif len(val) == 2: + val = tuple(val) + else: + raise ValueError(f"Invalid value: {val}") + elif isinstance(val, (int, float)): + val = (val, val) + else: + raise ValueError(f"Invalid value: {val}") + return val + + +class ExportONNX(object): + def __init__(self, args, models_root_path): + self.args = args + self.model = None + # Set device and disable gradient + self.device = "cuda" if torch.cuda.is_available() else "cpu" + torch.set_grad_enabled(False) + + # Check arguments + t2i_root_path = Path(models_root_path) / "t2i" + self.root = t2i_root_path + logger.info(f"Got text-to-image model root path: {t2i_root_path}") + + # Create folder to save onnx model + onnx_workdir = Path(self.args.onnx_workdir) + self.onnx_workdir = onnx_workdir + self.onnx_export = self.onnx_workdir / "export/model.onnx" + self.onnx_export.parent.mkdir(parents=True, exist_ok=True) + self.onnx_modify = self.onnx_workdir / "export_modified/model.onnx" + self.onnx_modify.parent.mkdir(parents=True, exist_ok=True) + self.onnx_fmha = self.onnx_workdir / "export_modified_fmha/model.onnx" + self.onnx_fmha.parent.mkdir(parents=True, exist_ok=True) + + def load_model(self): + # ======================================================================== + # Create model structure and load the checkpoint + logger.info(f"Building HunYuan-DiT model...") + model_config = HUNYUAN_DIT_CONFIG[self.args.model] + image_size = _to_tuple(self.args.image_size) + latent_size = (image_size[0] // 8, image_size[1] // 8) + + model_dir = self.root / "model" + model_path = model_dir / f"pytorch_model_{self.args.load_key}.pt" + if not model_path.exists(): + raise ValueError(f"model_path not exists: {model_path}") + + # Build model structure + self.model = HunYuanDiT(self.args, + input_size=latent_size, + **model_config, + log_fn=logger.info, + ).half().to(self.device) # Force to use fp16 + # Load model checkpoint + logger.info(f"Loading torch model {model_path}...") + state_dict = torch.load(model_path, map_location=lambda storage, loc: storage) + self.model.load_state_dict(state_dict) + self.model.eval() + logger.info(f"Loading torch model finished") + logger.info("==================================================") + logger.info(f" Model is ready. ") + logger.info("==================================================") + + def export(self): + if self.model is None: + self.load_model() + + # Construct model inputs + latent_model_input = torch.randn(2, 4, 128, 128, device=self.device).half() + t_expand = torch.randint(0, 1000, [2], device=self.device).half() + prompt_embeds = torch.randn(2, 77, 1024, device=self.device).half() + attention_mask = torch.randint(0, 2, [2, 77], device=self.device).long() + prompt_embeds_t5 = torch.randn(2, 256, 2048, device=self.device).half() + attention_mask_t5 = torch.randint(0, 2, [2, 256], device=self.device).long() + ims = torch.tensor([[1024, 1024, 1024, 1024, 0, 0], [1024, 1024, 1024, 1024, 0, 0]], device=self.device).half() + style = torch.tensor([0, 0], device=self.device).long() + freqs_cis_img = ( + torch.randn(4096, 88), + torch.randn(4096, 88), + ) + + save_to = self.onnx_export + logger.info(f"Exporting ONNX model {save_to}...") + logger.info(f"Exporting ONNX external data {save_to.parent}...") + model_args = ( + latent_model_input, + t_expand, + prompt_embeds, + attention_mask, + prompt_embeds_t5, + attention_mask_t5, + ims, style, + freqs_cis_img[0], + freqs_cis_img[1] + ) + torch.onnx.export(self.model, + model_args, + str(save_to), + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names=["x", "t", "encoder_hidden_states", "text_embedding_mask", + "encoder_hidden_states_t5", "text_embedding_mask_t5", "image_meta_size", "style", + "cos_cis_img", "sin_cis_img"], + output_names=["output"], + dynamic_axes={"x": {0: "2B", 2: "H", 3: "W"}, "t": {0: "2B"}, + "encoder_hidden_states": {0: "2B"}, + "text_embedding_mask": {0: "2B"}, "encoder_hidden_states_t5": {0: "2B"}, + "text_embedding_mask_t5": {0: "2B"}, + "image_meta_size": {0: "2B"}, "style": {0: "2B"}, "cos_cis_img": {0: "seqlen"}, + "sin_cis_img": {0: "seqlen"}}, + ) + logger.info("Exporting onnx finished") + + def postprocessing(self): + load_from = self.onnx_export + save_to = self.onnx_modify + logger.info(f"Postprocessing ONNX model {load_from}...") + + onnxModel = onnx.load(str(load_from), load_external_data=False) + onnx.load_external_data_for_model(onnxModel, str(load_from.parent)) + graph = gs.import_onnx(onnxModel) + + # ADD GAMMA BETA FOR LN + for node in graph.nodes: + if node.name == "/final_layer/norm_final/LayerNormalization": + constantKernel = gs.Constant("final_layer.norm_final.weight", + np.ascontiguousarray(np.ones((1408,), dtype=np.float16))) + constantBias = gs.Constant("final_layer.norm_final.bias", + np.ascontiguousarray(np.zeros((1408,), dtype=np.float16))) + node.inputs = [node.inputs[0], constantKernel, constantBias] + + graph.cleanup().toposort() + onnx.save(gs.export_onnx(graph.cleanup()), + str(save_to), + save_as_external_data=True, + all_tensors_to_one_file=False, + location=str(save_to.parent), + ) + logger.info(f"Postprocessing ONNX model finished: {save_to}") + + def fuse_attn(self): + load_from = self.onnx_modify + save_to = self.onnx_fmha + logger.info(f"FuseAttn ONNX model {load_from}...") + + onnx_graph = polygraphy.backend.onnx.loader.fold_constants( + onnx.load(str(load_from)), + allow_onnxruntime_shape_inference=True, + ) + graph = gs.import_onnx(onnx_graph) + + cnt = 0 + for node in graph.nodes: + + if node.op == "Softmax" and node.i().op == "MatMul" and node.o().op == "MatMul" and \ + node.o().o().op == "Transpose": + + if "pooler" in node.name: + continue + + if "attn1" in node.name: + matmul_0 = node.i() + transpose = matmul_0.i(1, 0) + transpose.attrs["perm"] = [0, 2, 1, 3] + k = transpose.outputs[0] + q = gs.Variable("transpose_0_v_{}".format(cnt), np.dtype(np.float16)) + transpose_0 = gs.Node("Transpose", "Transpose_0_{}".format(cnt), + attrs={"perm": [0, 2, 1, 3]}, + inputs=[matmul_0.inputs[0]], + outputs=[q]) + graph.nodes.append(transpose_0) + + matmul_1 = node.o() + v = gs.Variable("transpose_1_v_{}".format(cnt), np.dtype(np.float16)) + transpose_1 = gs.Node("Transpose", "Transpose_1_{}".format(cnt), + attrs={"perm": [0, 2, 1, 3]}, + inputs=[matmul_1.inputs[1]], + outputs=[v]) + graph.nodes.append(transpose_1) + + output_variable = node.o().o().outputs[0] + # fMHA_v = gs.Variable("fMHA_v", np.dtype(np.float16)) + fMHA = gs.Node("fMHAPlugin", "fMHAPlugin_1_{}".format(cnt), + # attrs={"scale": 1.0}, + inputs=[q, k, v], + outputs=[output_variable]) + graph.nodes.append(fMHA) + node.o().o().outputs = [] + cnt = cnt + 1 + + elif "attn2" in node.name: + matmul_0 = node.i() + transpose_q = matmul_0.i() + transpose_k = matmul_0.i(1, 0) + matmul_1 = node.o() + transpose_v = matmul_1.i(1, 0) + q = transpose_q.inputs[0] + k = transpose_k.inputs[0] + v = transpose_v.inputs[0] + output_variable = node.o().o().outputs[0] + fMHA = gs.Node("fMHAPlugin", "fMHAPlugin_1_{}".format(cnt), + # attrs={"scale": 1.0}, + inputs=[q, k, v], + outputs=[output_variable]) + graph.nodes.append(fMHA) + node.o().o().outputs = [] + cnt = cnt + 1 + + logger.info("mha count: ", cnt) + + onnx.save(gs.export_onnx(graph.cleanup()), + str(save_to), + save_as_external_data=True, + ) + logger.info(f"FuseAttn ONNX model finished: {save_to}") + + +if __name__ == "__main__": + args = get_args() + models_root_path = Path(args.model_root) + if not models_root_path.exists(): + raise ValueError(f"`models_root` not exists: {models_root_path}") + + exporter = ExportONNX(args, models_root_path) + exporter.export() + exporter.postprocessing() + exporter.fuse_attn() diff --git a/PyTorch/built-in/mlm/HunyuanDiT/trt/install.sh b/PyTorch/built-in/mlm/HunyuanDiT/trt/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..0de9d8fe7876984ef609e71e95f7e2555ed15c78 --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/trt/install.sh @@ -0,0 +1,80 @@ +# ============================================================================== +# Description: Install TensorRT and prepare the environment for TensorRT. +# ============================================================================== + +# ---------------------------------------- +# Check the system, tools and arguments +# ---------------------------------------- + +# Check system. Only support TensorRT on Linux (MacOS is not supported.) +if [ "$(uname)" != "Linux" ]; then + echo "Only support TensorRT on Linux" + exit 1 +fi + +# Check if the model_trt path is provided. If not, use the default path. +if [ -z "$1" ]; then + MODEL_TRT_DIR=$(cd ckpts/t2i/model_trt; pwd) +else + MODEL_TRT_DIR=$(cd "$1"; pwd) +fi + +# Check if the model_trt path exists. +if [ ! -d "${MODEL_TRT_DIR}" ]; then + echo "The model_trt directory (${MODEL_TRT_DIR}) does not exist. Please specify the path by:" + echo " sh trt/install.sh " + exit 1 +fi + +# Check if ldconfig exists. +if [ ! -x "$(command -v ldconfig)" ]; then + echo "ldconfig is not installed. Please install it first." + exit 1 +fi + +export TENSORRT_VERSION='9.2.0.5' +TENSORRT_PACKAGE="${MODEL_TRT_DIR}/TensorRT-${TENSORRT_VERSION}.tar.gz" + +# Check if the TensorRT package is downloaded. +if [ ! -f "${TENSORRT_PACKAGE}" ]; then + echo "The TensorRT package (${TENSORRT_PACKAGE}) does not exist. Please download it first with following steps:" + echo "1. cd HunyuanDiT" + echo "2. huggingface-cli download Tencent-Hunyuan/HunyuanDiT-TensorRT --local-dir ./ckpts/t2i/model_trt" + exit 1 +else + echo "Found TensorRT package: ${TENSORRT_PACKAGE}" +fi + +# ---------------------------------------- +# Start to install TensorRT +# ---------------------------------------- + +# Extract the TensorRT package. +echo "Extracting the TensorRT package..." +tar xf "${TENSORRT_PACKAGE}" -C "${MODEL_TRT_DIR}" +TENSORRT_DIR="${MODEL_TRT_DIR}/TensorRT-${TENSORRT_VERSION}" +echo "Extracting the TensorRT package finished" + +# Add the TensorRT library path to the system library path. +echo "${MODEL_TRT_DIR}/lib/" >> /etc/ld.so.conf.d/nvidia.conf && ldconfig + +# Install the TensorRT Python wheel. +echo "Installing the TensorRT Python wheel..." +# Get python version, e.g., cp38 for Python 3.8; cp310 for Python 3.10 +PYTHON_VERSION=$(python -c 'import sys; print(f"cp{sys.version_info.major}{sys.version_info.minor}")') +python -m pip install --no-cache-dir ${TENSORRT_DIR}/python/tensorrt*-${PYTHON_VERSION}* +echo "Installing the TensorRT Python wheel finished" + +# Prepare activate.sh and deactivate.sh +{ + echo "TENSORRT_DIR=${TENSORRT_DIR}" + echo 'export LD_LIBRARY_PATH=${TENSORRT_DIR}/lib/:$LD_LIBRARY_PATH' + echo 'export LIBRARY_PATH=${TENSORRT_DIR}/lib/:$LIBRARY_PATH' + echo 'export PATH=${TENSORRT_DIR}/bin/:$PATH' +} > $(dirname "$0")/activate.sh +{ + echo "TENSORRT_DIR=${TENSORRT_DIR}" + echo 'export LD_LIBRARY_PATH=${LD_LIBRARY_PATH/${TENSORRT_DIR}\/lib\/:}' + echo 'export LIBRARY_PATH=${LIBRARY_PATH/${TENSORRT_DIR}\/lib\/:}' + echo 'export PATH=${PATH/${TENSORRT_DIR}\/bin\/:}' +} > $(dirname "$0")/deactivate.sh diff --git a/PyTorch/built-in/mlm/HunyuanDiT/utils/__init__.py b/PyTorch/built-in/mlm/HunyuanDiT/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/PyTorch/built-in/mlm/HunyuanDiT/utils/collect_env.py b/PyTorch/built-in/mlm/HunyuanDiT/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..53a71cdcf70b01fbf7bfa6e3def1bf512e0b90bb --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/utils/collect_env.py @@ -0,0 +1,202 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""This file holding some environment constant for sharing by other files.""" +import os +import os.path as osp +import subprocess +import sys +from collections import OrderedDict, defaultdict + +import numpy as np +import torch + + +def is_rocm_pytorch() -> bool: + """Check whether the PyTorch is compiled on ROCm.""" + is_rocm = False + if TORCH_VERSION != 'parrots': + try: + from torch.utils.cpp_extension import ROCM_HOME + is_rocm = True if ((torch.version.hip is not None) and + (ROCM_HOME is not None)) else False + except ImportError: + pass + return is_rocm + +TORCH_VERSION = torch.__version__ + +def get_build_config(): + """Obtain the build information of PyTorch or Parrots.""" + if TORCH_VERSION == 'parrots': + from parrots.config import get_build_info + return get_build_info() + else: + return torch.__config__.show() + +try: + import torch_musa # noqa: F401 + IS_MUSA_AVAILABLE = True +except Exception: + IS_MUSA_AVAILABLE = False + +def is_musa_available() -> bool: + return IS_MUSA_AVAILABLE + +def is_cuda_available() -> bool: + """Returns True if cuda devices exist.""" + return torch.cuda.is_available() + +def _get_cuda_home(): + if TORCH_VERSION == 'parrots': + from parrots.utils.build_extension import CUDA_HOME + else: + if is_rocm_pytorch(): + from torch.utils.cpp_extension import ROCM_HOME + CUDA_HOME = ROCM_HOME + else: + from torch.utils.cpp_extension import CUDA_HOME + return CUDA_HOME + + +def _get_musa_home(): + return os.environ.get('MUSA_HOME') + + +def collect_env(): + """Collect the information of the running environments. + + Returns: + dict: The environment information. The following fields are contained. + + - sys.platform: The variable of ``sys.platform``. + - Python: Python version. + - CUDA available: Bool, indicating if CUDA is available. + - GPU devices: Device type of each GPU. + - CUDA_HOME (optional): The env var ``CUDA_HOME``. + - NVCC (optional): NVCC version. + - GCC: GCC version, "n/a" if GCC is not installed. + - MSVC: Microsoft Virtual C++ Compiler version, Windows only. + - PyTorch: PyTorch version. + - PyTorch compiling details: The output of \ + ``torch.__config__.show()``. + - TorchVision (optional): TorchVision version. + - OpenCV (optional): OpenCV version. + """ + from distutils import errors + + env_info = OrderedDict() + env_info['sys.platform'] = sys.platform + env_info['Python'] = sys.version.replace('\n', '') + + cuda_available = is_cuda_available() + musa_available = is_musa_available() + env_info['CUDA available'] = cuda_available + env_info['MUSA available'] = musa_available + env_info['numpy_random_seed'] = np.random.get_state()[1][0] + + if cuda_available: + devices = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + for name, device_ids in devices.items(): + env_info['GPU ' + ','.join(device_ids)] = name + + CUDA_HOME = _get_cuda_home() + env_info['CUDA_HOME'] = CUDA_HOME + + if CUDA_HOME is not None and osp.isdir(CUDA_HOME): + if CUDA_HOME == '/opt/rocm': + try: + nvcc = osp.join(CUDA_HOME, 'hip/bin/hipcc') + nvcc = subprocess.check_output( + f'"{nvcc}" --version', shell=True) + nvcc = nvcc.decode('utf-8').strip() + release = nvcc.rfind('HIP version:') + build = nvcc.rfind('') + nvcc = nvcc[release:build].strip() + except subprocess.SubprocessError: + nvcc = 'Not Available' + else: + try: + nvcc = osp.join(CUDA_HOME, 'bin/nvcc') + nvcc = subprocess.check_output(f'"{nvcc}" -V', shell=True) + nvcc = nvcc.decode('utf-8').strip() + release = nvcc.rfind('Cuda compilation tools') + build = nvcc.rfind('Build ') + nvcc = nvcc[release:build].strip() + except subprocess.SubprocessError: + nvcc = 'Not Available' + env_info['NVCC'] = nvcc + elif musa_available: + devices = defaultdict(list) + for k in range(torch.musa.device_count()): + devices[torch.musa.get_device_name(k)].append(str(k)) + for name, device_ids in devices.items(): + env_info['GPU ' + ','.join(device_ids)] = name + + MUSA_HOME = _get_musa_home() + env_info['MUSA_HOME'] = MUSA_HOME + + if MUSA_HOME is not None and osp.isdir(MUSA_HOME): + try: + mcc = osp.join(MUSA_HOME, 'bin/mcc') + subprocess.check_output(f'"{mcc}" -v', shell=True) + except subprocess.SubprocessError: + mcc = 'Not Available' + env_info['mcc'] = mcc + try: + # Check C++ Compiler. + # For Unix-like, sysconfig has 'CC' variable like 'gcc -pthread ...', + # indicating the compiler used, we use this to get the compiler name + import io + import sysconfig + cc = sysconfig.get_config_var('CC') + if cc: + cc = osp.basename(cc.split()[0]) + cc_info = subprocess.check_output(f'{cc} --version', shell=True) + env_info['GCC'] = cc_info.decode('utf-8').partition( + '\n')[0].strip() + else: + # on Windows, cl.exe is not in PATH. We need to find the path. + # distutils.ccompiler.new_compiler() returns a msvccompiler + # object and after initialization, path to cl.exe is found. + import locale + import os + from distutils.ccompiler import new_compiler + ccompiler = new_compiler() + ccompiler.initialize() + cc = subprocess.check_output( + f'{ccompiler.cc}', stderr=subprocess.STDOUT, shell=True) + encoding = os.device_encoding( + sys.stdout.fileno()) or locale.getpreferredencoding() + env_info['MSVC'] = cc.decode(encoding).partition('\n')[0].strip() + env_info['GCC'] = 'n/a' + except (subprocess.CalledProcessError, errors.DistutilsPlatformError): + env_info['GCC'] = 'n/a' + except io.UnsupportedOperation as e: + # JupyterLab on Windows changes sys.stdout, which has no `fileno` attr + # Refer to: https://github.com/open-mmlab/mmengine/issues/931 + # TODO: find a solution to get compiler info in Windows JupyterLab, + # while preserving backward-compatibility in other systems. + env_info['MSVC'] = f'n/a, reason: {str(e)}' + + env_info['PyTorch'] = torch.__version__ + env_info['PyTorch compiling details'] = get_build_config() + + try: + import torchvision + env_info['TorchVision'] = torchvision.__version__ + except ModuleNotFoundError: + pass + + try: + import cv2 + env_info['OpenCV'] = cv2.__version__ + except ImportError: + pass + + + return env_info + +if __name__ == '__main__': + for name, val in collect_env().items(): + print(f'{name}: {val}') \ No newline at end of file diff --git a/PyTorch/built-in/mlm/HunyuanDiT/utils/npu_utils.py b/PyTorch/built-in/mlm/HunyuanDiT/utils/npu_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0b8da7b9c077b5eed8654491cf6162a4e087b52c --- /dev/null +++ b/PyTorch/built-in/mlm/HunyuanDiT/utils/npu_utils.py @@ -0,0 +1,76 @@ +import os +import random +from typing import List, Optional +import importlib +from functools import lru_cache + +import math +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor +from torch.optim.optimizer import Optimizer + + +AUTOCAST_MAPPING = { + "bf16": torch.bfloat16, + "fp16": torch.float16, + "fp32": torch.float32, +} + +@lru_cache +def is_npu_available(): + if importlib.util.find_spec("torch") is None or importlib.util.find_spec('torch_npu') is None: + return False + + import torch + import torch_npu + + try: + _ = torch.npu.device_count() + return torch.npu.is_available() + except RuntimeError: + return False + + +if is_npu_available(): + import torch_npu + + +def seed_all(is_gpu=True, seed=1234, mode=False): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.use_deterministic_algorithms(mode) + if is_gpu: + torch.cuda.manual_seed_all(seed) + torch.cuda.manual_seed(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.enable = False + torch.backends.cudnn.benchmark = False + else: + torch_npu.npu.manual_seed_all(seed) + torch_npu.npu.manual_seed(seed) + + +class FlashAttention(torch.nn.Module): + def __init__(self, attention_dropout=0): + super().__init__() + self.attention_dropout = attention_dropout + + def forward(self, query, key, value): + heads = query.shape[2] + attention_mask = None + output = torch_npu.npu_fusion_attention( + query, key, value, heads, input_layout='BSND', + pse=None, + atten_mask=attention_mask, + scale=1.0 / math.sqrt(query.shape[-1]), + pre_tockens=65536, + next_tockens=65536, + keep_prob=1. - self.attention_dropout, + sync=False, + inner_precise=0, + )[0] + return output